@Service
public abstract class FantasyGame {
  private static final Log LOG = LogFactory.getLog(FantasyGame.class);

  protected FantasyTeam fantasyTeam;

  public FantasyGame() {}

  public void tradeForTomorrow() throws Exception {
    tradeForDate(DateUtil.getGameTomorrow());
  }

  public void tradeForDate(Date date) throws Exception {
    BbcLeague bbcLeague = getLeague(date);

    Starters expectedStarters = fantasyTeam.getStrategy().pickStarters(date, bbcLeague);
    Starters actualStarters = tradeForStarters(expectedStarters);

    boolean startersSet = actualStarters.equals(expectedStarters);
    System.out.println("starters look good - " + startersSet);
  }

  public abstract BbcLeague getLeague(Date date) throws IOException;

  public abstract Starters tradeForStarters(Starters starters) throws IOException;
}
Exemple #2
0
/**
 * MDC server
 *
 * @author yjiang
 */
public abstract class MDCServer extends IoHandlerAdapter {

  static final Log log = LogFactory.getLog(MDCServer.class);

  static final AtomicInteger counter = new AtomicInteger(0);

  /** the the max size of a packet, 32KB */
  static int MAX_SIZE = 10240 * 1024; // test

  protected InetSocketAddress address;
  protected Selector selector;
  protected ServerSocketChannel server;
  protected final int PROCESS_NUMBER = 4;
  protected static Configuration _conf;

  protected IoAcceptor acceptor;
  protected boolean isRunning = false;

  protected boolean testKey() {
    String data = UID.random(24);
    byte[] bb = RSA.encode(data.getBytes(), TConn.pub_key);
    if (bb != null) {
      bb = RSA.decode(bb, TConn.pri_key);
      if (bb != null && data.equals(new String(bb))) {
        return true;
      }
    }

    return false;
  }

  /** Close. */
  public void close() {
    if (selector != null) {
      selector.wakeup();
      try {
        selector.close();
      } catch (IOException e1) {
        log.warn("close selector fails", e1);
      } finally {
        selector = null;
      }
    }

    if (server != null) {
      try {
        server.socket().close();
        server.close();
      } catch (IOException e) {
        log.warn("close socket server fails", e);
      } finally {
        server = null;
      }
    }
  }

  /**
   * Instantiates a new MDC server.
   *
   * @param host the host
   * @param port the port
   */
  protected MDCServer(String host, int port) {
    _conf = Config.getConfig();

    address = (host == null) ? new InetSocketAddress(port) : new InetSocketAddress(host, port);

    /** initialize app command */
    Command.init();

    /** initialize the connection center */
    TConnCenter.init(_conf, port);

    synchronized (_conf) {
      /** load public key from database */
      TConn.pub_key = SystemConfig.s("pub_key", null);
      TConn.pri_key = SystemConfig.s("pri_key", null);

      /** initialize the RSA key, hardcode 2048 bits */
      if (TConn.pub_key == null
          || TConn.pri_key == null
          || "".equals(TConn.pub_key)
          || "".equals(TConn.pri_key)) {
        /** print out the old state */
        log.warn(
            "the pub_key or pri_key missed, the old state are pub_key:["
                + TConn.pub_key
                + "], pri_key:["
                + TConn.pri_key
                + "]");

        Key k = RSA.generate(2048);
        TConn.pri_key = k.pri_key;
        TConn.pub_key = k.pub_key;

        /** print out the new public key */
        log.warn("create new RSA key pair, pub_key:[" + TConn.pub_key + ']');

        /** set back in database */
        SystemConfig.setConfig("pri_key", TConn.pri_key);
        SystemConfig.setConfig("pub_key", TConn.pub_key);
      }

      MAX_SIZE = SystemConfig.i("mdc.max_size", MAX_SIZE);
    }
  }

  /**
   * Start.
   *
   * @return the MDC server
   */
  public abstract MDCServer start();

  /** Stop. */
  public void stop() {
    acceptor.unbind();
  }

  /**
   * Service.
   *
   * @param o the o
   * @param session the session
   */
  void service(IoBuffer o, IoSession session) {
    try {
      // System.out.println(o.remaining() + "/" + o.capacity());

      session.setAttribute("last", System.currentTimeMillis());

      SimpleIoBuffer in = (SimpleIoBuffer) session.getAttribute("buf");
      if (in == null) {
        in = SimpleIoBuffer.create(4096);
        session.setAttribute("buf", in);
      }
      byte[] data = new byte[o.remaining()];
      o.get(data);
      in.append(data);

      // log.debug("recv: " + data.length + ", " +
      // session.getRemoteAddress());

      while (in.length() > 5) {
        in.mark();
        /**
         * Byte 1: head of the package<br>
         * bit 7-6: "01", indicator of MDC<br>
         * bit 5: encrypt indicator, "0": no; "1": encrypted<br>
         * bit 4: zip indicator, "0": no, "1": ziped<br>
         * bit 0-3: reserved<br>
         * Byte 2-5: length of data<br>
         * Byte[…]: data array<br>
         */
        byte head = in.read();
        /** test the head indicator, if not correct close it */
        if ((head & 0xC0) != 0x40) {
          log.info("flag is not correct! flag:" + head + ",from: " + session.getRemoteAddress());

          session.write("error.head");
          session.close(true);
          return;
        }

        int len = in.getInt();

        if (len <= 0 || len > MAX_SIZE) {
          log.error(
              "mdcserver.Wrong lendth: "
                  + len
                  + "/"
                  + MAX_SIZE
                  + " - "
                  + session.getRemoteAddress());
          session.write("error.packet.size");
          session.close(true);
          break;
        }

        // log.info("packet.len:" + len + ", len in buffer:" +
        // in.length());
        if (in.length() < len) {
          in.reset();
          break;
        } else {
          // do it

          byte[] b = new byte[len];
          in.read(b);

          // log.info("stub.package.size: " + len + ", head:" + head +
          // ", cmd:" + Bean.toString(b));
          // log.info("stub.package.size: " + len + ", head:" + head);

          /** test the zip flag */
          if ((head & 0x10) != 0) {
            b = Zip.unzip(b);
          }

          final TConn d = (TConn) session.getAttribute("conn");
          if (d != null) {
            /** test the encrypted flag */
            if ((head & 0x20) != 0) {
              b = DES.decode(b, d.deskey);
            }

            final byte[] bb = b;

            /** test if the packet is for mdc or app */
            new WorkerTask() {

              @Override
              public void onExecute() {
                d.process(bb);
              }
            }.schedule(0);

            session.setAttribute("last", System.currentTimeMillis());
          } else {
            session.write("error.getconnection");

            log.error("error to get connection: " + session.getRemoteAddress());
            session.close(true);
          }
        }
      }
    } catch (Throwable e) {
      log.error("closing stub: " + session.getRemoteAddress(), e);
      session.write("exception." + e.getMessage());
      session.close(true);
    }
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#sessionCreated(org.apache
   * .mina.core.session.IoSession)
   */
  public void sessionCreated(IoSession session) throws Exception {
    log.info("stub created:" + session.getRemoteAddress());

    Counter.add("mdc", "connection", 1);

    TConn d = new TConn(session);
    d.set("x-forwarded-for", session.getRemoteAddress().toString());

    session.setAttribute("conn", d);
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#sessionClosed(org.apache
   * .mina.core.session.IoSession)
   */
  public void sessionClosed(IoSession session) throws Exception {
    log.info("closed stub: " + session.getRemoteAddress());
    TConn d = (TConn) session.getAttribute("conn");
    if (d != null) {
      d.close();

      session.removeAttribute("conn");
    }
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#sessionIdle(org.apache.
   * mina.core.session.IoSession, org.apache.mina.core.session.IdleStatus)
   */
  public void sessionIdle(IoSession session, IdleStatus status) throws Exception {
    if (IdleStatus.BOTH_IDLE.equals(status)) {
      Long l = (Long) session.getAttribute("last");
      if (l != null && System.currentTimeMillis() - l > 60 * 1000) {
        session.close(true);
      }
    }
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#messageReceived(org.apache
   * .mina.core.session.IoSession, java.lang.Object)
   */
  public void messageReceived(IoSession session, Object message) throws Exception {
    // System.out.println(message);
    if (message instanceof IoBuffer) {
      service((IoBuffer) message, session);
    }
  }

  /**
   * Creates the tcp server.
   *
   * @param host the host
   * @param port the port
   * @return the MDC server
   */
  public static synchronized MDCServer createTcpServer(String host, int port) {
    return new TDCServer(host, port);
  }

  /**
   * Creates the udp server.
   *
   * @param host the host
   * @param port the port
   * @return the MDC server
   */
  public static synchronized MDCServer createUdpServer(String host, int port) {
    return new UDCServer(host, port);
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#exceptionCaught(org.apache
   * .mina.core.session.IoSession, java.lang.Throwable)
   */
  @Override
  public void exceptionCaught(IoSession session, Throwable cause) throws Exception {
    TConn d = (TConn) session.getAttribute("conn");
    if (d != null && d.valid()) {
      App.bye(d);
    }
  }
}
/**
 * The <code>DefaultTcpTransportMapping</code> implements a TCP transport mapping with the Java 1.4
 * new IO API.
 *
 * <p>It uses a single thread for processing incoming and outgoing messages. The thread is started
 * when the <code>listen</code> method is called, or when an outgoing request is sent using the
 * <code>sendMessage</code> method.
 *
 * @author Frank Fock
 * @version 1.7.4a
 */
public class DefaultTcpTransportMapping extends TcpTransportMapping {

  private static final LogAdapter logger = LogFactory.getLogger(DefaultTcpTransportMapping.class);

  private Hashtable sockets = new Hashtable();
  private ServerThread server;

  private Timer socketCleaner;
  // 1 minute default timeout
  private long connectionTimeout = 60000;
  private boolean serverEnabled = false;

  private static final int MIN_SNMP_HEADER_LENGTH = 6;
  private MessageLengthDecoder messageLengthDecoder = new SnmpMesssageLengthDecoder();

  /**
   * Creates a default TCP transport mapping with the server for incoming messages disabled.
   *
   * @throws UnknownHostException
   * @throws IOException on failure of binding a local port.
   */
  public DefaultTcpTransportMapping() throws UnknownHostException, IOException {
    super(new TcpAddress(InetAddress.getLocalHost(), 0));
  }

  /**
   * Creates a default TCP transport mapping that binds to the given address (interface) on the
   * local host.
   *
   * @param serverAddress the TcpAddress instance that describes the server address to listen on
   *     incoming connection requests.
   * @throws UnknownHostException if the specified interface does not exist.
   * @throws IOException if the given address cannot be bound.
   */
  public DefaultTcpTransportMapping(TcpAddress serverAddress)
      throws UnknownHostException, IOException {
    super(serverAddress);
    this.serverEnabled = true;
  }

  /**
   * Listen for incoming and outgoing requests. If the <code>serverEnabled</code> member is <code>
   * false</code> the server for incoming requests is not started. This starts the internal server
   * thread that processes messages.
   *
   * @throws SocketException when the transport is already listening for incoming/outgoing messages.
   * @throws IOException
   */
  public synchronized void listen() throws java.io.IOException {
    if (server != null) {
      throw new SocketException("Port already listening");
    }
    server = new ServerThread();
    if (connectionTimeout > 0) {
      socketCleaner = new Timer(true); // run as daemon
    }
    server.setDaemon(true);
    server.start();
  }

  /**
   * Changes the priority of the server thread for this TCP transport mapping. This method has no
   * effect, if called before {@link #listen()} has been called for this transport mapping.
   *
   * @param newPriority the new priority.
   * @see Thread#setPriority
   * @since 1.2.2
   */
  public void setPriority(int newPriority) {
    ServerThread st = server;
    if (st != null) {
      st.setPriority(newPriority);
    }
  }

  /**
   * Returns the priority of the internal listen thread.
   *
   * @return a value between {@link Thread#MIN_PRIORITY} and {@link Thread#MAX_PRIORITY}.
   * @since 1.2.2
   */
  public int getPriority() {
    ServerThread st = server;
    if (st != null) {
      return st.getPriority();
    } else {
      return Thread.NORM_PRIORITY;
    }
  }

  /**
   * Sets the name of the listen thread for this UDP transport mapping. This method has no effect,
   * if called before {@link #listen()} has been called for this transport mapping.
   *
   * @param name the new thread name.
   * @since 1.6
   */
  public void setThreadName(String name) {
    ServerThread st = server;
    if (st != null) {
      st.setName(name);
    }
  }

  /**
   * Returns the name of the listen thread.
   *
   * @return the thread name if in listening mode, otherwise <code>null</code>.
   * @since 1.6
   */
  public String getThreadName() {
    ServerThread st = server;
    if (st != null) {
      return st.getName();
    } else {
      return null;
    }
  }

  /** Closes all open sockets and stops the internal server thread that processes messages. */
  public void close() {
    ServerThread st = server;
    if (st != null) {
      st.close();
      try {
        st.join();
      } catch (InterruptedException ex) {
        logger.warn(ex);
      }
      server = null;
      for (Iterator it = sockets.values().iterator(); it.hasNext(); ) {
        SocketEntry entry = (SocketEntry) it.next();
        try {
          synchronized (entry) {
            entry.getSocket().close();
          }
          logger.debug("Socket to " + entry.getPeerAddress() + " closed");
        } catch (IOException iox) {
          // ingore
          logger.debug(iox);
        }
      }
      if (socketCleaner != null) {
        socketCleaner.cancel();
      }
      socketCleaner = null;
    }
  }

  /**
   * Closes a connection to the supplied remote address, if it is open. This method is particularly
   * useful when not using a timeout for remote connections.
   *
   * @param remoteAddress the address of the peer socket.
   * @return <code>true</code> if the connection has been closed and <code>false</code> if there was
   *     nothing to close.
   * @since 1.7.1
   */
  public synchronized boolean close(Address remoteAddress) throws IOException {
    if (logger.isDebugEnabled()) {
      logger.debug("Closing socket for peer address " + remoteAddress);
    }
    SocketEntry entry = (SocketEntry) sockets.remove(remoteAddress);
    if (entry != null) {
      synchronized (entry) {
        entry.getSocket().close();
      }
      logger.info("Socket to " + entry.getPeerAddress() + " closed");
      return true;
    }
    return false;
  }

  /**
   * Sends a SNMP message to the supplied address.
   *
   * @param address an <code>TcpAddress</code>. A <code>ClassCastException</code> is thrown if
   *     <code>address</code> is not a <code>TcpAddress</code> instance.
   * @param message byte[] the message to sent.
   * @throws IOException
   */
  public void sendMessage(Address address, byte[] message) throws java.io.IOException {
    if (server == null) {
      listen();
    }
    server.sendMessage(address, message);
  }

  /**
   * Gets the connection timeout. This timeout specifies the time a connection may be idle before it
   * is closed.
   *
   * @return long the idle timeout in milliseconds.
   */
  public long getConnectionTimeout() {
    return connectionTimeout;
  }

  /**
   * Sets the connection timeout. This timeout specifies the time a connection may be idle before it
   * is closed.
   *
   * @param connectionTimeout the idle timeout in milliseconds. A zero or negative value will
   *     disable any timeout and connections opened by this transport mapping will stay opened until
   *     they are explicitly closed.
   */
  public void setConnectionTimeout(long connectionTimeout) {
    this.connectionTimeout = connectionTimeout;
  }

  /**
   * Checks whether a server for incoming requests is enabled.
   *
   * @return boolean
   */
  public boolean isServerEnabled() {
    return serverEnabled;
  }

  public MessageLengthDecoder getMessageLengthDecoder() {
    return messageLengthDecoder;
  }

  /**
   * Sets whether a server for incoming requests should be created when the transport is set into
   * listen state. Setting this value has no effect until the {@link #listen()} method is called (if
   * the transport is already listening, {@link #close()} has to be called before).
   *
   * @param serverEnabled if <code>true</code> if the transport will listens for incoming requests
   *     after {@link #listen()} has been called.
   */
  public void setServerEnabled(boolean serverEnabled) {
    this.serverEnabled = serverEnabled;
  }

  /**
   * Sets the message length decoder. Default message length decoder is the {@link
   * SnmpMesssageLengthDecoder}. The message length decoder must be able to decode the total length
   * of a message for this transport mapping protocol(s).
   *
   * @param messageLengthDecoder a <code>MessageLengthDecoder</code> instance.
   */
  public void setMessageLengthDecoder(MessageLengthDecoder messageLengthDecoder) {
    if (messageLengthDecoder == null) {
      throw new NullPointerException();
    }
    this.messageLengthDecoder = messageLengthDecoder;
  }

  /**
   * Gets the inbound buffer size for incoming requests. When SNMP packets are received that are
   * longer than this maximum size, the messages will be silently dropped and the connection will be
   * closed.
   *
   * @return the maximum inbound buffer size in bytes.
   */
  public int getMaxInboundMessageSize() {
    return super.getMaxInboundMessageSize();
  }

  /**
   * Sets the maximum buffer size for incoming requests. When SNMP packets are received that are
   * longer than this maximum size, the messages will be silently dropped and the connection will be
   * closed.
   *
   * @param maxInboundMessageSize the length of the inbound buffer in bytes.
   */
  public void setMaxInboundMessageSize(int maxInboundMessageSize) {
    this.maxInboundMessageSize = maxInboundMessageSize;
  }

  private synchronized void timeoutSocket(SocketEntry entry) {
    if (connectionTimeout > 0) {
      socketCleaner.schedule(new SocketTimeout(entry), connectionTimeout);
    }
  }

  public boolean isListening() {
    return (server != null);
  }

  class SocketEntry {
    private Socket socket;
    private TcpAddress peerAddress;
    private long lastUse;
    private LinkedList message = new LinkedList();
    private ByteBuffer readBuffer = null;

    public SocketEntry(TcpAddress address, Socket socket) {
      this.peerAddress = address;
      this.socket = socket;
      this.lastUse = System.currentTimeMillis();
    }

    public long getLastUse() {
      return lastUse;
    }

    public void used() {
      lastUse = System.currentTimeMillis();
    }

    public Socket getSocket() {
      return socket;
    }

    public TcpAddress getPeerAddress() {
      return peerAddress;
    }

    public synchronized void addMessage(byte[] message) {
      this.message.add(message);
    }

    public byte[] nextMessage() {
      if (this.message.size() > 0) {
        return (byte[]) this.message.removeFirst();
      }
      return null;
    }

    public void setReadBuffer(ByteBuffer byteBuffer) {
      this.readBuffer = byteBuffer;
    }

    public ByteBuffer getReadBuffer() {
      return readBuffer;
    }

    public String toString() {
      return "SocketEntry[peerAddress="
          + peerAddress
          + ",socket="
          + socket
          + ",lastUse="
          + new Date(lastUse)
          + "]";
    }
  }

  public static class SnmpMesssageLengthDecoder implements MessageLengthDecoder {
    public int getMinHeaderLength() {
      return MIN_SNMP_HEADER_LENGTH;
    }

    public MessageLength getMessageLength(ByteBuffer buf) throws IOException {
      MutableByte type = new MutableByte();
      BERInputStream is = new BERInputStream(buf);
      int ml = BER.decodeHeader(is, type);
      int hl = (int) is.getPosition();
      MessageLength messageLength = new MessageLength(hl, ml);
      return messageLength;
    }
  }

  class SocketTimeout extends TimerTask {
    private SocketEntry entry;

    public SocketTimeout(SocketEntry entry) {
      this.entry = entry;
    }

    /** run */
    public void run() {
      long now = System.currentTimeMillis();
      if ((socketCleaner == null) || (now - entry.getLastUse() >= connectionTimeout)) {
        if (logger.isDebugEnabled()) {
          logger.debug(
              "Socket has not been used for "
                  + (now - entry.getLastUse())
                  + " micro seconds, closing it");
        }
        sockets.remove(entry.getPeerAddress());
        try {
          synchronized (entry) {
            entry.getSocket().close();
          }
          logger.info("Socket to " + entry.getPeerAddress() + " closed due to timeout");
        } catch (IOException ex) {
          logger.error(ex);
        }
      } else {
        if (logger.isDebugEnabled()) {
          logger.debug("Scheduling " + ((entry.getLastUse() + connectionTimeout) - now));
        }
        socketCleaner.schedule(
            new SocketTimeout(entry), (entry.getLastUse() + connectionTimeout) - now);
      }
    }
  }

  class ServerThread extends Thread {
    private byte[] buf;
    private volatile boolean stop = false;
    private Throwable lastError = null;
    private ServerSocketChannel ssc;
    private Selector selector;

    private LinkedList pending = new LinkedList();

    public ServerThread() throws IOException {
      setName("DefaultTCPTransportMapping_" + getAddress());
      buf = new byte[getMaxInboundMessageSize()];
      // Selector for incoming requests
      selector = Selector.open();

      if (serverEnabled) {
        // Create a new server socket and set to non blocking mode
        ssc = ServerSocketChannel.open();
        ssc.configureBlocking(false);

        // Bind the server socket
        InetSocketAddress isa =
            new InetSocketAddress(tcpAddress.getInetAddress(), tcpAddress.getPort());
        ssc.socket().bind(isa);
        // Register accepts on the server socket with the selector. This
        // step tells the selector that the socket wants to be put on the
        // ready list when accept operations occur, so allowing multiplexed
        // non-blocking I/O to take place.
        ssc.register(selector, SelectionKey.OP_ACCEPT);
      }
    }

    private void processPending() {
      synchronized (pending) {
        while (pending.size() > 0) {
          SocketEntry entry = (SocketEntry) pending.removeFirst();
          try {
            // Register the channel with the selector, indicating
            // interest in connection completion and attaching the
            // target object so that we can get the target back
            // after the key is added to the selector's
            // selected-key set
            if (entry.getSocket().isConnected()) {
              entry.getSocket().getChannel().register(selector, SelectionKey.OP_WRITE, entry);
            } else {
              entry.getSocket().getChannel().register(selector, SelectionKey.OP_CONNECT, entry);
            }

          } catch (IOException iox) {
            logger.error(iox);
            // Something went wrong, so close the channel and
            // record the failure
            try {
              entry.getSocket().getChannel().close();
              TransportStateEvent e =
                  new TransportStateEvent(
                      DefaultTcpTransportMapping.this,
                      entry.getPeerAddress(),
                      TransportStateEvent.STATE_CLOSED,
                      iox);
              fireConnectionStateChanged(e);
            } catch (IOException ex) {
              logger.error(ex);
            }
            lastError = iox;
          }
        }
      }
    }

    public Throwable getLastError() {
      return lastError;
    }

    public void sendMessage(Address address, byte[] message) throws java.io.IOException {
      Socket s = null;
      SocketEntry entry = (SocketEntry) sockets.get(address);
      if (logger.isDebugEnabled()) {
        logger.debug("Looking up connection for destination '" + address + "' returned: " + entry);
        logger.debug(sockets.toString());
      }
      if (entry != null) {
        s = entry.getSocket();
      }
      if ((s == null) || (s.isClosed())) {
        if (logger.isDebugEnabled()) {
          logger.debug("Socket for address '" + address + "' is closed, opening it...");
        }
        SocketChannel sc = null;
        try {
          // Open the channel, set it to non-blocking, initiate connect
          sc = SocketChannel.open();
          sc.configureBlocking(false);
          sc.connect(
              new InetSocketAddress(
                  ((TcpAddress) address).getInetAddress(), ((TcpAddress) address).getPort()));
          s = sc.socket();
          entry = new SocketEntry((TcpAddress) address, s);
          entry.addMessage(message);
          sockets.put(address, entry);

          synchronized (pending) {
            pending.add(entry);
          }

          selector.wakeup();
          logger.debug("Trying to connect to " + address);
        } catch (IOException iox) {
          logger.error(iox);
          throw iox;
        }
      } else {
        entry.addMessage(message);
        synchronized (pending) {
          pending.add(entry);
        }
        selector.wakeup();
      }
    }

    public void run() {
      // Here's where everything happens. The select method will
      // return when any operations registered above have occurred, the
      // thread has been interrupted, etc.
      try {
        while (!stop) {
          try {
            if (selector.select() > 0) {
              if (stop) {
                break;
              }
              // Someone is ready for I/O, get the ready keys
              Set readyKeys = selector.selectedKeys();
              Iterator it = readyKeys.iterator();

              // Walk through the ready keys collection and process date requests.
              while (it.hasNext()) {
                SelectionKey sk = (SelectionKey) it.next();
                it.remove();
                SocketChannel readChannel = null;
                TcpAddress incomingAddress = null;
                if (sk.isAcceptable()) {
                  // The key indexes into the selector so you
                  // can retrieve the socket that's ready for I/O
                  ServerSocketChannel nextReady = (ServerSocketChannel) sk.channel();
                  // Accept the date request and send back the date string
                  Socket s = nextReady.accept().socket();
                  readChannel = s.getChannel();
                  readChannel.configureBlocking(false);
                  readChannel.register(selector, SelectionKey.OP_READ);

                  incomingAddress = new TcpAddress(s.getInetAddress(), s.getPort());
                  SocketEntry entry = new SocketEntry(incomingAddress, s);
                  sockets.put(incomingAddress, entry);
                  timeoutSocket(entry);
                  TransportStateEvent e =
                      new TransportStateEvent(
                          DefaultTcpTransportMapping.this,
                          incomingAddress,
                          TransportStateEvent.STATE_CONNECTED,
                          null);
                  fireConnectionStateChanged(e);
                } else if (sk.isReadable()) {
                  readChannel = (SocketChannel) sk.channel();
                  incomingAddress =
                      new TcpAddress(
                          readChannel.socket().getInetAddress(), readChannel.socket().getPort());
                } else if (sk.isWritable()) {
                  try {
                    SocketEntry entry = (SocketEntry) sk.attachment();
                    SocketChannel sc = (SocketChannel) sk.channel();
                    if (entry != null) {
                      writeMessage(entry, sc);
                    }
                  } catch (IOException iox) {
                    if (logger.isDebugEnabled()) {
                      iox.printStackTrace();
                    }
                    logger.warn(iox);
                    TransportStateEvent e =
                        new TransportStateEvent(
                            DefaultTcpTransportMapping.this,
                            incomingAddress,
                            TransportStateEvent.STATE_DISCONNECTED_REMOTELY,
                            iox);
                    fireConnectionStateChanged(e);
                    sk.cancel();
                  }
                } else if (sk.isConnectable()) {
                  try {
                    SocketEntry entry = (SocketEntry) sk.attachment();
                    SocketChannel sc = (SocketChannel) sk.channel();
                    if ((!sc.isConnected()) && (sc.finishConnect())) {
                      sc.configureBlocking(false);
                      logger.debug("Connected to " + entry.getPeerAddress());
                      // make sure conncetion is closed if not used for timeout
                      // micro seconds
                      timeoutSocket(entry);
                      sc.register(selector, SelectionKey.OP_WRITE, entry);
                    }
                    TransportStateEvent e =
                        new TransportStateEvent(
                            DefaultTcpTransportMapping.this,
                            incomingAddress,
                            TransportStateEvent.STATE_CONNECTED,
                            null);
                    fireConnectionStateChanged(e);
                  } catch (IOException iox) {
                    if (logger.isDebugEnabled()) {
                      iox.printStackTrace();
                    }
                    logger.warn(iox);
                    sk.cancel();
                  }
                }

                if (readChannel != null) {
                  try {
                    readMessage(sk, readChannel, incomingAddress);
                  } catch (IOException iox) {
                    // IO exception -> channel closed remotely
                    if (logger.isDebugEnabled()) {
                      iox.printStackTrace();
                    }
                    logger.warn(iox);
                    sk.cancel();
                    readChannel.close();
                    TransportStateEvent e =
                        new TransportStateEvent(
                            DefaultTcpTransportMapping.this,
                            incomingAddress,
                            TransportStateEvent.STATE_DISCONNECTED_REMOTELY,
                            iox);
                    fireConnectionStateChanged(e);
                  }
                }
              }
            }
          } catch (NullPointerException npex) {
            // There seems to happen a NullPointerException within the select()
            npex.printStackTrace();
            logger.warn("NullPointerException within select()?");
          }
          processPending();
        }
        if (ssc != null) {
          ssc.close();
        }
      } catch (IOException iox) {
        logger.error(iox);
        lastError = iox;
      }
      if (!stop) {
        stop = true;
        synchronized (DefaultTcpTransportMapping.this) {
          server = null;
        }
      }
    }

    private void readMessage(SelectionKey sk, SocketChannel readChannel, TcpAddress incomingAddress)
        throws IOException {
      // note that socket has been used
      SocketEntry entry = (SocketEntry) sockets.get(incomingAddress);
      if (entry != null) {
        entry.used();
        ByteBuffer readBuffer = entry.getReadBuffer();
        if (readBuffer != null) {
          readChannel.read(readBuffer);
          if (readBuffer.hasRemaining()) {
            readChannel.register(selector, SelectionKey.OP_READ, entry);
          } else {
            dispatchMessage(incomingAddress, readBuffer, readBuffer.capacity());
          }
          return;
        }
      }
      ByteBuffer byteBuffer = ByteBuffer.wrap(buf);
      byteBuffer.limit(messageLengthDecoder.getMinHeaderLength());
      long bytesRead = readChannel.read(byteBuffer);
      if (logger.isDebugEnabled()) {
        logger.debug("Reading header " + bytesRead + " bytes from " + incomingAddress);
      }
      MessageLength messageLength = new MessageLength(0, Integer.MIN_VALUE);
      if (bytesRead == messageLengthDecoder.getMinHeaderLength()) {
        messageLength = messageLengthDecoder.getMessageLength(ByteBuffer.wrap(buf));
        if (logger.isDebugEnabled()) {
          logger.debug("Message length is " + messageLength);
        }
        if ((messageLength.getMessageLength() > getMaxInboundMessageSize())
            || (messageLength.getMessageLength() <= 0)) {
          logger.error(
              "Received message length "
                  + messageLength
                  + " is greater than inboundBufferSize "
                  + getMaxInboundMessageSize());
          synchronized (entry) {
            entry.getSocket().close();
            logger.info("Socket to " + entry.getPeerAddress() + " closed due to an error");
          }
        } else {
          byteBuffer.limit(messageLength.getMessageLength());
          bytesRead += readChannel.read(byteBuffer);
          if (bytesRead == messageLength.getMessageLength()) {
            dispatchMessage(incomingAddress, byteBuffer, bytesRead);
          } else {
            byte[] message = new byte[byteBuffer.limit()];
            byteBuffer.flip();
            byteBuffer.get(message, 0, byteBuffer.limit() - byteBuffer.remaining());
            entry.setReadBuffer(ByteBuffer.wrap(message));
          }
          readChannel.register(selector, SelectionKey.OP_READ, entry);
        }
      } else if (bytesRead < 0) {
        logger.debug("Socket closed remotely");
        sk.cancel();
        readChannel.close();
        TransportStateEvent e =
            new TransportStateEvent(
                DefaultTcpTransportMapping.this,
                incomingAddress,
                TransportStateEvent.STATE_DISCONNECTED_REMOTELY,
                null);
        fireConnectionStateChanged(e);
      }
    }

    private void dispatchMessage(
        TcpAddress incomingAddress, ByteBuffer byteBuffer, long bytesRead) {
      byteBuffer.flip();
      if (logger.isDebugEnabled()) {
        logger.debug(
            "Received message from "
                + incomingAddress
                + " with length "
                + bytesRead
                + ": "
                + new OctetString(byteBuffer.array(), 0, (int) bytesRead).toHexString());
      }
      ByteBuffer bis;
      if (isAsyncMsgProcessingSupported()) {
        byte[] bytes = new byte[(int) bytesRead];
        System.arraycopy(byteBuffer.array(), 0, bytes, 0, (int) bytesRead);
        bis = ByteBuffer.wrap(bytes);
      } else {
        bis = ByteBuffer.wrap(byteBuffer.array(), 0, (int) bytesRead);
      }
      fireProcessMessage(incomingAddress, bis);
    }

    private void writeMessage(SocketEntry entry, SocketChannel sc) throws IOException {
      byte[] message = entry.nextMessage();
      if (message != null) {
        ByteBuffer buffer = ByteBuffer.wrap(message);
        sc.write(buffer);
        if (logger.isDebugEnabled()) {
          logger.debug(
              "Send message with length "
                  + message.length
                  + " to "
                  + entry.getPeerAddress()
                  + ": "
                  + new OctetString(message).toHexString());
        }
        sc.register(selector, SelectionKey.OP_READ);
      }
    }

    public void close() {
      stop = true;
      ServerThread st = server;
      if (st != null) {
        st.interrupt();
      }
    }
  }
}
/** @author Javier Paniza */
public class MetaFinder implements Serializable {

  private static Log log = LogFactory.getLog(MetaFinder.class);

  private static Map argumentsJBoss11ToEJBQL;
  private static Map argumentsToHQL;
  private static Map tokensToChangeDollarsAndNL;

  private String name;
  private String arguments;
  private boolean collection;
  private String condition;
  private String order;
  private MetaModel metaModel;

  public String getArguments() {
    arguments = Strings.change(arguments, "String", "java.lang.String");
    arguments = Strings.change(arguments, "java.lang.java.lang.String", "java.lang.String");
    return arguments;
  }

  public Collection getMetaPropertiesArguments() throws XavaException {
    StringTokenizer st = new StringTokenizer(getArguments(), ",");
    Collection result = new ArrayList();
    while (st.hasMoreTokens()) {
      String argument = st.nextToken();
      StringTokenizer argumentSt = new StringTokenizer(argument);
      String type = argumentSt.nextToken().trim();
      String name = argumentSt.nextToken().trim();
      MetaProperty p = new MetaProperty();
      p.setName(name);
      p.setTypeName(type);
      result.add(p);
    }
    return result;
  }

  public boolean isCollection() {
    return collection;
  }

  public String getCondition() {
    return condition;
  }

  public String getName() {
    return name;
  }

  public void setArguments(String arguments) {
    this.arguments = arguments;
  }

  public void setCollection(boolean collection) {
    this.collection = collection;
  }

  public void setCondition(String condition) {
    this.condition = condition;
  }

  public void setName(String name) {
    this.name = name;
  }

  public boolean isSupportedForEJB2() throws XavaException {
    return !hasSome3LevelProperty(getCondition()) && !hasSome3LevelProperty(getOrder());
  }

  private boolean hasSome3LevelProperty(String sentence) throws XavaException {
    if (sentence == null) return false;
    int i = sentence.indexOf("${");
    int f = 0;
    while (i >= 0) {
      f = sentence.indexOf("}", i + 2);
      if (f < 0) break;
      String property = sentence.substring(i + 2, f);
      StringTokenizer st = new StringTokenizer(property, ".");
      if (st.countTokens() > 3) {
        log.warn(XavaResources.getString("property_3_level_in_ejb2_finder", property, getName()));
        return true;
      }
      if (st.countTokens() == 3) {
        if (!getMetaModel().getMetaProperty(property).isKey()) {
          log.warn(XavaResources.getString("property_3_level_in_ejb2_finder", property, getName()));
          return true;
        }
      }
      i = sentence.indexOf("${", i + 1);
    }
    return false;
  }

  public String getEJBQLCondition() throws XavaException {
    StringBuffer sb = new StringBuffer("SELECT OBJECT(o) FROM ");
    sb.append(getMetaModel().getName());
    sb.append(" o");
    if (!Is.emptyString(this.condition)) {
      sb.append(" WHERE ");
      String attributesCondition =
          getMetaModel().getMapping().changePropertiesByCMPAttributes(this.condition);
      sb.append(Strings.change(attributesCondition, getArgumentsJBoss11ToEJBQL()));
    }
    if (!Is.emptyString(this.order)) {
      sb.append(" ORDER BY ");
      sb.append(getMetaModel().getMapping().changePropertiesByCMPAttributes(this.order));
    }
    return sb.toString();
  }

  public String getHQLCondition() throws XavaException {
    return getHQLCondition(true);
  }

  private String getHQLCondition(boolean order) throws XavaException {
    StringBuffer sb = new StringBuffer("from ");
    sb.append(getMetaModel().getName());
    sb.append(" as o");
    if (!Is.emptyString(this.condition)) {
      sb.append(" where ");
      String condition = transformAggregateProperties(getCondition());
      condition = Strings.change(condition, getArgumentsToHQL());
      sb.append(Strings.change(condition, getTokensToChangeDollarsAndNL()));
    }
    if (order && !Is.emptyString(this.order)) {
      sb.append(" order by ");
      sb.append(
          Strings.change(
              transformAggregateProperties(this.order), getTokensToChangeDollarsAndNL()));
    }
    return sb.toString();
  }

  /**
   * Transforms ${address.street} in ${address_street} if address if an aggregate of container
   * model.
   *
   * @param condition
   * @return
   */
  private String transformAggregateProperties(String condition) {
    int i = condition.indexOf("${");
    if (i < 0) return condition;
    StringBuffer result = new StringBuffer(condition.substring(0, i + 2));
    while (i >= 0) {
      int f = condition.indexOf("}", i);
      String property = condition.substring(i + 2, f);
      String transformedProperty = transformAgregateProperty(property);
      result.append(transformedProperty);
      i = condition.indexOf("${", f);
      if (i >= 0) result.append(condition.substring(f, i));
      else result.append(condition.substring(f));
    }
    return result.toString();
  }

  private String transformAgregateProperty(String property) {
    StringBuffer result = new StringBuffer();
    StringTokenizer st = new StringTokenizer(property, ".");
    String member = "";
    while (st.hasMoreTokens()) {
      String token = st.nextToken();
      result.append(token);
      if (!st.hasMoreTokens()) break;
      member = member + token;
      try {
        MetaReference ref = getMetaModel().getMetaReference(member);
        if (ref.isAggregate()) result.append('_');
        else result.append('.');
      } catch (XavaException ex) {
        result.append('.');
      }
      member = member + ".";
    }
    return result.toString();
  }

  public String getHQLCountSentence() throws XavaException {
    StringBuffer sb = new StringBuffer("select count(*) ");
    sb.append(getHQLCondition(false));
    return sb.toString();
  }

  public MetaModel getMetaModel() {
    return metaModel;
  }

  public void setMetaModel(MetaModel metaModel) {
    this.metaModel = metaModel;
  }

  public String getOrder() {
    return order;
  }

  public void setOrder(String order) {
    this.order = order;
  }

  private static Map getArgumentsJBoss11ToEJBQL() {
    if (argumentsJBoss11ToEJBQL == null) {
      argumentsJBoss11ToEJBQL = new HashMap();
      for (int i = 0; i < 30; i++) {
        argumentsJBoss11ToEJBQL.put("{" + i + "}", "?" + (i + 1));
      }
    }
    return argumentsJBoss11ToEJBQL;
  }

  private static Map getArgumentsToHQL() {
    if (argumentsToHQL == null) {
      argumentsToHQL = new HashMap();
      for (int i = 0; i < 30; i++) {
        argumentsToHQL.put("{" + i + "}", ":arg" + i);
      }
    }
    return argumentsToHQL;
  }

  static Map getTokensToChangeDollarsAndNL() {
    if (tokensToChangeDollarsAndNL == null) {
      tokensToChangeDollarsAndNL = new HashMap();
      tokensToChangeDollarsAndNL.put("${", "o.");
      tokensToChangeDollarsAndNL.put("}", "");
      tokensToChangeDollarsAndNL.put("\n", "");
    }
    return tokensToChangeDollarsAndNL;
  }

  public boolean equals(Object other) {
    if (!(other instanceof MetaFinder)) return false;
    return toString().equals(other.toString());
  }

  public int hashCode() {
    return toString().hashCode();
  }

  public String toString() {
    return "Finder: " + getMetaModel().getName() + "." + getName();
  }
}
/**
 * Distribute application-specific large, read-only files efficiently.
 *
 * <p><code>DistributedCache</code> is a facility provided by the Map-Reduce framework to cache
 * files (text, archives, jars etc.) needed by applications.
 *
 * <p>Applications specify the files, via urls (hdfs:// or http://) to be cached via the {@link
 * org.apache.hadoop.mapred.JobConf}. The <code>DistributedCache</code> assumes that the files
 * specified via hdfs:// urls are already present on the {@link FileSystem} at the path specified by
 * the url.
 *
 * <p>The framework will copy the necessary files on to the slave node before any tasks for the job
 * are executed on that node. Its efficiency stems from the fact that the files are only copied once
 * per job and the ability to cache archives which are un-archived on the slaves.
 *
 * <p><code>DistributedCache</code> can be used to distribute simple, read-only data/text files
 * and/or more complex types such as archives, jars etc. Archives (zip, tar and tgz/tar.gz files)
 * are un-archived at the slave nodes. Jars may be optionally added to the classpath of the tasks, a
 * rudimentary software distribution mechanism. Files have execution permissions. Optionally users
 * can also direct it to symlink the distributed cache file(s) into the working directory of the
 * task.
 *
 * <p><code>DistributedCache</code> tracks modification timestamps of the cache files. Clearly the
 * cache files should not be modified by the application or externally while the job is executing.
 *
 * <p>Here is an illustrative example on how to use the <code>DistributedCache</code>:
 *
 * <p>
 *
 * <blockquote>
 *
 * <pre>
 *     // Setting up the cache for the application
 *
 *     1. Copy the requisite files to the <code>FileSystem</code>:
 *
 *     $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat
 *     $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip
 *     $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar
 *     $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar
 *     $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz
 *     $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz
 *
 *     2. Setup the application's <code>JobConf</code>:
 *
 *     JobConf job = new JobConf();
 *     DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"),
 *                                   job);
 *     DistributedCache.addCacheArchive(new URI("/myapp/map.zip", job);
 *     DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job);
 *     DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar", job);
 *     DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz", job);
 *     DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz", job);
 *
 *     3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper}
 *     or {@link org.apache.hadoop.mapred.Reducer}:
 *
 *     public static class MapClass extends MapReduceBase
 *     implements Mapper&lt;K, V, K, V&gt; {
 *
 *       private Path[] localArchives;
 *       private Path[] localFiles;
 *
 *       public void configure(JobConf job) {
 *         // Get the cached archives/files
 *         localArchives = DistributedCache.getLocalCacheArchives(job);
 *         localFiles = DistributedCache.getLocalCacheFiles(job);
 *       }
 *
 *       public void map(K key, V value,
 *                       OutputCollector&lt;K, V&gt; output, Reporter reporter)
 *       throws IOException {
 *         // Use data from the cached archives/files here
 *         // ...
 *         // ...
 *         output.collect(k, v);
 *       }
 *     }
 *
 * </pre>
 *
 * </blockquote>
 *
 * @see org.apache.hadoop.mapred.JobConf
 * @see org.apache.hadoop.mapred.JobClient
 */
public class DistributedCache {
  // cacheID to cacheStatus mapping
  private static TreeMap<String, CacheStatus> cachedArchives = new TreeMap<String, CacheStatus>();

  private static TreeMap<Path, Long> baseDirSize = new TreeMap<Path, Long>();
  private static TreeMap<Path, Integer> baseDirNumberSubDir = new TreeMap<Path, Integer>();

  // default total cache size
  private static final long DEFAULT_CACHE_SIZE = 10737418240L;
  private static final long DEFAULT_CACHE_SUBDIR_LIMIT = 10000;

  private static final Log LOG = LogFactory.getLog(DistributedCache.class);
  private static Random random = new Random();

  /**
   * Get the locally cached file or archive; it could either be previously cached (and valid) or
   * copy it from the {@link FileSystem} now.
   *
   * @param cache the cache to be localized, this should be specified as new
   *     URI(hdfs://hostname:port/absolute_path_to_file#LINKNAME). If no schema or hostname:port is
   *     provided the file is assumed to be in the filesystem being used in the Configuration
   * @param conf The Confguration file which contains the filesystem
   * @param baseDir The base cache Dir where you wnat to localize the files/archives
   * @param fileStatus The file status on the dfs.
   * @param isArchive if the cache is an archive or a file. In case it is an archive with a .zip or
   *     .jar or .tar or .tgz or .tar.gz extension it will be unzipped/unjarred/untarred
   *     automatically and the directory where the archive is unzipped/unjarred/untarred is returned
   *     as the Path. In case of a file, the path to the file is returned
   * @param confFileStamp this is the hdfs file modification timestamp to verify that the file to be
   *     cached hasn't changed since the job started
   * @param currentWorkDir this is the directory where you would want to create symlinks for the
   *     locally cached files/archives
   * @return the path to directory where the archives are unjarred in case of archives, the path to
   *     the file where the file is copied locally
   * @throws IOException
   */
  public static Path getLocalCache(
      URI cache,
      Configuration conf,
      Path baseDir,
      FileStatus fileStatus,
      boolean isArchive,
      long confFileStamp,
      Path currentWorkDir,
      MRAsyncDiskService asyncDiskService)
      throws IOException {
    return getLocalCache(
        cache,
        conf,
        baseDir,
        fileStatus,
        isArchive,
        confFileStamp,
        fileStatus.getLen(),
        currentWorkDir,
        true,
        asyncDiskService,
        new LocalDirAllocator("mapred.local.dir"));
  }

  public static Path getLocalCacheFromTimestamps(
      URI cache,
      Configuration conf,
      Path subDir,
      FileStatus fileStatus,
      boolean isArchive,
      long confFileStamp,
      long fileLength,
      Path currentWorkDir,
      boolean honorSymLinkConf,
      MRAsyncDiskService asyncDiskService,
      LocalDirAllocator lDirAllocator)
      throws IOException {
    return getLocalCache(
        cache,
        conf,
        subDir,
        fileStatus,
        isArchive,
        confFileStamp,
        fileLength,
        currentWorkDir,
        honorSymLinkConf,
        asyncDiskService,
        lDirAllocator);
  }

  public static Path getLocalCacheFromURI(
      URI cache,
      Configuration conf,
      Path subDir,
      boolean isArchive,
      long fileLength,
      Path currentWorkDir,
      boolean honorSymLinkConf,
      MRAsyncDiskService asyncDiskService,
      LocalDirAllocator lDirAllocator)
      throws IOException {
    return getLocalCache(
        cache,
        conf,
        subDir,
        null,
        isArchive,
        0,
        fileLength,
        currentWorkDir,
        honorSymLinkConf,
        asyncDiskService,
        lDirAllocator);
  }

  /** Added for back compatibility. */
  public static Path getLocalCache(
      URI cache,
      Configuration conf,
      Path subdir,
      FileStatus fileStatus,
      boolean isArchive,
      long confFileStamp,
      Path currentWorkDir,
      boolean honorSymLinkConf,
      MRAsyncDiskService asyncDiskService,
      LocalDirAllocator lDirAllocator)
      throws IOException {
    return getLocalCache(
        cache,
        conf,
        subdir,
        fileStatus,
        isArchive,
        confFileStamp,
        fileStatus.getLen(),
        currentWorkDir,
        honorSymLinkConf,
        asyncDiskService,
        lDirAllocator);
  }

  /**
   * Get the locally cached file or archive; it could either be previously cached (and valid) or
   * copy it from the {@link FileSystem} now.
   *
   * @param cache the cache to be localized, this should be specified as new
   *     URI(hdfs://hostname:port/absolute_path_to_file#LINKNAME). If no schema or hostname:port is
   *     provided the file is assumed to be in the filesystem being used in the Configuration
   * @param conf The Confguration file which contains the filesystem
   * @param subDir The sub cache Dir where you want to localize the files/archives
   * @param fileStatus The file status on the dfs.
   * @param isArchive if the cache is an archive or a file. In case it is an archive with a .zip or
   *     .jar or .tar or .tgz or .tar.gz extension it will be unzipped/unjarred/untarred
   *     automatically and the directory where the archive is unzipped/unjarred/untarred is returned
   *     as the Path. In case of a file, the path to the file is returned
   * @param confFileStamp this is the hdfs file modification timestamp to verify that the file to be
   *     cached hasn't changed since the job started
   * @param fileLength this is the length of the cache file
   * @param currentWorkDir this is the directory where you would want to create symlinks for the
   *     locally cached files/archives
   * @param honorSymLinkConf if this is false, then the symlinks are not created even if conf says
   *     so (this is required for an optimization in task launches
   * @param lDirAllocator LocalDirAllocator of the tracker
   * @return the path to directory where the archives are unjarred in case of archives, the path to
   *     the file where the file is copied locally
   * @throws IOException
   */
  private static Path getLocalCache(
      URI cache,
      Configuration conf,
      Path subDir,
      FileStatus fileStatus,
      boolean isArchive,
      long confFileStamp,
      long fileLength,
      Path currentWorkDir,
      boolean honorSymLinkConf,
      MRAsyncDiskService asyncDiskService,
      LocalDirAllocator lDirAllocator)
      throws IOException {
    String key = getKey(cache, conf, confFileStamp);

    CacheStatus lcacheStatus;
    Path localizedPath;
    synchronized (cachedArchives) {
      lcacheStatus = cachedArchives.get(key);
      if (lcacheStatus == null) {
        // was never localized
        Path uniqueParentDir = new Path(subDir, String.valueOf(random.nextLong()));
        String cachePath = new Path(uniqueParentDir, makeRelative(cache, conf)).toString();
        Path localPath = lDirAllocator.getLocalPathForWrite(cachePath, fileLength, conf);
        lcacheStatus =
            new CacheStatus(
                new Path(localPath.toString().replace(cachePath, "")), localPath, uniqueParentDir);
        cachedArchives.put(key, lcacheStatus);
      }
      lcacheStatus.refcount++;
    }
    boolean initSuccessful = false;
    try {
      synchronized (lcacheStatus) {
        if (!lcacheStatus.isInited()) {
          localizedPath = localizeCache(conf, cache, confFileStamp, lcacheStatus, isArchive);
          lcacheStatus.initComplete();
        } else {
          if (fileStatus != null) {
            localizedPath =
                checkCacheStatusValidity(
                    conf, cache, confFileStamp, lcacheStatus, fileStatus, isArchive);
          } else {
            // if fileStatus is null, then the md5 must be correct
            // so there is no need to check for cache validity
            localizedPath = lcacheStatus.localizedLoadPath;
          }
        }
        createSymlink(conf, cache, lcacheStatus, isArchive, currentWorkDir, honorSymLinkConf);
      }

      // try deleting stuff if you can
      long size = 0;
      int numberSubDir = 0;
      synchronized (lcacheStatus) {
        synchronized (baseDirSize) {
          Long get = baseDirSize.get(lcacheStatus.getBaseDir());
          if (get != null) {
            size = get.longValue();
          } else {
            LOG.warn("Cannot find size of baseDir: " + lcacheStatus.getBaseDir());
          }
        }
        synchronized (baseDirNumberSubDir) {
          Integer get = baseDirNumberSubDir.get(lcacheStatus.getBaseDir());
          if (get != null) {
            numberSubDir = get.intValue();
          } else {
            LOG.warn("Cannot find subdirectories limit of baseDir: " + lcacheStatus.getBaseDir());
          }
        }
      }
      // setting the cache size to a default of 10GB
      long allowedSize = conf.getLong("local.cache.size", DEFAULT_CACHE_SIZE);
      long allowedNumberSubDir =
          conf.getLong("local.cache.numbersubdir", DEFAULT_CACHE_SUBDIR_LIMIT);
      if (allowedSize < size || allowedNumberSubDir < numberSubDir) {
        // try some cache deletions
        LOG.debug(
            "Start deleting released cache because"
                + " [size, allowedSize, numberSubDir, allowedNumberSubDir] ="
                + " ["
                + size
                + ", "
                + allowedSize
                + ", "
                + numberSubDir
                + ", "
                + allowedNumberSubDir
                + "]");
        deleteCache(conf, asyncDiskService);
      }
      initSuccessful = true;
      return localizedPath;
    } finally {
      if (!initSuccessful) {
        synchronized (cachedArchives) {
          lcacheStatus.refcount--;
        }
      }
    }
  }

  /**
   * Get the locally cached file or archive; it could either be previously cached (and valid) or
   * copy it from the {@link FileSystem} now.
   *
   * @param cache the cache to be localized, this should be specified as new
   *     URI(hdfs://hostname:port/absolute_path_to_file#LINKNAME). If no schema or hostname:port is
   *     provided the file is assumed to be in the filesystem being used in the Configuration
   * @param conf The Confguration file which contains the filesystem
   * @param baseDir The base cache Dir where you wnat to localize the files/archives
   * @param isArchive if the cache is an archive or a file. In case it is an archive with a .zip or
   *     .jar or .tar or .tgz or .tar.gz extension it will be unzipped/unjarred/untarred
   *     automatically and the directory where the archive is unzipped/unjarred/untarred is returned
   *     as the Path. In case of a file, the path to the file is returned
   * @param confFileStamp this is the hdfs file modification timestamp to verify that the file to be
   *     cached hasn't changed since the job started
   * @param currentWorkDir this is the directory where you would want to create symlinks for the
   *     locally cached files/archives
   * @return the path to directory where the archives are unjarred in case of archives, the path to
   *     the file where the file is copied locally
   * @throws IOException
   */
  public static Path getLocalCache(
      URI cache,
      Configuration conf,
      Path baseDir,
      boolean isArchive,
      long confFileStamp,
      Path currentWorkDir,
      MRAsyncDiskService asyncDiskService)
      throws IOException {
    return getLocalCache(
        cache, conf, baseDir, null, isArchive, confFileStamp, currentWorkDir, asyncDiskService);
  }

  /**
   * This is the opposite of getlocalcache. When you are done with using the cache, you need to
   * release the cache
   *
   * @param cache The cache URI to be released
   * @param conf configuration which contains the filesystem the cache is contained in.
   * @throws IOException
   */
  public static void releaseCache(URI cache, Configuration conf, long timeStamp)
      throws IOException {
    String cacheId = getKey(cache, conf, timeStamp);

    synchronized (cachedArchives) {
      CacheStatus lcacheStatus = cachedArchives.get(cacheId);
      if (lcacheStatus == null) {
        LOG.warn(
            "Cannot find localized cache: " + cache + " (key: " + cacheId + ") in releaseCache!");
        return;
      }
      lcacheStatus.refcount--;
    }
  }

  /** Runnable which removes the cache directories from the disk */
  private static class CacheFileCleanTask implements Runnable {
    private MRAsyncDiskService asyncDiskService;
    private LocalFileSystem fs;
    private List<CacheStatus> toBeDeletedCache;

    public CacheFileCleanTask(
        MRAsyncDiskService asyncDiskService,
        LocalFileSystem fs,
        List<CacheStatus> toBeDeletedCache) {
      this.asyncDiskService = asyncDiskService;
      this.fs = fs;
      this.toBeDeletedCache = toBeDeletedCache;
    }

    @Override
    public void run() {
      for (CacheStatus lcacheStatus : toBeDeletedCache) {
        synchronized (lcacheStatus) {
          Path fullUniqueParentDir =
              new Path(lcacheStatus.localizedBaseDir, lcacheStatus.uniqueParentDir);
          try {
            LOG.info("Deleting local cached path: " + fullUniqueParentDir.toString());
            deleteLocalPath(asyncDiskService, fs, fullUniqueParentDir);
            // decrement the size of the cache from baseDirSize
            deleteCacheInfoUpdate(lcacheStatus);
            LOG.info("Removed cache " + lcacheStatus.localizedLoadPath);
          } catch (IOException e) {
            LOG.warn("Error when deleting " + fullUniqueParentDir, e);
          }
        }
      }
    }
  }
  // To delete the caches which have a refcount of zero

  private static void deleteCache(Configuration conf, MRAsyncDiskService asyncDiskService)
      throws IOException {
    List<CacheStatus> deleteSet = new LinkedList<CacheStatus>();
    // try deleting cache Status with refcount of zero
    synchronized (cachedArchives) {
      for (Iterator<String> it = cachedArchives.keySet().iterator(); it.hasNext(); ) {
        String cacheId = (String) it.next();
        CacheStatus lcacheStatus = cachedArchives.get(cacheId);
        if (lcacheStatus.refcount == 0) {
          // delete this cache entry from the global list
          // and mark the localized file for deletion
          deleteSet.add(lcacheStatus);
          it.remove();
        }
      }
    }
    // do the deletion asynchronously, after releasing the global lock
    Thread cacheFileCleaner =
        new Thread(new CacheFileCleanTask(asyncDiskService, FileSystem.getLocal(conf), deleteSet));
    cacheFileCleaner.start();
  }

  /**
   * Delete a local path with asyncDiskService if available, or otherwise synchronously with local
   * file system.
   */
  private static void deleteLocalPath(
      MRAsyncDiskService asyncDiskService, LocalFileSystem fs, Path path) throws IOException {
    boolean deleted = false;
    if (asyncDiskService != null) {
      // Try to delete using asyncDiskService
      String localPathToDelete = path.toUri().getPath();
      deleted = asyncDiskService.moveAndDeleteAbsolutePath(localPathToDelete);
      if (!deleted) {
        LOG.warn(
            "Cannot find DistributedCache path "
                + localPathToDelete
                + " on any of the asyncDiskService volumes!");
      }
    }
    if (!deleted) {
      // If no asyncDiskService, we will delete the files synchronously
      fs.delete(path, true);
    }
    LOG.info("Deleted path " + path);
  }

  /*
   * Returns the relative path of the dir this cache will be localized in
   * relative path that this cache will be localized in. For
   * hdfs://hostname:port/absolute_path -- the relative path is
   * hostname/absolute path -- if it is just /absolute_path -- then the
   * relative path is hostname of DFS this mapred cluster is running
   * on/absolute_path
   */
  public static String makeRelative(URI cache, Configuration conf) throws IOException {
    String host = cache.getHost();
    if (host == null) {
      host = cache.getScheme();
    }
    if (host == null) {
      URI defaultUri = FileSystem.get(conf).getUri();
      host = defaultUri.getHost();
      if (host == null) {
        host = defaultUri.getScheme();
      }
    }
    String path = host + cache.getPath();
    path = path.replace(":/", "/"); // remove windows device colon
    return path;
  }

  static String getKey(URI cache, Configuration conf, long timeStamp) throws IOException {
    return makeRelative(cache, conf) + String.valueOf(timeStamp);
  }

  private static Path checkCacheStatusValidity(
      Configuration conf,
      URI cache,
      long confFileStamp,
      CacheStatus cacheStatus,
      FileStatus fileStatus,
      boolean isArchive)
      throws IOException {
    FileSystem fs = FileSystem.get(cache, conf);
    // Has to be
    if (!ifExistsAndFresh(conf, fs, cache, confFileStamp, cacheStatus, fileStatus)) {
      throw new IOException(
          "Stale cache file: " + cacheStatus.localizedLoadPath + " for cache-file: " + cache);
    }
    LOG.info(
        String.format(
            "Using existing cache of %s->%s", cache.toString(), cacheStatus.localizedLoadPath));
    return cacheStatus.localizedLoadPath;
  }

  private static void createSymlink(
      Configuration conf,
      URI cache,
      CacheStatus cacheStatus,
      boolean isArchive,
      Path currentWorkDir,
      boolean honorSymLinkConf)
      throws IOException {
    boolean doSymlink = honorSymLinkConf && DistributedCache.getSymlink(conf);
    if (cache.getFragment() == null) {
      doSymlink = false;
    }
    String link = currentWorkDir.toString() + Path.SEPARATOR + cache.getFragment();
    File flink = new File(link);
    if (doSymlink) {
      if (!flink.exists()) {
        FileUtil.symLink(cacheStatus.localizedLoadPath.toString(), link);
      }
    }
  }

  // the method which actually copies the caches locally and unjars/unzips them
  // and does chmod for the files
  private static Path localizeCache(
      Configuration conf, URI cache, long confFileStamp, CacheStatus cacheStatus, boolean isArchive)
      throws IOException {
    FileSystem fs = getFileSystem(cache, conf);
    FileSystem localFs = FileSystem.getLocal(conf);
    Path parchive = null;

    if (isArchive) {
      parchive =
          new Path(
              cacheStatus.localizedLoadPath, new Path(cacheStatus.localizedLoadPath.getName()));
    } else {
      parchive = cacheStatus.localizedLoadPath;
    }
    if (!localFs.mkdirs(parchive.getParent())) {
      throw new IOException(
          "Mkdirs failed to create directory " + cacheStatus.localizedLoadPath.toString());
    }
    String cacheId = cache.getPath();

    fs.copyToLocalFile(new Path(cacheId), parchive);
    if (isArchive) {
      String tmpArchive = parchive.toString().toLowerCase();
      File srcFile = new File(parchive.toString());
      File destDir = new File(parchive.getParent().toString());
      if (tmpArchive.endsWith(".jar")) {
        RunJar.unJar(srcFile, destDir);
      } else if (tmpArchive.endsWith(".zip")) {
        FileUtil.unZip(srcFile, destDir);
      } else if (isTarFile(tmpArchive)) {
        FileUtil.unTar(srcFile, destDir);
      }
      // else will not do anyhting
      // and copy the file into the dir as it is
    }
    long cacheSize = FileUtil.getDU(new File(parchive.getParent().toString()));
    cacheStatus.size = cacheSize;
    addCacheInfoUpdate(cacheStatus);

    // do chmod here
    try {
      // Setting recursive permission to grant everyone read and execute
      Path localDir = new Path(cacheStatus.localizedBaseDir, cacheStatus.uniqueParentDir);
      LOG.info("Doing chmod on localdir :" + localDir);
      FileUtil.chmod(localDir.toString(), "ugo+rx", true);
    } catch (InterruptedException e) {
      LOG.warn("Exception in chmod" + e.toString());
    }

    // update cacheStatus to reflect the newly cached file
    cacheStatus.mtime = getTimestamp(conf, cache);
    return cacheStatus.localizedLoadPath;
  }

  private static boolean isTarFile(String filename) {
    return (filename.endsWith(".tgz") || filename.endsWith(".tar.gz") || filename.endsWith(".tar"));
  }

  // Checks if the cache has already been localized and is fresh
  private static boolean ifExistsAndFresh(
      Configuration conf,
      FileSystem fs,
      URI cache,
      long confFileStamp,
      CacheStatus lcacheStatus,
      FileStatus fileStatus)
      throws IOException {
    // check for existence of the cache
    long dfsFileStamp;
    if (fileStatus != null) {
      dfsFileStamp = fileStatus.getModificationTime();
    } else {
      dfsFileStamp = getTimestamp(conf, cache);
    }

    // ensure that the file on hdfs hasn't been modified since the job started
    if (dfsFileStamp != confFileStamp) {
      LOG.fatal("File: " + cache + " has changed on HDFS since job started");
      throw new IOException("File: " + cache + " has changed on HDFS since job started");
    }

    if (dfsFileStamp != lcacheStatus.mtime) {
      // needs refreshing
      return false;
    }

    return true;
  }

  /**
   * Returns mtime of a given cache file on hdfs.
   *
   * @param conf configuration
   * @param cache cache file
   * @return mtime of a given cache file on hdfs
   * @throws IOException
   */
  public static long getTimestamp(Configuration conf, URI cache) throws IOException {
    FileSystem fileSystem = FileSystem.get(cache, conf);
    Path filePath = new Path(cache.getPath());

    return fileSystem.getFileStatus(filePath).getModificationTime();
  }

  /**
   * Returns the status of a given cache file on hdfs.
   *
   * @param conf configuration
   * @param cache cache file
   * @return FileStatus object of the file
   * @throws IOException
   */
  public static FileStatus getFileStatus(Configuration conf, URI cache) throws IOException {
    FileSystem fileSystem = FileSystem.get(cache, conf);
    Path filePath = new Path(cache.getPath());

    return fileSystem.getFileStatus(filePath);
  }

  /**
   * This method create symlinks for all files in a given dir in another directory
   *
   * @param conf the configuration
   * @param jobCacheDir the target directory for creating symlinks
   * @param workDir the directory in which the symlinks are created
   * @throws IOException
   */
  public static void createAllSymlink(Configuration conf, File jobCacheDir, File workDir)
      throws IOException {
    if ((jobCacheDir == null || !jobCacheDir.isDirectory())
        || workDir == null
        || (!workDir.isDirectory())) {
      return;
    }
    boolean createSymlink = getSymlink(conf);
    if (createSymlink) {
      File[] list = jobCacheDir.listFiles();
      for (int i = 0; i < list.length; i++) {
        FileUtil.symLink(
            list[i].getAbsolutePath(), new File(workDir, list[i].getName()).toString());
      }
    }
  }

  private static String getFileSysName(URI url) {
    String fsname = url.getScheme();
    if ("hdfs".equals(fsname)) {
      String host = url.getHost();
      int port = url.getPort();
      return (port == (-1)) ? host : (host + ":" + port);
    } else {
      return null;
    }
  }

  private static FileSystem getFileSystem(URI cache, Configuration conf) throws IOException {
    String fileSysName = getFileSysName(cache);
    if (fileSysName != null) return FileSystem.getNamed(fileSysName, conf);
    else return FileSystem.get(conf);
  }

  /**
   * Set the configuration with the given set of archives
   *
   * @param archives The list of archives that need to be localized
   * @param conf Configuration which will be changed
   */
  public static void setCacheArchives(URI[] archives, Configuration conf) {
    String sarchives = StringUtils.uriToString(archives);
    conf.set("mapred.cache.archives", sarchives);
  }

  /**
   * Set the configuration with the given set of files
   *
   * @param files The list of files that need to be localized
   * @param conf Configuration which will be changed
   */
  public static void setCacheFiles(URI[] files, Configuration conf) {
    String sfiles = StringUtils.uriToString(files);
    conf.set("mapred.cache.files", sfiles);
  }

  /**
   * Get cache archives set in the Configuration
   *
   * @param conf The configuration which contains the archives
   * @return A URI array of the caches set in the Configuration
   * @throws IOException
   */
  public static URI[] getCacheArchives(Configuration conf) throws IOException {
    return StringUtils.stringToURI(conf.getStrings("mapred.cache.archives"));
  }

  /**
   * Get cache archives set in the Configuration
   *
   * @param conf The configuration which contains the archives
   * @return A URI array of the caches set in the Configuration
   * @throws IOException
   */
  public static URI[] getSharedCacheArchives(Configuration conf) throws IOException {
    return StringUtils.stringToURI(conf.getStrings("mapred.cache.shared.archives"));
  }

  /**
   * Get cache files set in the Configuration
   *
   * @param conf The configuration which contains the files
   * @return A URI array of the files set in the Configuration
   * @throws IOException
   */
  public static URI[] getCacheFiles(Configuration conf) throws IOException {
    return StringUtils.stringToURI(conf.getStrings("mapred.cache.files"));
  }

  /**
   * Get cache files set in the Configuration
   *
   * @param conf The configuration which contains the files
   * @return A URI array of the files set in the Configuration
   * @throws IOException
   */
  public static URI[] getSharedCacheFiles(Configuration conf) throws IOException {
    return StringUtils.stringToURI(conf.getStrings("mapred.cache.shared.files"));
  }

  /**
   * Return the path array of the localized caches
   *
   * @param conf Configuration that contains the localized archives
   * @return A path array of localized caches
   * @throws IOException
   */
  public static Path[] getLocalCacheArchives(Configuration conf) throws IOException {
    return StringUtils.stringToPath(conf.getStrings("mapred.cache.localArchives"));
  }

  /**
   * Return the path array of the localized caches
   *
   * @param conf Configuration that contains the localized archives
   * @return A path array of localized caches
   * @throws IOException
   */
  public static Path[] getLocalSharedCacheArchives(Configuration conf) throws IOException {
    return StringUtils.stringToPath(conf.getStrings("mapred.cache.shared.localArchives"));
  }

  /**
   * Return the path array of the localized files
   *
   * @param conf Configuration that contains the localized files
   * @return A path array of localized files
   * @throws IOException
   */
  public static Path[] getLocalCacheFiles(Configuration conf) throws IOException {
    return StringUtils.stringToPath(conf.getStrings("mapred.cache.localFiles"));
  }

  /**
   * Return the path array of the localized files
   *
   * @param conf Configuration that contains the localized files
   * @return A path array of localized files
   * @throws IOException
   */
  public static Path[] getLocalSharedCacheFiles(Configuration conf) throws IOException {
    return StringUtils.stringToPath(conf.getStrings("mapred.cache.shared.localFiles"));
  }

  /**
   * Get the timestamps of the archives
   *
   * @param conf The configuration which stored the timestamps
   * @return a string array of timestamps
   * @throws IOException
   */
  public static String[] getArchiveTimestamps(Configuration conf) {
    return conf.getStrings("mapred.cache.archives.timestamps");
  }

  /**
   * Get the timestamps of the files
   *
   * @param conf The configuration which stored the timestamps
   * @return a string array of timestamps
   * @throws IOException
   */
  public static String[] getFileTimestamps(Configuration conf) {
    return conf.getStrings("mapred.cache.files.timestamps");
  }

  public static String[] getSharedArchiveLength(Configuration conf) {
    return conf.getStrings("mapred.cache.shared.archives.length");
  }

  public static String[] getSharedFileLength(Configuration conf) {
    return conf.getStrings("mapred.cache.shared.files.length");
  }

  /**
   * This is to check the timestamp of the archives to be localized
   *
   * @param conf Configuration which stores the timestamp's
   * @param timestamps comma separated list of timestamps of archives. The order should be the same
   *     as the order in which the archives are added.
   */
  public static void setArchiveTimestamps(Configuration conf, String timestamps) {
    conf.set("mapred.cache.archives.timestamps", timestamps);
  }

  public static void setSharedArchiveLength(Configuration conf, String length) {
    conf.set("mapred.cache.shared.archives.length", length);
  }

  /**
   * This is to check the timestamp of the files to be localized
   *
   * @param conf Configuration which stores the timestamp's
   * @param timestamps comma separated list of timestamps of files. The order should be the same as
   *     the order in which the files are added.
   */
  public static void setFileTimestamps(Configuration conf, String timestamps) {
    conf.set("mapred.cache.files.timestamps", timestamps);
  }

  public static void setSharedFileLength(Configuration conf, String length) {
    conf.set("mapred.cache.shared.files.length", length);
  }

  /**
   * Set the conf to contain the location for localized archives
   *
   * @param conf The conf to modify to contain the localized caches
   * @param str a comma separated list of local archives
   */
  public static void setLocalArchives(Configuration conf, String str) {
    conf.set("mapred.cache.localArchives", str);
  }

  /**
   * Set the conf to contain the location for localized archives
   *
   * @param conf The conf to modify to contain the localized caches
   * @param str a comma separated list of local archives
   */
  public static void setLocalSharedArchives(Configuration conf, String str) {
    conf.set("mapred.cache.shared.localArchives", str);
  }

  /**
   * Set the conf to contain the location for localized files
   *
   * @param conf The conf to modify to contain the localized caches
   * @param str a comma separated list of local files
   */
  public static void setLocalFiles(Configuration conf, String str) {
    conf.set("mapred.cache.localFiles", str);
  }

  /**
   * Set the conf to contain the location for localized files
   *
   * @param conf The conf to modify to contain the localized caches
   * @param str a comma separated list of local files
   */
  public static void setLocalSharedFiles(Configuration conf, String str) {
    conf.set("mapred.cache.shared.localFiles", str);
  }

  /**
   * Add a archives to be localized to the conf
   *
   * @param uri The uri of the cache to be localized
   * @param conf Configuration to add the cache to
   */
  public static void addCacheArchive(URI uri, Configuration conf) {
    String archives = conf.get("mapred.cache.archives");
    conf.set(
        "mapred.cache.archives",
        archives == null ? uri.toString() : archives + "," + uri.toString());
  }

  /**
   * Add a archives to be localized to the conf
   *
   * @param uri The uri of the cache to be localized
   * @param conf Configuration to add the cache to
   */
  public static void addSharedCacheArchive(URI uri, Configuration conf) {
    String archives = conf.get("mapred.cache.shared.archives");
    conf.set(
        "mapred.cache.shared.archives",
        archives == null ? uri.toString() : archives + "," + uri.toString());
  }

  /**
   * Add a file to be localized to the conf
   *
   * @param uri The uri of the cache to be localized
   * @param conf Configuration to add the cache to
   */
  public static void addCacheFile(URI uri, Configuration conf) {
    String files = conf.get("mapred.cache.files");
    conf.set("mapred.cache.files", files == null ? uri.toString() : files + "," + uri.toString());
  }

  /**
   * Add a file to be localized to the conf
   *
   * @param uri The uri of the cache to be localized
   * @param conf Configuration to add the cache to
   */
  public static void addSharedCacheFile(URI uri, Configuration conf) {
    String files = conf.get("mapred.cache.shared.files");
    conf.set(
        "mapred.cache.shared.files", files == null ? uri.toString() : files + "," + uri.toString());
  }

  /**
   * Add an file path to the current set of classpath entries It adds the file to cache as well.
   *
   * @param file Path of the file to be added
   * @param conf Configuration that contains the classpath setting
   */
  public static void addFileToClassPath(Path file, Configuration conf) throws IOException {
    String classpath = conf.get("mapred.job.classpath.files");
    conf.set(
        "mapred.job.classpath.files",
        classpath == null
            ? file.toString()
            : classpath + System.getProperty("path.separator") + file.toString());
    URI uri = file.makeQualified(file.getFileSystem(conf)).toUri();

    addCacheFile(uri, conf);
  }

  /**
   * Get the file entries in classpath as an array of Path
   *
   * @param conf Configuration that contains the classpath setting
   */
  public static Path[] getFileClassPaths(Configuration conf) {
    String classpath = conf.get("mapred.job.classpath.files");
    if (classpath == null) return null;
    ArrayList list =
        Collections.list(new StringTokenizer(classpath, System.getProperty("path.separator")));
    Path[] paths = new Path[list.size()];
    for (int i = 0; i < list.size(); i++) {
      paths[i] = new Path((String) list.get(i));
    }
    return paths;
  }

  private static URI addArchiveToClassPathHelper(Path archive, Configuration conf)
      throws IOException {

    String classpath = conf.get("mapred.job.classpath.archives");

    // the scheme/authority use ':' as separator. put the unqualified path in classpath
    String archivePath = archive.toUri().getPath();

    conf.set(
        "mapred.job.classpath.archives",
        classpath == null
            ? archivePath
            : classpath + System.getProperty("path.separator") + archivePath);
    return archive.makeQualified(archive.getFileSystem(conf)).toUri();
  }

  /**
   * Add an archive path to the current set of classpath entries. It adds the archive to cache as
   * well.
   *
   * @param archive Path of the archive to be added
   * @param conf Configuration that contains the classpath setting
   */
  public static void addArchiveToClassPath(Path archive, Configuration conf) throws IOException {
    URI uri = addArchiveToClassPathHelper(archive, conf);
    addCacheArchive(uri, conf);
  }

  /**
   * Add an archive path to the current set of classpath entries. It adds the archive to cache as
   * well.
   *
   * @param archive Path of the archive to be added
   * @param conf Configuration that contains the classpath setting
   */
  public static void addSharedArchiveToClassPath(Path archive, Configuration conf)
      throws IOException {
    URI uri = addArchiveToClassPathHelper(archive, conf);
    addSharedCacheArchive(uri, conf);
  }

  /**
   * Get the archive entries in classpath as an array of Path
   *
   * @param conf Configuration that contains the classpath setting
   */
  public static Path[] getArchiveClassPaths(Configuration conf) {
    String classpath = conf.get("mapred.job.classpath.archives");
    if (classpath == null) return null;
    ArrayList list =
        Collections.list(new StringTokenizer(classpath, System.getProperty("path.separator")));
    Path[] paths = new Path[list.size()];
    for (int i = 0; i < list.size(); i++) {
      paths[i] = new Path((String) list.get(i));
    }
    return paths;
  }

  /**
   * This method allows you to create symlinks in the current working directory of the task to all
   * the cache files/archives
   *
   * @param conf the jobconf
   */
  public static void createSymlink(Configuration conf) {
    conf.set("mapred.create.symlink", "yes");
  }

  /**
   * This method checks to see if symlinks are to be create for the localized cache files in the
   * current working directory
   *
   * @param conf the jobconf
   * @return true if symlinks are to be created- else return false
   */
  public static boolean getSymlink(Configuration conf) {
    String result = conf.get("mapred.create.symlink");
    if ("yes".equals(result)) {
      return true;
    }
    return false;
  }

  /**
   * This method checks if there is a conflict in the fragment names of the uris. Also makes sure
   * that each uri has a fragment. It is only to be called if you want to create symlinks for the
   * various archives and files.
   *
   * @param uriFiles The uri array of urifiles
   * @param uriArchives the uri array of uri archives
   */
  public static boolean checkURIs(URI[] uriFiles, URI[] uriArchives) {
    if ((uriFiles == null) && (uriArchives == null)) {
      return true;
    }
    if (uriFiles != null) {
      for (int i = 0; i < uriFiles.length; i++) {
        String frag1 = uriFiles[i].getFragment();
        if (frag1 == null) return false;
        for (int j = i + 1; j < uriFiles.length; j++) {
          String frag2 = uriFiles[j].getFragment();
          if (frag2 == null) return false;
          if (frag1.equalsIgnoreCase(frag2)) return false;
        }
        if (uriArchives != null) {
          for (int j = 0; j < uriArchives.length; j++) {
            String frag2 = uriArchives[j].getFragment();
            if (frag2 == null) {
              return false;
            }
            if (frag1.equalsIgnoreCase(frag2)) return false;
            for (int k = j + 1; k < uriArchives.length; k++) {
              String frag3 = uriArchives[k].getFragment();
              if (frag3 == null) return false;
              if (frag2.equalsIgnoreCase(frag3)) return false;
            }
          }
        }
      }
    }
    return true;
  }

  private static class CacheStatus {
    // the local load path of this cache
    Path localizedLoadPath;

    // the base dir where the cache lies
    Path localizedBaseDir;

    // the unique directory in localizedBaseDir, where the cache lies
    Path uniqueParentDir;

    // the size of this cache
    long size;

    // number of instances using this cache
    int refcount;

    // the cache-file modification time
    long mtime;

    // is it initialized
    boolean inited = false;

    public CacheStatus(Path baseDir, Path localLoadPath, Path uniqueParentDir) {
      super();
      this.localizedLoadPath = localLoadPath;
      this.refcount = 0;
      this.mtime = -1;
      this.localizedBaseDir = baseDir;
      this.size = 0;
      this.uniqueParentDir = uniqueParentDir;
    }

    // get the base dir for the cache
    Path getBaseDir() {
      return localizedBaseDir;
    }

    // Is it initialized?
    boolean isInited() {
      return inited;
    }

    // mark it as initalized
    void initComplete() {
      inited = true;
    }
  }

  /**
   * Clear the entire contents of the cache and delete the backing files. This should only be used
   * when the server is reinitializing, because the users are going to lose their files.
   */
  public static void purgeCache(Configuration conf, MRAsyncDiskService service) throws IOException {
    synchronized (cachedArchives) {
      LocalFileSystem localFs = FileSystem.getLocal(conf);
      for (Map.Entry<String, CacheStatus> f : cachedArchives.entrySet()) {
        try {
          deleteLocalPath(service, localFs, f.getValue().localizedLoadPath);
        } catch (IOException ie) {
          LOG.debug("Error cleaning up cache", ie);
        }
      }
      cachedArchives.clear();
    }
  }

  /**
   * Update the maps baseDirSize and baseDirNumberSubDir when deleting cache.
   *
   * @param cacheStatus cache status of the cache is deleted
   */
  private static void deleteCacheInfoUpdate(CacheStatus cacheStatus) {
    if (!cacheStatus.isInited()) {
      // if it is not created yet, do nothing.
      return;
    }
    synchronized (baseDirSize) {
      Long dirSize = baseDirSize.get(cacheStatus.getBaseDir());
      if (dirSize != null) {
        dirSize -= cacheStatus.size;
        baseDirSize.put(cacheStatus.getBaseDir(), dirSize);
      }
    }
    synchronized (baseDirNumberSubDir) {
      Integer dirSubDir = baseDirNumberSubDir.get(cacheStatus.getBaseDir());
      if (dirSubDir != null) {
        dirSubDir--;
        baseDirNumberSubDir.put(cacheStatus.getBaseDir(), dirSubDir);
      }
    }
  }

  /**
   * Update the maps baseDirSize and baseDirNumberSubDir when adding cache.
   *
   * @param cacheStatus cache status of the cache is added
   */
  private static void addCacheInfoUpdate(CacheStatus cacheStatus) {
    long cacheSize = cacheStatus.size;
    synchronized (baseDirSize) {
      Long dirSize = baseDirSize.get(cacheStatus.getBaseDir());
      if (dirSize == null) {
        dirSize = Long.valueOf(cacheSize);
      } else {
        dirSize += cacheSize;
      }
      baseDirSize.put(cacheStatus.getBaseDir(), dirSize);
    }
    synchronized (baseDirNumberSubDir) {
      Integer dirSubDir = baseDirNumberSubDir.get(cacheStatus.getBaseDir());
      if (dirSubDir == null) {
        dirSubDir = 1;
      } else {
        dirSubDir += 1;
      }
      baseDirNumberSubDir.put(cacheStatus.getBaseDir(), dirSubDir);
    }
  }
}
/**
 * Shared functionality for hadoopStreaming formats. A custom reader can be defined to be a
 * RecordReader with the constructor below and is selected with the option bin/hadoopStreaming
 * -inputreader ...
 *
 * @see StreamXmlRecordReader
 */
public abstract class StreamBaseRecordReader implements RecordReader<Text, Text> {

  protected static final Log LOG = LogFactory.getLog(StreamBaseRecordReader.class.getName());

  // custom JobConf properties for this class are prefixed with this namespace
  static final String CONF_NS = "stream.recordreader.";

  public StreamBaseRecordReader(
      FSDataInputStream in, FileSplit split, Reporter reporter, JobConf job, FileSystem fs)
      throws IOException {
    in_ = in;
    split_ = split;
    start_ = split_.getStart();
    length_ = split_.getLength();
    end_ = start_ + length_;
    splitName_ = split_.getPath().getName();
    reporter_ = reporter;
    job_ = job;
    fs_ = fs;

    statusMaxRecordChars_ = job_.getInt(CONF_NS + "statuschars", 200);
  }

  /// RecordReader API

  /** Read a record. Implementation should call numRecStats at the end */
  public abstract boolean next(Text key, Text value) throws IOException;

  /** This implementation always returns true. */
  public void validateInput(JobConf job) throws IOException {}

  /** Returns the current position in the input. */
  public synchronized long getPos() throws IOException {
    return in_.getPos();
  }

  /** Close this to future operations. */
  public synchronized void close() throws IOException {
    in_.close();
  }

  public float getProgress() throws IOException {
    if (end_ == start_) {
      return 1.0f;
    } else {
      return ((float) (in_.getPos() - start_)) / ((float) (end_ - start_));
    }
  }

  public Text createKey() {
    return new Text();
  }

  public Text createValue() {
    return new Text();
  }

  /// StreamBaseRecordReader API

  /**
   * Implementation should seek forward in_ to the first byte of the next record. The initial byte
   * offset in the stream is arbitrary.
   */
  public abstract void seekNextRecordBoundary() throws IOException;

  void numRecStats(byte[] record, int start, int len) throws IOException {
    numRec_++;
    if (numRec_ == nextStatusRec_) {
      String recordStr = new String(record, start, Math.min(len, statusMaxRecordChars_), "UTF-8");
      nextStatusRec_ += 100; // *= 10;
      String status = getStatus(recordStr);
      LOG.info(status);
      reporter_.setStatus(status);
    }
  }

  long lastMem = 0;

  String getStatus(CharSequence record) {
    long pos = -1;
    try {
      pos = getPos();
    } catch (IOException io) {
    }
    String recStr;
    if (record.length() > statusMaxRecordChars_) {
      recStr = record.subSequence(0, statusMaxRecordChars_) + "...";
    } else {
      recStr = record.toString();
    }
    String unqualSplit =
        split_.getPath().getName() + ":" + split_.getStart() + "+" + split_.getLength();
    String status =
        "HSTR "
            + StreamUtil.HOST
            + " "
            + numRec_
            + ". pos="
            + pos
            + " "
            + unqualSplit
            + " Processing record="
            + recStr;
    status += " " + splitName_;
    return status;
  }

  FSDataInputStream in_;
  FileSplit split_;
  long start_;
  long end_;
  long length_;
  String splitName_;
  Reporter reporter_;
  JobConf job_;
  FileSystem fs_;
  int numRec_ = 0;
  int nextStatusRec_ = 1;
  int statusMaxRecordChars_;
}
  private static class AllocatorPerContext {

    private final Log LOG = LogFactory.getLog(AllocatorPerContext.class);

    private int dirNumLastAccessed;
    private Random dirIndexRandomizer = new Random();
    private FileSystem localFS;
    private DF[] dirDF;
    private String contextCfgItemName;
    private Path[] localDirsPath;
    private String savedLocalDirs = "";

    public AllocatorPerContext(String contextCfgItemName) {
      this.contextCfgItemName = contextCfgItemName;
    }

    /**
     * This method gets called everytime before any read/write to make sure that any change to
     * localDirs is reflected immediately.
     */
    private synchronized void confChanged(Configuration conf) throws IOException {
      String newLocalDirs = conf.get(contextCfgItemName);
      if (!newLocalDirs.equals(savedLocalDirs)) {
        String[] localDirs = conf.getStrings(contextCfgItemName);
        localFS = FileSystem.getLocal(conf);
        int numDirs = localDirs.length;
        ArrayList<String> dirs = new ArrayList<String>(numDirs);
        ArrayList<DF> dfList = new ArrayList<DF>(numDirs);
        for (int i = 0; i < numDirs; i++) {
          try {
            // filter problematic directories
            Path tmpDir = new Path(localDirs[i]);
            if (localFS.mkdirs(tmpDir) || localFS.exists(tmpDir)) {
              try {
                DiskChecker.checkDir(new File(localDirs[i]));
                dirs.add(localDirs[i]);
                dfList.add(new DF(new File(localDirs[i]), 30000));
              } catch (DiskErrorException de) {
                LOG.warn(localDirs[i] + "is not writable\n" + StringUtils.stringifyException(de));
              }
            } else {
              LOG.warn("Failed to create " + localDirs[i]);
            }
          } catch (IOException ie) {
            LOG.warn(
                "Failed to create "
                    + localDirs[i]
                    + ": "
                    + ie.getMessage()
                    + "\n"
                    + StringUtils.stringifyException(ie));
          } // ignore
        }
        localDirsPath = new Path[dirs.size()];
        for (int i = 0; i < localDirsPath.length; i++) {
          localDirsPath[i] = new Path(dirs.get(i));
        }
        dirDF = dfList.toArray(new DF[dirs.size()]);
        savedLocalDirs = newLocalDirs;

        // randomize the first disk picked in the round-robin selection
        dirNumLastAccessed = dirIndexRandomizer.nextInt(dirs.size());
      }
    }

    private Path createPath(Path path, boolean checkWrite) throws IOException {
      Path file = new Path(localDirsPath[dirNumLastAccessed], path);

      if (checkWrite) {
        // check whether we are able to create a directory here. If the disk
        // happens to be RDONLY we will fail
        try {
          DiskChecker.checkDir(new File(file.getParent().toUri().getPath()));
        } catch (DiskErrorException d) {
          LOG.warn(StringUtils.stringifyException(d));
          return null;
        }
      }
      return file;
    }

    /**
     * Get the current directory index.
     *
     * @return the current directory index.
     */
    int getCurrentDirectoryIndex() {
      return dirNumLastAccessed;
    }

    /**
     * Get a path from the local FS. If size is known, we go round-robin over the set of disks (via
     * the configured dirs) and return the first complete path which has enough space.
     *
     * <p>If size is not known, use roulette selection -- pick directories with probability
     * proportional to their available space.
     */
    public synchronized Path getLocalPathForWrite(
        String pathStr, long size, Configuration conf, boolean checkWrite) throws IOException {
      confChanged(conf);
      int numDirs = localDirsPath.length;
      int numDirsSearched = 0;
      // remove the leading slash from the path (to make sure that the uri
      // resolution results in a valid path on the dir being checked)
      if (pathStr.startsWith("/")) {
        pathStr = pathStr.substring(1);
      }
      Path returnPath = null;
      Path path = new Path(pathStr);

      if (size == SIZE_UNKNOWN) { // do roulette selection: pick dir with probability
        // proportional to available size
        long[] availableOnDisk = new long[dirDF.length];
        long totalAvailable = 0;

        // build the "roulette wheel"
        for (int i = 0; i < dirDF.length; ++i) {
          availableOnDisk[i] = dirDF[i].getAvailable();
          totalAvailable += availableOnDisk[i];
        }

        // Keep rolling the wheel till we get a valid path
        Random r = new java.util.Random();
        while (numDirsSearched < numDirs && returnPath == null) {
          long randomPosition = Math.abs(r.nextLong()) % totalAvailable;
          int dir = 0;
          while (randomPosition > availableOnDisk[dir]) {
            randomPosition -= availableOnDisk[dir];
            dir++;
          }
          dirNumLastAccessed = dir;
          returnPath = createPath(path, checkWrite);
          if (returnPath == null) {
            totalAvailable -= availableOnDisk[dir];
            availableOnDisk[dir] = 0; // skip this disk
            numDirsSearched++;
          }
        }
      } else {
        while (numDirsSearched < numDirs && returnPath == null) {
          long capacity = dirDF[dirNumLastAccessed].getAvailable();
          if (capacity > size) {
            returnPath = createPath(path, checkWrite);
          }
          dirNumLastAccessed++;
          dirNumLastAccessed = dirNumLastAccessed % numDirs;
          numDirsSearched++;
        }
      }
      if (returnPath != null) {
        return returnPath;
      }

      // no path found
      throw new DiskErrorException("Could not find any valid local " + "directory for " + pathStr);
    }

    /**
     * Creates a file on the local FS. Pass size as {@link LocalDirAllocator.SIZE_UNKNOWN} if not
     * known apriori. We round-robin over the set of disks (via the configured dirs) and return a
     * file on the first path which has enough space. The file is guaranteed to go away when the JVM
     * exits.
     */
    public File createTmpFileForWrite(String pathStr, long size, Configuration conf)
        throws IOException {

      // find an appropriate directory
      Path path = getLocalPathForWrite(pathStr, size, conf, true);
      File dir = new File(path.getParent().toUri().getPath());
      String prefix = path.getName();

      // create a temp file on this directory
      File result = File.createTempFile(prefix, null, dir);
      result.deleteOnExit();
      return result;
    }

    /**
     * Get a path from the local FS for reading. We search through all the configured dirs for the
     * file's existence and return the complete path to the file when we find one
     */
    public synchronized Path getLocalPathToRead(String pathStr, Configuration conf)
        throws IOException {
      confChanged(conf);
      int numDirs = localDirsPath.length;
      int numDirsSearched = 0;
      // remove the leading slash from the path (to make sure that the uri
      // resolution results in a valid path on the dir being checked)
      if (pathStr.startsWith("/")) {
        pathStr = pathStr.substring(1);
      }
      Path childPath = new Path(pathStr);
      while (numDirsSearched < numDirs) {
        Path file = new Path(localDirsPath[numDirsSearched], childPath);
        if (localFS.exists(file)) {
          return file;
        }
        numDirsSearched++;
      }

      // no path found
      throw new DiskErrorException(
          "Could not find " + pathStr + " in any of" + " the configured local directories");
    }

    private static class PathIterator implements Iterator<Path>, Iterable<Path> {
      private final FileSystem fs;
      private final String pathStr;
      private int i = 0;
      private final Path[] rootDirs;
      private Path next = null;

      private PathIterator(FileSystem fs, String pathStr, Path[] rootDirs) throws IOException {
        this.fs = fs;
        this.pathStr = pathStr;
        this.rootDirs = rootDirs;
        advance();
      }

      @Override
      public boolean hasNext() {
        return next != null;
      }

      private void advance() throws IOException {
        while (i < rootDirs.length) {
          next = new Path(rootDirs[i++], pathStr);
          if (fs.exists(next)) {
            return;
          }
        }
        next = null;
      }

      @Override
      public Path next() {
        Path result = next;
        try {
          advance();
        } catch (IOException ie) {
          throw new RuntimeException("Can't check existance of " + next, ie);
        }
        return result;
      }

      @Override
      public void remove() {
        throw new UnsupportedOperationException("read only iterator");
      }

      @Override
      public Iterator<Path> iterator() {
        return this;
      }
    }

    /**
     * Get all of the paths that currently exist in the working directories.
     *
     * @param pathStr the path underneath the roots
     * @param conf the configuration to look up the roots in
     * @return all of the paths that exist under any of the roots
     * @throws IOException
     */
    synchronized Iterable<Path> getAllLocalPathsToRead(String pathStr, Configuration conf)
        throws IOException {
      confChanged(conf);
      if (pathStr.startsWith("/")) {
        pathStr = pathStr.substring(1);
      }
      return new PathIterator(localFS, pathStr, localDirsPath);
    }

    /**
     * We search through all the configured dirs for the file's existence and return true when we
     * find one
     */
    public synchronized boolean ifExists(String pathStr, Configuration conf) {
      try {
        int numDirs = localDirsPath.length;
        int numDirsSearched = 0;
        // remove the leading slash from the path (to make sure that the uri
        // resolution results in a valid path on the dir being checked)
        if (pathStr.startsWith("/")) {
          pathStr = pathStr.substring(1);
        }
        Path childPath = new Path(pathStr);
        while (numDirsSearched < numDirs) {
          Path file = new Path(localDirsPath[numDirsSearched], childPath);
          if (localFS.exists(file)) {
            return true;
          }
          numDirsSearched++;
        }
      } catch (IOException e) {
        // IGNORE and try again
      }
      return false;
    }
  }
/**
 * To generate automatically reports from list mode.
 *
 * <p>Uses JasperReports.
 *
 * @author Javier Paniza
 */
public class GenerateReportServlet extends HttpServlet {

  private static Log log = LogFactory.getLog(GenerateReportServlet.class);

  public static class TableModelDecorator implements TableModel {

    private TableModel original;
    private List metaProperties;
    private boolean withValidValues = false;
    private Locale locale;
    private boolean labelAsHeader = false;
    private HttpServletRequest request;
    private boolean format =
        false; // format or no the values. If format = true, all values to the report are String
    private Integer columnCountLimit;

    public TableModelDecorator(
        HttpServletRequest request,
        TableModel original,
        List metaProperties,
        Locale locale,
        boolean labelAsHeader,
        boolean format,
        Integer columnCountLimit)
        throws Exception {
      this.request = request;
      this.original = original;
      this.metaProperties = metaProperties;
      this.locale = locale;
      this.withValidValues = calculateWithValidValues();
      this.labelAsHeader = labelAsHeader;
      this.format = format;
      this.columnCountLimit = columnCountLimit;
    }

    private boolean calculateWithValidValues() {
      Iterator it = metaProperties.iterator();
      while (it.hasNext()) {
        MetaProperty m = (MetaProperty) it.next();
        if (m.hasValidValues()) return true;
      }
      return false;
    }

    private MetaProperty getMetaProperty(int i) {
      return (MetaProperty) metaProperties.get(i);
    }

    public int getRowCount() {
      return original.getRowCount();
    }

    public int getColumnCount() {
      return columnCountLimit == null ? original.getColumnCount() : columnCountLimit;
    }

    public String getColumnName(int c) {
      return labelAsHeader
          ? getMetaProperty(c).getLabel(locale)
          : Strings.change(getMetaProperty(c).getQualifiedName(), ".", "_");
    }

    public Class getColumnClass(int c) {
      return original.getColumnClass(c);
    }

    public boolean isCellEditable(int row, int column) {
      return original.isCellEditable(row, column);
    }

    public Object getValueAt(int row, int column) {
      if (isFormat()) return getValueWithWebEditorsFormat(row, column);
      else return getValueWithoutWebEditorsFormat(row, column);
    }

    private Object getValueWithoutWebEditorsFormat(int row, int column) {
      Object r = original.getValueAt(row, column);

      if (r instanceof Boolean) {
        if (((Boolean) r).booleanValue()) return XavaResources.getString(locale, "yes");
        return XavaResources.getString(locale, "no");
      }
      if (withValidValues) {
        MetaProperty p = getMetaProperty(column);
        if (p.hasValidValues()) {
          return p.getValidValueLabel(locale, original.getValueAt(row, column));
        }
      }

      if (r instanceof java.util.Date) {
        MetaProperty p =
            getMetaProperty(column); // In order to use the type declared by the developer
        // and not the one returned by JDBC or the JPA engine
        if (java.sql.Time.class.isAssignableFrom(p.getType())) {
          return DateFormat.getTimeInstance(DateFormat.SHORT, locale).format(r);
        }
        if (java.sql.Timestamp.class.isAssignableFrom(p.getType())) {
          DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
          return dateFormat.format(r);
        }
        return DateFormat.getDateInstance(DateFormat.SHORT, locale).format(r);
      }

      if (r instanceof BigDecimal) {
        return formatBigDecimal(r, locale);
      }

      return r;
    }

    private Object getValueWithWebEditorsFormat(int row, int column) {
      Object r = original.getValueAt(row, column);
      MetaProperty metaProperty = getMetaProperty(column);
      String result = WebEditors.format(this.request, metaProperty, r, null, "", true);
      if (isHtml(result)) { // this avoids that the report shows html content
        result = WebEditors.format(this.request, metaProperty, r, null, "", false);
      }
      return result;
    }

    public void setValueAt(Object value, int row, int column) {
      original.setValueAt(value, row, column);
    }

    public void addTableModelListener(TableModelListener l) {
      original.addTableModelListener(l);
    }

    public void removeTableModelListener(TableModelListener l) {
      original.removeTableModelListener(l);
    }

    private boolean isHtml(String value) {
      return value.matches("<.*>");
    }

    public boolean isFormat() {
      return format;
    }

    public void setFormat(boolean format) {
      this.format = format;
    }
  }

  protected void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    try {
      Locales.setCurrent(request);
      if (Users.getCurrent() == null) { // for a bug in websphere portal 5.1 with Domino LDAP
        Users.setCurrent((String) request.getSession().getAttribute("xava.user"));
      }
      request.getParameter("application"); // for a bug in websphere 5.1
      request.getParameter("module"); // for a bug in websphere 5.1
      Tab tab = (Tab) request.getSession().getAttribute("xava_reportTab");
      int[] selectedRowsNumber =
          (int[]) request.getSession().getAttribute("xava_selectedRowsReportTab");
      Map[] selectedKeys = (Map[]) request.getSession().getAttribute("xava_selectedKeysReportTab");
      int[] selectedRows = getSelectedRows(selectedRowsNumber, selectedKeys, tab);
      request.getSession().removeAttribute("xava_selectedRowsReportTab");
      Integer columnCountLimit =
          (Integer) request.getSession().getAttribute("xava_columnCountLimitReportTab");
      request.getSession().removeAttribute("xava_columnCountLimitReportTab");

      setDefaultSchema(request);
      String user = (String) request.getSession().getAttribute("xava_user");
      request.getSession().removeAttribute("xava_user");
      Users.setCurrent(user);
      String uri = request.getRequestURI();
      if (uri.endsWith(".pdf")) {
        InputStream is;
        JRDataSource ds;
        Map parameters = new HashMap();
        synchronized (tab) {
          tab.setRequest(request);
          parameters.put("Title", tab.getTitle());
          parameters.put("Organization", getOrganization());
          parameters.put("Date", getCurrentDate());
          for (String totalProperty : tab.getTotalPropertiesNames()) {
            parameters.put(totalProperty + "__TOTAL__", getTotal(request, tab, totalProperty));
          }
          TableModel tableModel = getTableModel(request, tab, selectedRows, false, true, null);
          tableModel.getValueAt(0, 0);
          if (tableModel.getRowCount() == 0) {
            generateNoRowsPage(response);
            return;
          }
          is = getReport(request, response, tab, tableModel, columnCountLimit);
          ds = new JRTableModelDataSource(tableModel);
        }
        JasperPrint jprint = JasperFillManager.fillReport(is, parameters, ds);
        response.setContentType("application/pdf");
        response.setHeader(
            "Content-Disposition", "inline; filename=\"" + getFileName(tab) + ".pdf\"");
        JasperExportManager.exportReportToPdfStream(jprint, response.getOutputStream());
      } else if (uri.endsWith(".csv")) {
        String csvEncoding = XavaPreferences.getInstance().getCSVEncoding();
        if (!Is.emptyString(csvEncoding)) {
          response.setCharacterEncoding(csvEncoding);
        }
        response.setContentType("text/x-csv");
        response.setHeader(
            "Content-Disposition", "inline; filename=\"" + getFileName(tab) + ".csv\"");
        synchronized (tab) {
          tab.setRequest(request);
          response
              .getWriter()
              .print(
                  TableModels.toCSV(
                      getTableModel(request, tab, selectedRows, true, false, columnCountLimit)));
        }
      } else {
        throw new ServletException(
            XavaResources.getString("report_type_not_supported", "", ".pdf .csv"));
      }
    } catch (Exception ex) {
      log.error(ex.getMessage(), ex);
      throw new ServletException(XavaResources.getString("report_error"));
    } finally {
      request.getSession().removeAttribute("xava_reportTab");
    }
  }

  private void generateNoRowsPage(HttpServletResponse response) throws Exception {
    response.setContentType("text/html");
    response.getWriter().println("<html><head><title>");
    response.getWriter().println(XavaResources.getString("no_rows_report_message_title"));
    response
        .getWriter()
        .println(
            "</title></head><body style='font-family:Tahoma,Arial,sans-serif;color:black;background-color:white;'>");
    response.getWriter().println("<h1 style='font-size:22px;'>");
    response.getWriter().println(XavaResources.getString("no_rows_report_message_title"));
    response.getWriter().println("</h1>");
    response.getWriter().println("<p style='font-size:16px;'>");
    response.getWriter().println(XavaResources.getString("no_rows_report_message_detail"));
    response.getWriter().println("</p></body></html>");
  }

  private String getCurrentDate() {
    return java.text.DateFormat.getDateInstance(DateFormat.MEDIUM, Locales.getCurrent())
        .format(new java.util.Date());
  }

  private String getFileName(Tab tab) {
    String now = new SimpleDateFormat("yyyyMMdd_HHmm").format(new Date());
    return tab.getTitle() + " " + now;
  }

  private Object getTotal(HttpServletRequest request, Tab tab, String totalProperty) {
    Object total = tab.getTotal(totalProperty);
    return WebEditors.format(
        request, tab.getMetaProperty(totalProperty), total, new Messages(), null, true);
  }

  private void setDefaultSchema(HttpServletRequest request) {
    String hibernateDefaultSchemaTab =
        (String) request.getSession().getAttribute("xava_hibernateDefaultSchemaTab");
    if (hibernateDefaultSchemaTab != null) {
      request.getSession().removeAttribute("xava_hibernateDefaultSchemaTab");
      XHibernate.setDefaultSchema(hibernateDefaultSchemaTab);
    }
    String jpaDefaultSchemaTab =
        (String) request.getSession().getAttribute("xava_jpaDefaultSchemaTab");
    if (jpaDefaultSchemaTab != null) {
      request.getSession().removeAttribute("xava_jpaDefaultSchemaTab");
      XPersistence.setDefaultSchema(jpaDefaultSchemaTab);
    }
  }

  protected String getOrganization() throws MissingResourceException, XavaException {
    return ReportParametersProviderFactory.getInstance().getOrganization();
  }

  private InputStream getReport(
      HttpServletRequest request,
      HttpServletResponse response,
      Tab tab,
      TableModel tableModel,
      Integer columnCountLimit)
      throws ServletException, IOException {
    StringBuffer suri = new StringBuffer();
    suri.append("/xava/jasperReport");
    suri.append("?language=");
    suri.append(Locales.getCurrent().getLanguage());
    suri.append("&widths=");
    suri.append(Arrays.toString(getWidths(tableModel)));
    if (columnCountLimit != null) {
      suri.append("&columnCountLimit=");
      suri.append(columnCountLimit);
    }
    response.setCharacterEncoding(XSystem.getEncoding());
    return Servlets.getURIAsStream(request, response, suri.toString());
  }

  private int[] getWidths(TableModel tableModel) {
    int[] widths = new int[tableModel.getColumnCount()];
    for (int r = 0;
        r < Math.min(tableModel.getRowCount(), 500);
        r++) { // 500 is not for performance, but for using only a sample of data with huge table
      for (int c = 0; c < tableModel.getColumnCount(); c++) {
        Object o = tableModel.getValueAt(r, c);
        if (o instanceof String) {
          String s = ((String) o).trim();
          if (s.length() > widths[c]) widths[c] = s.length();
        }
      }
    }
    return widths;
  }

  private TableModel getTableModel(
      HttpServletRequest request,
      Tab tab,
      int[] selectedRows,
      boolean labelAsHeader,
      boolean format,
      Integer columnCountLimit)
      throws Exception {
    TableModel data = null;
    if (selectedRows != null && selectedRows.length > 0) {
      data = new SelectedRowsXTableModel(tab.getTableModel(), selectedRows);
    } else {
      data = tab.getAllDataTableModel();
    }
    return new TableModelDecorator(
        request,
        data,
        tab.getMetaProperties(),
        Locales.getCurrent(),
        labelAsHeader,
        format,
        columnCountLimit);
  }

  private static Object formatBigDecimal(Object number, Locale locale) {
    NumberFormat nf = NumberFormat.getNumberInstance(locale);
    nf.setMinimumFractionDigits(2);
    return nf.format(number);
  }

  private int[] getSelectedRows(int[] selectedRowsNumber, Map[] selectedRowsKeys, Tab tab) {
    if (selectedRowsKeys == null || selectedRowsKeys.length == 0) return new int[0];
    // selectedRowsNumber is the most performant so we use it when possible
    else if (selectedRowsNumber.length == selectedRowsKeys.length) return selectedRowsNumber;
    else {
      // find the rows from the selectedKeys

      // This has a poor performance, but it covers the case when the selected
      // rows are not loaded for the tab, something that can occurs if the user
      // select rows and afterwards reorder the list.
      try {
        int[] s = new int[selectedRowsKeys.length];
        List selectedKeys = Arrays.asList(selectedRowsKeys);
        int end = tab.getTableModel().getTotalSize();
        int x = 0;
        for (int i = 0; i < end; i++) {
          Map key = (Map) tab.getTableModel().getObjectAt(i);
          if (selectedKeys.contains(key)) {
            s[x] = i;
            x++;
          }
        }
        return s;
      } catch (Exception ex) {
        log.warn(XavaResources.getString("fails_selected"), ex);
        throw new XavaException("fails_selected");
      }
    }
  }
}
Exemple #9
0
/**
 * A simple RPC mechanism.
 *
 * <p>A <i>protocol</i> is a Java interface. All parameters and return types must be one of:
 *
 * <ul>
 *   <li>a primitive type, <code>boolean</code>, <code>byte</code>, <code>char</code>, <code>short
 *       </code>, <code>int</code>, <code>long</code>, <code>float</code>, <code>double</code>, or
 *       <code>void</code>; or
 *   <li>a {@link String}; or
 *   <li>a {@link Writable}; or
 *   <li>an array of the above types
 * </ul>
 *
 * All methods in the protocol should throw only IOException. No field data of the protocol instance
 * is transmitted.
 */
public class RPC {
  private static final Log LOG = LogFactory.getLog(RPC.class);

  private RPC() {} // no public ctor

  /** A method invocation, including the method name and its parameters. */
  private static class Invocation implements Writable, Configurable {
    private String methodName;
    private Class[] parameterClasses;
    private Object[] parameters;
    private Configuration conf;

    public Invocation() {}

    public Invocation(Method method, Object[] parameters) {
      this.methodName = method.getName();
      this.parameterClasses = method.getParameterTypes();
      this.parameters = parameters;
    }

    /** The name of the method invoked. */
    public String getMethodName() {
      return methodName;
    }

    /** The parameter classes. */
    public Class[] getParameterClasses() {
      return parameterClasses;
    }

    /** The parameter instances. */
    public Object[] getParameters() {
      return parameters;
    }

    public void readFields(DataInput in) throws IOException {
      methodName = UTF8.readString(in);
      parameters = new Object[in.readInt()];
      parameterClasses = new Class[parameters.length];
      ObjectWritable objectWritable = new ObjectWritable();
      for (int i = 0; i < parameters.length; i++) {
        parameters[i] = ObjectWritable.readObject(in, objectWritable, this.conf);
        parameterClasses[i] = objectWritable.getDeclaredClass();
      }
    }

    public void write(DataOutput out) throws IOException {
      ObjectWritable.writeStringCached(out, methodName);
      out.writeInt(parameterClasses.length);
      for (int i = 0; i < parameterClasses.length; i++) {
        ObjectWritable.writeObject(out, parameters[i], parameterClasses[i], conf);
      }
    }

    public String toString() {
      StringBuffer buffer = new StringBuffer();
      buffer.append(methodName);
      buffer.append("(");
      for (int i = 0; i < parameters.length; i++) {
        if (i != 0) buffer.append(", ");
        buffer.append(parameters[i]);
      }
      buffer.append(")");
      return buffer.toString();
    }

    public void setConf(Configuration conf) {
      this.conf = conf;
    }

    public Configuration getConf() {
      return this.conf;
    }
  }

  /* Cache a client using its socket factory as the hash key */
  private static class ClientCache {
    private Map<SocketFactory, Client> clients = new HashMap<SocketFactory, Client>();

    /**
     * Construct & cache an IPC client with the user-provided SocketFactory if no cached client
     * exists.
     *
     * @param conf Configuration
     * @return an IPC client
     */
    private synchronized Client getClient(Configuration conf, SocketFactory factory) {
      // Construct & cache client.  The configuration is only used for timeout,
      // and Clients have connection pools.  So we can either (a) lose some
      // connection pooling and leak sockets, or (b) use the same timeout for all
      // configurations.  Since the IPC is usually intended globally, not
      // per-job, we choose (a).
      Client client = clients.get(factory);
      if (client == null) {
        client = new Client(ObjectWritable.class, conf, factory);
        clients.put(factory, client);
      } else {
        client.incCount();
      }
      return client;
    }

    /**
     * Construct & cache an IPC client with the default SocketFactory if no cached client exists.
     *
     * @param conf Configuration
     * @return an IPC client
     */
    private synchronized Client getClient(Configuration conf) {
      return getClient(conf, SocketFactory.getDefault());
    }

    /**
     * Stop a RPC client connection A RPC client is closed only when its reference count becomes
     * zero.
     */
    private void stopClient(Client client) {
      synchronized (this) {
        client.decCount();
        if (client.isZeroReference()) {
          clients.remove(client.getSocketFactory());
        }
      }
      if (client.isZeroReference()) {
        client.stop();
      }
    }
  }

  private static ClientCache CLIENTS = new ClientCache();

  private static class Invoker implements InvocationHandler {
    private InetSocketAddress address;
    private UserGroupInformation ticket;
    private Client client;
    private boolean isClosed = false;
    private boolean needCheckDnsUpdate = false;
    private long timeLastDnsCheck = 0;
    private final long MIN_DNS_CHECK_INTERVAL_MSEC = 120 * 1000;
    private final int rpcTimeout;
    private final Class<?> protocol;

    public Invoker(
        InetSocketAddress address,
        UserGroupInformation ticket,
        Configuration conf,
        SocketFactory factory,
        int rpcTimeout,
        Class<?> protocol) {
      this.address = address;
      this.ticket = ticket;
      this.client = CLIENTS.getClient(conf, factory);
      this.rpcTimeout = rpcTimeout;
      this.protocol = protocol;
    }

    private synchronized InetSocketAddress getAddress() {
      if (needCheckDnsUpdate
          && address != null
          && address.getHostName() != null
          && System.currentTimeMillis() - this.timeLastDnsCheck > MIN_DNS_CHECK_INTERVAL_MSEC) {
        try {
          InetSocketAddress newAddr = NetUtils.resolveAddress(address);
          if (newAddr != null) {
            LOG.info("DNS change: " + newAddr);
            address = newAddr;
          }
        } finally {
          this.timeLastDnsCheck = System.currentTimeMillis();
        }
      }
      needCheckDnsUpdate = false;
      return address;
    }

    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
      final boolean logDebug = LOG.isDebugEnabled();
      long startTime = 0;
      if (logDebug) {
        startTime = System.currentTimeMillis();
      }

      ObjectWritable value = null;
      try {
        value =
            (ObjectWritable)
                client.call(
                    new Invocation(method, args), getAddress(), protocol, ticket, rpcTimeout);
      } catch (RemoteException re) {
        throw re;
      } catch (ConnectException ce) {
        needCheckDnsUpdate = true;
        throw ce;
      } catch (NoRouteToHostException nrhe) {
        needCheckDnsUpdate = true;
        throw nrhe;
      } catch (PortUnreachableException pue) {
        needCheckDnsUpdate = true;
        throw pue;
      } catch (UnknownHostException uhe) {
        needCheckDnsUpdate = true;
        throw uhe;
      }
      if (logDebug) {
        long callTime = System.currentTimeMillis() - startTime;
        LOG.debug("Call: " + method.getName() + " " + callTime);
      }
      return value.get();
    }

    /* close the IPC client that's responsible for this invoker's RPCs */
    private synchronized void close() {
      if (!isClosed) {
        isClosed = true;
        CLIENTS.stopClient(client);
      }
    }
  }

  /**
   * An exception indicating that the client and server have incompatible versions. They are not
   * able to communicate with each other.
   */
  public static class VersionIncompatible extends IOException {
    private String interfaceName;
    private long clientVersion;
    private long serverVersion;

    /**
     * Create a version incompatible exception
     *
     * @param interfaceName the name of the protocol mismatch
     * @param clientVersion the client's version of the protocol
     * @param serverVersion the server's version of the protocol
     */
    public VersionIncompatible(String interfaceName, long clientVersion, long serverVersion) {
      super(
          "Protocol "
              + interfaceName
              + " version mismatch. (client = "
              + clientVersion
              + ", server = "
              + serverVersion
              + ")");
      this.interfaceName = interfaceName;
      this.clientVersion = clientVersion;
      this.serverVersion = serverVersion;
    }

    /**
     * Get the interface name
     *
     * @return the java class name (eg. org.apache.hadoop.mapred.InterTrackerProtocol)
     */
    public String getInterfaceName() {
      return interfaceName;
    }

    /** Get the client's preferred version */
    public long getClientVersion() {
      return clientVersion;
    }

    /** Get the server's agreed to version. */
    public long getServerVersion() {
      return serverVersion;
    }
  }

  /**
   * A version mismatch for the RPC protocol.
   *
   * <p>The client & server have different versions. But the server is not able to determine if they
   * are compatible mostly because the client is newer than the server. So the proxy is created and
   * the application client is left to decide if the client & server are compatible or not.
   */
  public static class VersionMismatch extends VersionIncompatible {
    private static final long serialVersionUID = 1L;
    private final VersionedProtocol proxy;

    /**
     * Create a version mismatch exception
     *
     * @param interfaceName the name of the protocol mismatch
     * @param clientVersion the client's version of the protocol
     * @param serverVersion the server's version of the protocol
     */
    public VersionMismatch(String interfaceName, long clientVersion, long serverVersion) {
      super(interfaceName, clientVersion, serverVersion);
      proxy = null;
    }

    /**
     * Create a version mismatch exception
     *
     * @param interfaceName the name of the protocol mismatch
     * @param clientVersion the client's version of the protocol
     * @param serverVersion the server's version of the protocol
     * @param proxy the proxy
     */
    public VersionMismatch(
        String interfaceName, long clientVersion, long serverVersion, VersionedProtocol proxy) {
      super(interfaceName, clientVersion, serverVersion);
      this.proxy = proxy;
    }

    /** Return the proxy */
    public VersionedProtocol getProxy() {
      return proxy;
    }
  }

  public static <T extends VersionedProtocol> T waitForProxy(
      Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf)
      throws IOException {
    return waitForProtocolProxy(protocol, clientVersion, addr, conf).getProxy();
  }

  public static <T extends VersionedProtocol> ProtocolProxy<T> waitForProtocolProxy(
      Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf)
      throws IOException {
    return waitForProtocolProxy(protocol, clientVersion, addr, conf, Long.MAX_VALUE);
  }

  public static <T extends VersionedProtocol> T waitForProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      long timeout)
      throws IOException {
    return waitForProtocolProxy(protocol, clientVersion, addr, conf, timeout).getProxy();
  }

  public static <T extends VersionedProtocol> ProtocolProxy<T> waitForProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      long timeout)
      throws IOException {
    return waitForProtocolProxy(protocol, clientVersion, addr, conf, timeout, 0);
  }

  /**
   * Get a proxy connection to a remote server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @param connTimeout time in milliseconds before giving up
   * @param rpcTimeout timeout for each RPC
   * @return the proxy
   * @throws IOException if the far end through a RemoteException
   */
  public static <T extends VersionedProtocol> T waitForProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      long connTimeout,
      int rpcTimeout)
      throws IOException {
    return waitForProtocolProxy(protocol, clientVersion, addr, conf, connTimeout, rpcTimeout)
        .getProxy();
  }

  /**
   * Get a proxy connection to a remote server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @param rpcTimeout timeout for each RPC
   * @param timeout time in milliseconds before giving up
   * @return the proxy
   * @throws IOException if the far end through a RemoteException
   */
  static <T extends VersionedProtocol> ProtocolProxy<T> waitForProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      long timeout,
      int rpcTimeout)
      throws IOException {
    long startTime = System.currentTimeMillis();
    UserGroupInformation ugi = null;
    try {
      ugi = UserGroupInformation.login(conf);
    } catch (LoginException le) {
      throw new RuntimeException("Couldn't login!");
    }
    IOException ioe;
    while (true) {
      try {
        return getProtocolProxy(
            protocol,
            clientVersion,
            addr,
            ugi,
            conf,
            NetUtils.getDefaultSocketFactory(conf),
            rpcTimeout);
      } catch (ConnectException se) { // namenode has not been started
        LOG.info("Server at " + addr + " not available yet, Zzzzz...");
        ioe = se;
      } catch (SocketTimeoutException te) { // namenode is busy
        LOG.info("Problem connecting to server: " + addr);
        ioe = te;
      }
      // check if timed out
      if (System.currentTimeMillis() - timeout >= startTime) {
        throw ioe;
      }

      // wait for retry
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
        // IGNORE
      }
    }
  }

  /**
   * Construct a client-side proxy object that implements the named protocol, talking to a server at
   * the named address.
   */
  public static <T extends VersionedProtocol> T getProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      SocketFactory factory)
      throws IOException {
    return getProtocolProxy(protocol, clientVersion, addr, conf, factory).getProxy();
  }

  /**
   * Construct a client-side protocol proxy that contains a set of server methods and a proxy object
   * implementing the named protocol, talking to a server at the named address.
   */
  public static <T extends VersionedProtocol> ProtocolProxy<T> getProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      SocketFactory factory)
      throws IOException {
    UserGroupInformation ugi = null;
    try {
      ugi = UserGroupInformation.login(conf);
    } catch (LoginException le) {
      throw new RuntimeException("Couldn't login!");
    }
    return getProtocolProxy(protocol, clientVersion, addr, ugi, conf, factory);
  }

  /**
   * Construct a client-side proxy object that implements the named protocol, talking to a server at
   * the named address.
   */
  public static <T extends VersionedProtocol> T getProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      UserGroupInformation ticket,
      Configuration conf,
      SocketFactory factory)
      throws IOException {
    return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory).getProxy();
  }

  /**
   * Construct a client-side protocol proxy that contains a set of server methods and a proxy object
   * implementing the named protocol, talking to a server at the named address.
   */
  public static <T extends VersionedProtocol> ProtocolProxy<T> getProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      UserGroupInformation ticket,
      Configuration conf,
      SocketFactory factory)
      throws IOException {
    return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory, 0);
  }

  /**
   * Construct a client-side proxy that implements the named protocol, talking to a server at the
   * named address.
   *
   * @param protocol protocol
   * @param clientVersion client's version
   * @param addr server address
   * @param ticket security ticket
   * @param conf configuration
   * @param factory socket factory
   * @param rpcTimeout max time for each rpc; 0 means no timeout
   * @return the proxy
   * @throws IOException if any error occurs
   */
  public static <T extends VersionedProtocol> T getProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      UserGroupInformation ticket,
      Configuration conf,
      SocketFactory factory,
      int rpcTimeout)
      throws IOException {
    return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout)
        .getProxy();
  }

  /**
   * Construct a client-side proxy that implements the named protocol, talking to a server at the
   * named address.
   *
   * @param protocol protocol
   * @param clientVersion client's version
   * @param addr server address
   * @param ticket security ticket
   * @param conf configuration
   * @param factory socket factory
   * @param rpcTimeout max time for each rpc; 0 means no timeout
   * @return the proxy
   * @throws IOException if any error occurs
   */
  @SuppressWarnings("unchecked")
  public static <T extends VersionedProtocol> ProtocolProxy<T> getProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      UserGroupInformation ticket,
      Configuration conf,
      SocketFactory factory,
      int rpcTimeout)
      throws IOException {
    T proxy =
        (T)
            Proxy.newProxyInstance(
                protocol.getClassLoader(),
                new Class[] {protocol},
                new Invoker(addr, ticket, conf, factory, rpcTimeout, protocol));
    String protocolName = protocol.getName();

    try {
      ProtocolSignature serverInfo =
          proxy.getProtocolSignature(
              protocolName, clientVersion, ProtocolSignature.getFingerprint(protocol.getMethods()));
      return new ProtocolProxy<T>(protocol, proxy, serverInfo.getMethods());
    } catch (RemoteException re) {
      IOException ioe = re.unwrapRemoteException(IOException.class);
      if (ioe.getMessage()
          .startsWith(IOException.class.getName() + ": " + NoSuchMethodException.class.getName())) {
        // Method getProtocolSignature not supported
        long serverVersion = proxy.getProtocolVersion(protocol.getName(), clientVersion);
        if (serverVersion == clientVersion) {
          return new ProtocolProxy<T>(protocol, proxy, null);
        }
        throw new VersionMismatch(protocolName, clientVersion, serverVersion, proxy);
      }
      throw re;
    }
  }

  /**
   * Construct a client-side proxy object with the default SocketFactory
   *
   * @param protocol
   * @param clientVersion
   * @param addr
   * @param conf
   * @return a proxy instance
   * @throws IOException
   */
  public static <T extends VersionedProtocol> T getProxy(
      Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf)
      throws IOException {
    return getProtocolProxy(protocol, clientVersion, addr, conf).getProxy();
  }

  /**
   * Construct a client-side proxy object with the default SocketFactory
   *
   * @param protocol
   * @param clientVersion
   * @param addr
   * @param conf
   * @return a proxy instance
   * @throws IOException
   */
  public static <T extends VersionedProtocol> ProtocolProxy<T> getProtocolProxy(
      Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf)
      throws IOException {

    return getProtocolProxy(
        protocol, clientVersion, addr, conf, NetUtils.getDefaultSocketFactory(conf));
  }

  /**
   * Stop this proxy and release its invoker's resource
   *
   * @param <T>
   * @param proxy the proxy to be stopped
   */
  public static <T extends VersionedProtocol> void stopProxy(T proxy) {
    if (proxy != null) {
      ((Invoker) Proxy.getInvocationHandler(proxy)).close();
    }
  }

  /**
   * Expert: Make multiple, parallel calls to a set of servers.
   *
   * @deprecated Use {@link #call(Method, Object[][], InetSocketAddress[], UserGroupInformation,
   *     Configuration)} instead
   */
  public static Object[] call(
      Method method, Object[][] params, InetSocketAddress[] addrs, Configuration conf)
      throws IOException {
    return call(method, params, addrs, null, conf);
  }

  /** Expert: Make multiple, parallel calls to a set of servers. */
  public static Object[] call(
      Method method,
      Object[][] params,
      InetSocketAddress[] addrs,
      UserGroupInformation ticket,
      Configuration conf)
      throws IOException {

    Invocation[] invocations = new Invocation[params.length];
    for (int i = 0; i < params.length; i++) invocations[i] = new Invocation(method, params[i]);
    Client client = CLIENTS.getClient(conf);
    try {
      Writable[] wrappedValues =
          client.call(invocations, addrs, method.getDeclaringClass(), ticket);

      if (method.getReturnType() == Void.TYPE) {
        return null;
      }

      Object[] values = (Object[]) Array.newInstance(method.getReturnType(), wrappedValues.length);
      for (int i = 0; i < values.length; i++)
        if (wrappedValues[i] != null) values[i] = ((ObjectWritable) wrappedValues[i]).get();

      return values;
    } finally {
      CLIENTS.stopClient(client);
    }
  }

  static Client getClient(Configuration conf, SocketFactory socketFactory) {
    return CLIENTS.getClient(conf, socketFactory);
  }

  /** Construct a server for a protocol implementation instance listening on a port and address. */
  public static Server getServer(
      final Object instance, final String bindAddress, final int port, Configuration conf)
      throws IOException {
    return getServer(instance, bindAddress, port, 1, false, conf);
  }

  /** Construct a server for a protocol implementation instance listening on a port and address. */
  public static Server getServer(
      final Object instance,
      final String bindAddress,
      final int port,
      final int numHandlers,
      final boolean verbose,
      Configuration conf)
      throws IOException {
    return getServer(instance, bindAddress, port, numHandlers, verbose, conf, true);
  }

  /** Construct a server for a protocol implementation instance listening on a port and address. */
  public static Server getServer(
      final Object instance,
      final String bindAddress,
      final int port,
      final int numHandlers,
      final boolean verbose,
      Configuration conf,
      boolean supportOldJobConf)
      throws IOException {
    return new Server(instance, conf, bindAddress, port, numHandlers, verbose, supportOldJobConf);
  }

  /** An RPC Server. */
  public static class Server extends org.apache.hadoop.ipc.Server {
    private Object instance;
    private boolean verbose;
    private boolean authorize = false;

    /**
     * Construct an RPC server.
     *
     * @param instance the instance whose methods will be called
     * @param conf the configuration to use
     * @param bindAddress the address to bind on to listen for connection
     * @param port the port to listen for connections on
     */
    public Server(Object instance, Configuration conf, String bindAddress, int port)
        throws IOException {
      this(instance, conf, bindAddress, port, 1, false);
    }

    private static String classNameBase(String className) {
      String[] names = className.split("\\.", -1);
      if (names == null || names.length == 0) {
        return className;
      }
      return names[names.length - 1];
    }

    /**
     * Construct an RPC server.
     *
     * @param instance the instance whose methods will be called
     * @param conf the configuration to use
     * @param bindAddress the address to bind on to listen for connection
     * @param port the port to listen for connections on
     * @param numHandlers the number of method handler threads to run
     * @param verbose whether each call should be logged
     */
    public Server(
        Object instance,
        Configuration conf,
        String bindAddress,
        int port,
        int numHandlers,
        boolean verbose)
        throws IOException {
      this(instance, conf, bindAddress, port, numHandlers, verbose, true);
    }

    /**
     * Construct an RPC server.
     *
     * @param instance the instance whose methods will be called
     * @param conf the configuration to use
     * @param bindAddress the address to bind on to listen for connection
     * @param port the port to listen for connections on
     * @param numHandlers the number of method handler threads to run
     * @param verbose whether each call should be logged
     * @param supportOldJobConf supports server to deserialize old job conf
     */
    public Server(
        Object instance,
        Configuration conf,
        String bindAddress,
        int port,
        int numHandlers,
        boolean verbose,
        boolean supportOldJobConf)
        throws IOException {
      super(
          bindAddress,
          port,
          Invocation.class,
          numHandlers,
          conf,
          classNameBase(instance.getClass().getName()),
          supportOldJobConf);
      this.instance = instance;
      this.verbose = verbose;
      this.authorize =
          conf.getBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, false);
    }

    public Writable call(Class<?> protocol, Writable param, long receivedTime) throws IOException {
      try {
        Invocation call = (Invocation) param;
        if (verbose) log("Call: " + call);

        Method method = protocol.getMethod(call.getMethodName(), call.getParameterClasses());
        method.setAccessible(true);

        int qTime = (int) (System.currentTimeMillis() - receivedTime);
        long startNanoTime = System.nanoTime();
        Object value = method.invoke(instance, call.getParameters());
        long processingMicroTime = (System.nanoTime() - startNanoTime) / 1000;
        if (LOG.isDebugEnabled()) {
          LOG.debug(
              "Served: "
                  + call.getMethodName()
                  + " queueTime (millisec)= "
                  + qTime
                  + " procesingTime (microsec)= "
                  + processingMicroTime);
        }
        rpcMetrics.rpcQueueTime.inc(qTime);
        rpcMetrics.rpcProcessingTime.inc(processingMicroTime);

        MetricsTimeVaryingRate m =
            (MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName());
        if (m == null) {
          try {
            m = new MetricsTimeVaryingRate(call.getMethodName(), rpcMetrics.registry);
          } catch (IllegalArgumentException iae) {
            // the metrics has been registered; re-fetch the handle
            LOG.debug("Error register " + call.getMethodName(), iae);
            m = (MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName());
          }
        }
        // record call time in microseconds
        m.inc(processingMicroTime);

        if (verbose) log("Return: " + value);

        return new ObjectWritable(method.getReturnType(), value);

      } catch (InvocationTargetException e) {
        Throwable target = e.getTargetException();
        if (target instanceof IOException) {
          throw (IOException) target;
        } else {
          IOException ioe = new IOException(target.toString());
          ioe.setStackTrace(target.getStackTrace());
          throw ioe;
        }
      } catch (Throwable e) {
        if (!(e instanceof IOException)) {
          LOG.error("Unexpected throwable object ", e);
        }
        IOException ioe = new IOException(e.toString());
        ioe.setStackTrace(e.getStackTrace());
        throw ioe;
      }
    }

    @Override
    public void authorize(Subject user, ConnectionHeader connection) throws AuthorizationException {
      if (authorize) {
        Class<?> protocol = null;
        try {
          protocol = getProtocolClass(connection.getProtocol(), getConf());
        } catch (ClassNotFoundException cfne) {
          throw new AuthorizationException("Unknown protocol: " + connection.getProtocol());
        }
        ServiceAuthorizationManager.authorize(user, protocol);
      }
    }
  }

  private static void log(String value) {
    if (value != null && value.length() > 55) value = value.substring(0, 55) + "...";
    LOG.info(value);
  }
}
/** An RpcEngine implementation for Writable data. */
@InterfaceStability.Evolving
public class WritableRpcEngine implements RpcEngine {
  private static final Log LOG = LogFactory.getLog(RPC.class);

  // writableRpcVersion should be updated if there is a change
  // in format of the rpc messages.

  // 2L - added declared class to Invocation
  public static final long writableRpcVersion = 2L;

  /** Whether or not this class has been initialized. */
  private static boolean isInitialized = false;

  static {
    ensureInitialized();
  }

  /** Initialize this class if it isn't already. */
  public static synchronized void ensureInitialized() {
    if (!isInitialized) {
      initialize();
    }
  }

  /** Register the rpcRequest deserializer for WritableRpcEngine */
  private static synchronized void initialize() {
    org.apache.hadoop.ipc.Server.registerProtocolEngine(
        RPC.RpcKind.RPC_WRITABLE, Invocation.class, new Server.WritableRpcInvoker());
    isInitialized = true;
  }

  /** A method invocation, including the method name and its parameters. */
  private static class Invocation implements Writable, Configurable {
    private String methodName;
    private Class<?>[] parameterClasses;
    private Object[] parameters;
    private Configuration conf;
    private long clientVersion;
    private int clientMethodsHash;
    private String declaringClassProtocolName;

    // This could be different from static writableRpcVersion when received
    // at server, if client is using a different version.
    private long rpcVersion;

    @SuppressWarnings("unused") // called when deserializing an invocation
    public Invocation() {}

    public Invocation(Method method, Object[] parameters) {
      this.methodName = method.getName();
      this.parameterClasses = method.getParameterTypes();
      this.parameters = parameters;
      rpcVersion = writableRpcVersion;
      if (method.getDeclaringClass().equals(VersionedProtocol.class)) {
        // VersionedProtocol is exempted from version check.
        clientVersion = 0;
        clientMethodsHash = 0;
      } else {
        this.clientVersion = RPC.getProtocolVersion(method.getDeclaringClass());
        this.clientMethodsHash =
            ProtocolSignature.getFingerprint(method.getDeclaringClass().getMethods());
      }
      this.declaringClassProtocolName = RPC.getProtocolName(method.getDeclaringClass());
    }

    /** The name of the method invoked. */
    public String getMethodName() {
      return methodName;
    }

    /** The parameter classes. */
    public Class<?>[] getParameterClasses() {
      return parameterClasses;
    }

    /** The parameter instances. */
    public Object[] getParameters() {
      return parameters;
    }

    private long getProtocolVersion() {
      return clientVersion;
    }

    @SuppressWarnings("unused")
    private int getClientMethodsHash() {
      return clientMethodsHash;
    }

    /**
     * Returns the rpc version used by the client.
     *
     * @return rpcVersion
     */
    public long getRpcVersion() {
      return rpcVersion;
    }

    @SuppressWarnings("deprecation")
    public void readFields(DataInput in) throws IOException {
      rpcVersion = in.readLong();
      declaringClassProtocolName = UTF8.readString(in);
      methodName = UTF8.readString(in);
      clientVersion = in.readLong();
      clientMethodsHash = in.readInt();
      parameters = new Object[in.readInt()];
      parameterClasses = new Class[parameters.length];
      ObjectWritable objectWritable = new ObjectWritable();
      for (int i = 0; i < parameters.length; i++) {
        parameters[i] = ObjectWritable.readObject(in, objectWritable, this.conf);
        parameterClasses[i] = objectWritable.getDeclaredClass();
      }
    }

    @SuppressWarnings("deprecation")
    public void write(DataOutput out) throws IOException {
      out.writeLong(rpcVersion);
      UTF8.writeString(out, declaringClassProtocolName);
      UTF8.writeString(out, methodName);
      out.writeLong(clientVersion);
      out.writeInt(clientMethodsHash);
      out.writeInt(parameterClasses.length);
      for (int i = 0; i < parameterClasses.length; i++) {
        ObjectWritable.writeObject(out, parameters[i], parameterClasses[i], conf, true);
      }
    }

    public String toString() {
      StringBuilder buffer = new StringBuilder();
      buffer.append(methodName);
      buffer.append("(");
      for (int i = 0; i < parameters.length; i++) {
        if (i != 0) buffer.append(", ");
        buffer.append(parameters[i]);
      }
      buffer.append(")");
      buffer.append(", rpc version=" + rpcVersion);
      buffer.append(", client version=" + clientVersion);
      buffer.append(", methodsFingerPrint=" + clientMethodsHash);
      return buffer.toString();
    }

    public void setConf(Configuration conf) {
      this.conf = conf;
    }

    public Configuration getConf() {
      return this.conf;
    }
  }

  private static ClientCache CLIENTS = new ClientCache();

  private static class Invoker implements RpcInvocationHandler {
    private Client.ConnectionId remoteId;
    private Client client;
    private boolean isClosed = false;

    public Invoker(
        Class<?> protocol,
        InetSocketAddress address,
        UserGroupInformation ticket,
        Configuration conf,
        SocketFactory factory,
        int rpcTimeout)
        throws IOException {
      this.remoteId =
          Client.ConnectionId.getConnectionId(address, protocol, ticket, rpcTimeout, conf);
      this.client = CLIENTS.getClient(conf, factory);
    }

    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
      long startTime = 0;
      if (LOG.isDebugEnabled()) {
        startTime = Time.now();
      }

      ObjectWritable value =
          (ObjectWritable)
              client.call(RPC.RpcKind.RPC_WRITABLE, new Invocation(method, args), remoteId);
      if (LOG.isDebugEnabled()) {
        long callTime = Time.now() - startTime;
        LOG.debug("Call: " + method.getName() + " " + callTime);
      }
      return value.get();
    }

    /* close the IPC client that's responsible for this invoker's RPCs */
    public synchronized void close() {
      if (!isClosed) {
        isClosed = true;
        CLIENTS.stopClient(client);
      }
    }

    @Override
    public ConnectionId getConnectionId() {
      return remoteId;
    }
  }

  // for unit testing only
  @InterfaceAudience.Private
  @InterfaceStability.Unstable
  static Client getClient(Configuration conf) {
    return CLIENTS.getClient(conf);
  }

  /**
   * Construct a client-side proxy object that implements the named protocol, talking to a server at
   * the named address.
   *
   * @param <T>
   */
  @Override
  @SuppressWarnings("unchecked")
  public <T> ProtocolProxy<T> getProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      UserGroupInformation ticket,
      Configuration conf,
      SocketFactory factory,
      int rpcTimeout,
      RetryPolicy connectionRetryPolicy)
      throws IOException {

    if (connectionRetryPolicy != null) {
      throw new UnsupportedOperationException(
          "Not supported: connectionRetryPolicy=" + connectionRetryPolicy);
    }

    T proxy =
        (T)
            Proxy.newProxyInstance(
                protocol.getClassLoader(),
                new Class[] {protocol},
                new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
    return new ProtocolProxy<T>(protocol, proxy, true);
  }

  /* Construct a server for a protocol implementation instance listening on a
   * port and address. */
  @Override
  public RPC.Server getServer(
      Class<?> protocolClass,
      Object protocolImpl,
      String bindAddress,
      int port,
      int numHandlers,
      int numReaders,
      int queueSizePerHandler,
      boolean verbose,
      Configuration conf,
      SecretManager<? extends TokenIdentifier> secretManager,
      String portRangeConfig)
      throws IOException {
    return new Server(
        protocolClass,
        protocolImpl,
        conf,
        bindAddress,
        port,
        numHandlers,
        numReaders,
        queueSizePerHandler,
        verbose,
        secretManager,
        portRangeConfig);
  }

  /** An RPC Server. */
  public static class Server extends RPC.Server {
    /**
     * Construct an RPC server.
     *
     * @param instance the instance whose methods will be called
     * @param conf the configuration to use
     * @param bindAddress the address to bind on to listen for connection
     * @param port the port to listen for connections on
     * @deprecated Use #Server(Class, Object, Configuration, String, int)
     */
    @Deprecated
    public Server(Object instance, Configuration conf, String bindAddress, int port)
        throws IOException {
      this(null, instance, conf, bindAddress, port);
    }

    /**
     * Construct an RPC server.
     *
     * @param protocolClass class
     * @param protocolImpl the instance whose methods will be called
     * @param conf the configuration to use
     * @param bindAddress the address to bind on to listen for connection
     * @param port the port to listen for connections on
     */
    public Server(
        Class<?> protocolClass,
        Object protocolImpl,
        Configuration conf,
        String bindAddress,
        int port)
        throws IOException {
      this(protocolClass, protocolImpl, conf, bindAddress, port, 1, -1, -1, false, null, null);
    }

    /**
     * Construct an RPC server.
     *
     * @param protocolImpl the instance whose methods will be called
     * @param conf the configuration to use
     * @param bindAddress the address to bind on to listen for connection
     * @param port the port to listen for connections on
     * @param numHandlers the number of method handler threads to run
     * @param verbose whether each call should be logged
     * @deprecated use Server#Server(Class, Object, Configuration, String, int, int, int, int,
     *     boolean, SecretManager)
     */
    @Deprecated
    public Server(
        Object protocolImpl,
        Configuration conf,
        String bindAddress,
        int port,
        int numHandlers,
        int numReaders,
        int queueSizePerHandler,
        boolean verbose,
        SecretManager<? extends TokenIdentifier> secretManager)
        throws IOException {
      this(
          null,
          protocolImpl,
          conf,
          bindAddress,
          port,
          numHandlers,
          numReaders,
          queueSizePerHandler,
          verbose,
          secretManager,
          null);
    }

    /**
     * Construct an RPC server.
     *
     * @param protocolClass - the protocol being registered can be null for compatibility with old
     *     usage (see below for details)
     * @param protocolImpl the protocol impl that will be called
     * @param conf the configuration to use
     * @param bindAddress the address to bind on to listen for connection
     * @param port the port to listen for connections on
     * @param numHandlers the number of method handler threads to run
     * @param verbose whether each call should be logged
     */
    public Server(
        Class<?> protocolClass,
        Object protocolImpl,
        Configuration conf,
        String bindAddress,
        int port,
        int numHandlers,
        int numReaders,
        int queueSizePerHandler,
        boolean verbose,
        SecretManager<? extends TokenIdentifier> secretManager,
        String portRangeConfig)
        throws IOException {
      super(
          bindAddress,
          port,
          null,
          numHandlers,
          numReaders,
          queueSizePerHandler,
          conf,
          classNameBase(protocolImpl.getClass().getName()),
          secretManager,
          portRangeConfig);

      this.verbose = verbose;

      Class<?>[] protocols;
      if (protocolClass == null) { // derive protocol from impl
        /*
         * In order to remain compatible with the old usage where a single
         * target protocolImpl is suppled for all protocol interfaces, and
         * the protocolImpl is derived from the protocolClass(es)
         * we register all interfaces extended by the protocolImpl
         */
        protocols = RPC.getProtocolInterfaces(protocolImpl.getClass());

      } else {
        if (!protocolClass.isAssignableFrom(protocolImpl.getClass())) {
          throw new IOException(
              "protocolClass "
                  + protocolClass
                  + " is not implemented by protocolImpl which is of class "
                  + protocolImpl.getClass());
        }
        // register protocol class and its super interfaces
        registerProtocolAndImpl(RPC.RpcKind.RPC_WRITABLE, protocolClass, protocolImpl);
        protocols = RPC.getProtocolInterfaces(protocolClass);
      }
      for (Class<?> p : protocols) {
        if (!p.equals(VersionedProtocol.class)) {
          registerProtocolAndImpl(RPC.RpcKind.RPC_WRITABLE, p, protocolImpl);
        }
      }
    }

    private static void log(String value) {
      if (value != null && value.length() > 55) value = value.substring(0, 55) + "...";
      LOG.info(value);
    }

    static class WritableRpcInvoker implements RpcInvoker {

      @Override
      public Writable call(
          org.apache.hadoop.ipc.RPC.Server server,
          String protocolName,
          Writable rpcRequest,
          long receivedTime)
          throws IOException {
        try {
          Invocation call = (Invocation) rpcRequest;
          if (server.verbose) log("Call: " + call);

          // Verify rpc version
          if (call.getRpcVersion() != writableRpcVersion) {
            // Client is using a different version of WritableRpc
            throw new IOException(
                "WritableRpc version mismatch, client side version="
                    + call.getRpcVersion()
                    + ", server side version="
                    + writableRpcVersion);
          }

          long clientVersion = call.getProtocolVersion();
          final String protoName;
          ProtoClassProtoImpl protocolImpl;
          if (call.declaringClassProtocolName.equals(VersionedProtocol.class.getName())) {
            // VersionProtocol methods are often used by client to figure out
            // which version of protocol to use.
            //
            // Versioned protocol methods should go the protocolName protocol
            // rather than the declaring class of the method since the
            // the declaring class is VersionedProtocol which is not
            // registered directly.
            // Send the call to the highest  protocol version
            VerProtocolImpl highest =
                server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protocolName);
            if (highest == null) {
              throw new IOException("Unknown protocol: " + protocolName);
            }
            protocolImpl = highest.protocolTarget;
          } else {
            protoName = call.declaringClassProtocolName;

            // Find the right impl for the protocol based on client version.
            ProtoNameVer pv = new ProtoNameVer(call.declaringClassProtocolName, clientVersion);
            protocolImpl = server.getProtocolImplMap(RPC.RpcKind.RPC_WRITABLE).get(pv);
            if (protocolImpl == null) { // no match for Protocol AND Version
              VerProtocolImpl highest =
                  server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protoName);
              if (highest == null) {
                throw new IOException("Unknown protocol: " + protoName);
              } else { // protocol supported but not the version that client wants
                throw new RPC.VersionMismatch(protoName, clientVersion, highest.version);
              }
            }
          }

          // Invoke the protocol method

          long startTime = Time.now();
          Method method =
              protocolImpl.protocolClass.getMethod(
                  call.getMethodName(), call.getParameterClasses());
          method.setAccessible(true);
          server.rpcDetailedMetrics.init(protocolImpl.protocolClass);
          Object value = method.invoke(protocolImpl.protocolImpl, call.getParameters());
          int processingTime = (int) (Time.now() - startTime);
          int qTime = (int) (startTime - receivedTime);
          if (LOG.isDebugEnabled()) {
            LOG.debug(
                "Served: "
                    + call.getMethodName()
                    + " queueTime= "
                    + qTime
                    + " procesingTime= "
                    + processingTime);
          }
          server.rpcMetrics.addRpcQueueTime(qTime);
          server.rpcMetrics.addRpcProcessingTime(processingTime);
          server.rpcDetailedMetrics.addProcessingTime(call.getMethodName(), processingTime);
          if (server.verbose) log("Return: " + value);

          return new ObjectWritable(method.getReturnType(), value);

        } catch (InvocationTargetException e) {
          Throwable target = e.getTargetException();
          if (target instanceof IOException) {
            throw (IOException) target;
          } else {
            IOException ioe = new IOException(target.toString());
            ioe.setStackTrace(target.getStackTrace());
            throw ioe;
          }
        } catch (Throwable e) {
          if (!(e instanceof IOException)) {
            LOG.error("Unexpected throwable object ", e);
          }
          IOException ioe = new IOException(e.toString());
          ioe.setStackTrace(e.getStackTrace());
          throw ioe;
        }
      }
    }
  }

  @Override
  public ProtocolProxy<ProtocolMetaInfoPB> getProtocolMetaInfoProxy(
      ConnectionId connId, Configuration conf, SocketFactory factory) throws IOException {
    throw new UnsupportedOperationException("This proxy is not supported");
  }
}
/** @author stone */
public class LSATApplication extends javax.swing.JFrame {
  // private user variables declaration
  private Project theProject;
  private JDesktopPane theDesktop;
  private DocumentViewerFrame documentViewerTable;
  private LSAResults currentResults;
  private DebugFrame debugFrame;
  private Log log = LogFactory.getLog();
  private LSATPreferences thePrefs = LSATPreferencesFactory.getPrefs(false);
  private DocumentFrameTableModel docFrameTableModel;
  private File lastPath;
  // end private user variables declaration

  /** Creates new form LSATApplication */
  public LSATApplication() {
    initComponents();
    initUserComponents();
    setWindowSizeAndTitle();
  }

  /**
   * This method is called from within the constructor to initialize the form. WARNING: Do NOT
   * modify this code. The content of this method is always regenerated by the Form Editor.
   */
  // <editor-fold defaultstate="collapsed" desc=" Generated Code ">//GEN-BEGIN:initComponents
  private void initComponents() {
    jMenuBar = new javax.swing.JMenuBar();
    jMenuFile = new javax.swing.JMenu();
    jMenuItemNewProject = new javax.swing.JMenuItem();
    jMenuItemLoadProject = new javax.swing.JMenuItem();
    jMenuItemSaveProject = new javax.swing.JMenuItem();
    jMenuItemCloseProject = new javax.swing.JMenuItem();
    jSeparator = new javax.swing.JSeparator();
    jMenuItemImport = new javax.swing.JMenuItem();
    jSeparator1 = new javax.swing.JSeparator();
    jMenuItemQuit = new javax.swing.JMenuItem();
    jMenuEdit = new javax.swing.JMenu();
    jMenuItemCut = new javax.swing.JMenuItem();
    jMenuItemCopy = new javax.swing.JMenuItem();
    jMenuItemPaste = new javax.swing.JMenuItem();
    jMenuView = new javax.swing.JMenu();
    jMenuItemViewDocuments = new javax.swing.JMenuItem();
    jMenuItemViewDebug = new javax.swing.JMenuItem();
    jMenuLSA = new javax.swing.JMenu();
    jMenuItemPerformLSA = new javax.swing.JMenuItem();
    jSeparator2 = new javax.swing.JSeparator();
    jMenuItemViewLSAResults = new javax.swing.JMenuItem();
    jMenuItemViewVisualResults = new javax.swing.JMenuItem();
    jSeparator3 = new javax.swing.JSeparator();
    jMenuItemLoadLSAResults = new javax.swing.JMenuItem();
    jMenuItemSaveLSAResults = new javax.swing.JMenuItem();
    jMenuItemDumpChunks = new javax.swing.JMenuItem();
    jMenuItemLogGraphingStats = new javax.swing.JMenuItem();
    jMenuItemLogRSGraphingStats = new javax.swing.JMenuItem();
    jMenuPreferences = new javax.swing.JMenu();
    jMenuItemShowPreferences = new javax.swing.JMenuItem();

    getContentPane().setLayout(null);

    setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);
    jMenuFile.setText("File");
    jMenuItemNewProject.setText("New Project");
    jMenuItemNewProject.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemNewProjectActionPerformed(evt);
          }
        });

    jMenuFile.add(jMenuItemNewProject);

    jMenuItemLoadProject.setText("Load Project");
    jMenuItemLoadProject.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemLoadProjectActionPerformed(evt);
          }
        });

    jMenuFile.add(jMenuItemLoadProject);

    jMenuItemSaveProject.setText("Save Project");
    jMenuItemSaveProject.setEnabled(false);
    jMenuItemSaveProject.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemSaveProjectActionPerformed(evt);
          }
        });

    jMenuFile.add(jMenuItemSaveProject);

    jMenuItemCloseProject.setText("Close Project");
    jMenuItemCloseProject.setEnabled(false);
    jMenuFile.add(jMenuItemCloseProject);

    jMenuFile.add(jSeparator);

    jMenuItemImport.setText("Import");
    jMenuItemImport.setEnabled(false);
    jMenuItemImport.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemImportActionPerformed(evt);
          }
        });

    jMenuFile.add(jMenuItemImport);

    jMenuFile.add(jSeparator1);

    jMenuItemQuit.setText("Quit");
    jMenuItemQuit.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemQuitActionPerformed(evt);
          }
        });

    jMenuFile.add(jMenuItemQuit);

    jMenuBar.add(jMenuFile);

    jMenuEdit.setText("Edit");
    jMenuItemCut.setText("Cut");
    jMenuItemCut.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemCutActionPerformed(evt);
          }
        });

    jMenuEdit.add(jMenuItemCut);

    jMenuItemCopy.setText("Copy");
    jMenuEdit.add(jMenuItemCopy);

    jMenuItemPaste.setText("Paste");
    jMenuEdit.add(jMenuItemPaste);

    jMenuBar.add(jMenuEdit);

    jMenuView.setText("View");
    jMenuView.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuViewActionPerformed(evt);
          }
        });

    jMenuItemViewDocuments.setText("Documents");
    jMenuItemViewDocuments.setEnabled(false);
    jMenuItemViewDocuments.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemViewDocumentsActionPerformed(evt);
          }
        });

    jMenuView.add(jMenuItemViewDocuments);

    jMenuItemViewDebug.setText("View Debug");
    jMenuItemViewDebug.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemViewDebugActionPerformed(evt);
          }
        });

    jMenuView.add(jMenuItemViewDebug);

    jMenuBar.add(jMenuView);

    jMenuLSA.setText("LSA");
    jMenuItemPerformLSA.setText("Perform LSA");
    jMenuItemPerformLSA.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemPerformLSAActionPerformed(evt);
          }
        });

    jMenuLSA.add(jMenuItemPerformLSA);

    jMenuLSA.add(jSeparator2);

    jMenuItemViewLSAResults.setText("View LSA Results");
    jMenuItemViewLSAResults.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemViewLSAResultsActionPerformed(evt);
          }
        });

    jMenuLSA.add(jMenuItemViewLSAResults);

    jMenuItemViewVisualResults.setText("View results graph");
    jMenuItemViewVisualResults.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemViewVisualResultsActionPerformed(evt);
          }
        });

    jMenuLSA.add(jMenuItemViewVisualResults);

    jMenuLSA.add(jSeparator3);

    jMenuItemLoadLSAResults.setText("Load LSA Results");
    jMenuItemLoadLSAResults.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemLoadLSAResultsActionPerformed(evt);
          }
        });

    jMenuLSA.add(jMenuItemLoadLSAResults);

    jMenuItemSaveLSAResults.setText("Save LSA Results");
    jMenuItemSaveLSAResults.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemSaveLSAResultsActionPerformed(evt);
          }
        });

    jMenuLSA.add(jMenuItemSaveLSAResults);

    jMenuItemDumpChunks.setText("Dump Chunks Belonging to Class");
    jMenuItemDumpChunks.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemDumpChunksActionPerformed(evt);
          }
        });

    jMenuLSA.add(jMenuItemDumpChunks);

    jMenuItemLogGraphingStats.setText("Log Graphing Stats");
    jMenuItemLogGraphingStats.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemLogGraphingStatsActionPerformed(evt);
          }
        });
    jMenuItemLogGraphingStats.addMouseListener(
        new java.awt.event.MouseAdapter() {
          public void mouseClicked(java.awt.event.MouseEvent evt) {
            jMenuItemLogGraphingStatsMouseClicked(evt);
          }
        });

    jMenuLSA.add(jMenuItemLogGraphingStats);

    jMenuItemLogRSGraphingStats.setText("Log RS Graphing Stats");
    jMenuItemLogRSGraphingStats.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemLogRSGraphingStatsActionPerformed(evt);
          }
        });
    jMenuItemLogRSGraphingStats.addMouseListener(
        new java.awt.event.MouseAdapter() {
          public void mouseClicked(java.awt.event.MouseEvent evt) {
            jMenuItemLogRSGraphingStatsMouseClicked(evt);
          }
        });

    jMenuLSA.add(jMenuItemLogRSGraphingStats);

    jMenuBar.add(jMenuLSA);

    jMenuPreferences.setText("Preferences");
    jMenuItemShowPreferences.setText("Show preferences dialog");
    jMenuItemShowPreferences.addActionListener(
        new java.awt.event.ActionListener() {
          public void actionPerformed(java.awt.event.ActionEvent evt) {
            jMenuItemShowPreferencesActionPerformed(evt);
          }
        });

    jMenuPreferences.add(jMenuItemShowPreferences);

    jMenuBar.add(jMenuPreferences);

    setJMenuBar(jMenuBar);

    pack();
  } // </editor-fold>//GEN-END:initComponents

  private void jMenuItemLogRSGraphingStatsMouseClicked(
      java.awt.event.MouseEvent evt) // GEN-FIRST:event_jMenuItemLogRSGraphingStatsMouseClicked
      { // GEN-HEADEREND:event_jMenuItemLogRSGraphingStatsMouseClicked
    // TODO add your handling code here:
  } // GEN-LAST:event_jMenuItemLogRSGraphingStatsMouseClicked

  private void jMenuItemLogRSGraphingStatsActionPerformed(
      java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemLogRSGraphingStatsActionPerformed
      { // GEN-HEADEREND:event_jMenuItemLogRSGraphingStatsActionPerformed
    if (currentResults != null) {
      this.currentResults.printGraphingStatsFromReqSimilieData();
    }
  } // GEN-LAST:event_jMenuItemLogRSGraphingStatsActionPerformed

  private void jMenuItemViewVisualResultsActionPerformed(
      java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemViewVisualResultsActionPerformed
      { // GEN-HEADEREND:event_jMenuItemViewVisualResultsActionPerformed
    if (this.currentResults != null) {
      VisualAnalysisFrame f = new VisualAnalysisFrame(this.currentResults, this.theProject);
      theDesktop.add(f);
    }
  } // GEN-LAST:event_jMenuItemViewVisualResultsActionPerformed

  private void jMenuItemLoadProjectActionPerformed(
      java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemLoadProjectActionPerformed
      { // GEN-HEADEREND:event_jMenuItemLoadProjectActionPerformed
    JFileChooser jfc = new JFileChooser();
    if (lastPath != null) {
      jfc.setCurrentDirectory(lastPath);
    }
    int fileDialogReturnVal = jfc.showOpenDialog(this);

    if (fileDialogReturnVal == JFileChooser.APPROVE_OPTION) {
      try {
        File inputFile = jfc.getSelectedFile();
        FileInputStream fis = new FileInputStream(inputFile);
        ObjectInputStream ois = new ObjectInputStream(fis);

        this.theProject = (Project) ois.readObject();
        this.currentResults = (LSAResults) ois.readObject();

        lastPath = new File(jfc.getSelectedFile().getPath());
      } catch (IOException e) {
        if (this.theProject == null) {
          log.log(Log.ERROR, "Failed to load project");
        }
        if (this.currentResults == null) {
          log.log(Log.WARNING, "Failed to load results");
        }

        log.log(Log.WARNING, e.getMessage());
      } catch (ClassNotFoundException e) {
        log.log(Log.ERROR, "Class not found error, version mismatch");
      }
    }
    if (this.theProject != null) {
      jMenuItemViewDocuments.setEnabled(true);
      jMenuItemSaveProject.setEnabled(true);
      this.setTitle(theProject.getProjectName());
      log.log(Log.INFO, "Project Loaded");
    }
    if (this.currentResults != null) {
      log.log(Log.INFO, "Results loaded");
    }
  } // GEN-LAST:event_jMenuItemLoadProjectActionPerformed

  private void jMenuItemSaveProjectActionPerformed(
      java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemSaveProjectActionPerformed
      { // GEN-HEADEREND:event_jMenuItemSaveProjectActionPerformed
    if (this.theProject != null) {
      JFileChooser jfc = new JFileChooser();
      int fileDialogReturnVal = jfc.showSaveDialog(this);

      if (fileDialogReturnVal == JFileChooser.APPROVE_OPTION) {
        try {
          File outputFile = jfc.getSelectedFile();
          FileOutputStream fos = new FileOutputStream(outputFile);
          ObjectOutputStream oos = new ObjectOutputStream(fos);

          oos.writeObject(this.theProject);
          if (this.currentResults != null) {
            oos.writeObject(this.currentResults);
          }
        } catch (IOException e) {
          log.log(Log.ERROR, "Failed to save file\n" + e.getMessage());
        }
      }
    }
  } // GEN-LAST:event_jMenuItemSaveProjectActionPerformed

  private void jMenuItemCutActionPerformed(
      java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemCutActionPerformed
      { // GEN-HEADEREND:event_jMenuItemCutActionPerformed
    Toolkit t = java.awt.Toolkit.getDefaultToolkit();
    Clipboard c = t.getSystemClipboard();

    JInternalFrame currentFrame = theDesktop.getSelectedFrame();
    /*StringSelection contents = new StringSelection(srcData);
    clipboard.setContents(contents, this);*/

  } // GEN-LAST:event_jMenuItemCutActionPerformed

  private void jMenuItemShowPreferencesActionPerformed(
      java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemShowPreferencesActionPerformed
      { // GEN-HEADEREND:event_jMenuItemShowPreferencesActionPerformed
    PreferencesDialog.showPreferencesDialog(this, this.thePrefs);
  } // GEN-LAST:event_jMenuItemShowPreferencesActionPerformed

  private void jMenuItemLogGraphingStatsActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemLogGraphingStatsActionPerformed
    if (currentResults != null) {
      // this.currentResults.printGraphingStats();
    }
  } // GEN-LAST:event_jMenuItemLogGraphingStatsActionPerformed

  private void jMenuItemLogGraphingStatsMouseClicked(
      java.awt.event.MouseEvent evt) { // GEN-FIRST:event_jMenuItemLogGraphingStatsMouseClicked
  } // GEN-LAST:event_jMenuItemLogGraphingStatsMouseClicked

  private void jMenuItemDumpChunksActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemDumpChunksActionPerformed
    if (currentResults != null) {
      DocumentClassDialog d = DocumentClassDialog.showClassDialog(this, this.theProject);
      DocumentClass selectedClass = d.getSelectedDocumentClass();

      currentResults.dumpChunksBelongingToDocumentsWithClass(selectedClass.getId());
    }
  } // GEN-LAST:event_jMenuItemDumpChunksActionPerformed

  private void jMenuItemViewDebugActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemViewDebugActionPerformed
    this.debugFrame.setVisible(true);
  } // GEN-LAST:event_jMenuItemViewDebugActionPerformed

  private void jMenuItemViewLSAResultsActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemViewLSAResultsActionPerformed
    if (this.currentResults != null) {
      LSAResultsFrame f = new LSAResultsFrame(this.currentResults, this.theProject);
      theDesktop.add(f);
    }
  } // GEN-LAST:event_jMenuItemViewLSAResultsActionPerformed

  private void jMenuItemLoadLSAResultsActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemLoadLSAResultsActionPerformed
    JFileChooser jfc = new JFileChooser();
    int fileDialogReturnVal = jfc.showOpenDialog(this);

    if (fileDialogReturnVal == JFileChooser.APPROVE_OPTION) {
      try {
        File inputFile = jfc.getSelectedFile();
        FileInputStream fis = new FileInputStream(inputFile);
        ObjectInputStream ois = new ObjectInputStream(fis);

        this.currentResults = (LSAResults) ois.readObject();
      } catch (IOException e) {
        log.log(Log.ERROR, "Failed to load LSA results\n" + e.getMessage());
      } catch (ClassNotFoundException e) {
        log.log(Log.ERROR, "Class not found : Error loading LSA results due to version mismatch");
      }

      System.out.println(currentResults == null);
    }
  } // GEN-LAST:event_jMenuItemLoadLSAResultsActionPerformed

  private void jMenuItemSaveLSAResultsActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemSaveLSAResultsActionPerformed
    if (this.currentResults != null) {
      JFileChooser jfc = new JFileChooser();
      int fileDialogReturnVal = jfc.showSaveDialog(this);

      if (fileDialogReturnVal == JFileChooser.APPROVE_OPTION) {
        try {
          File outputFile = jfc.getSelectedFile();
          FileOutputStream fos = new FileOutputStream(outputFile);
          ObjectOutputStream oos = new ObjectOutputStream(fos);

          oos.writeObject(this.currentResults);
        } catch (IOException e) {
          System.out.println("IOexception");
          System.out.println(e.getMessage());
        }
      }
    }
  } // GEN-LAST:event_jMenuItemSaveLSAResultsActionPerformed

  private void jMenuItemPerformLSAActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemPerformLSAActionPerformed
    Thread t =
        new LSAThread(
            theProject.getDocumentCollection(),
            theProject.getDocumentClassCollection(),
            thePrefs.get("lsa-regex"),
            this);
    t.start();
  } // GEN-LAST:event_jMenuItemPerformLSAActionPerformed

  private void jMenuItemImportActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemImportActionPerformed
    JFileChooser jfc = new JFileChooser();
    jfc.setMultiSelectionEnabled(true);
    if (lastPath != null) {
      jfc.setCurrentDirectory(lastPath);
    }
    int fileDialogReturnVal = jfc.showOpenDialog(this);

    // now select the file
    if (fileDialogReturnVal == JFileChooser.APPROVE_OPTION) {
      // add code here to allow selection of a document class

      DocumentClassDialog d = DocumentClassDialog.showClassDialog(this, this.theProject);

      DocumentClass selectedClass = d.getSelectedDocumentClass();

      File[] selected = jfc.getSelectedFiles();
      for (int i = 0; i < selected.length; i++) {
        theProject.addNewDocument(selected[i], 1.0f, selected[i].toString(), selectedClass);
      }

      docFrameTableModel.fireTableDataChanged();

      lastPath = new File(jfc.getSelectedFile().getPath());
    }
  } // GEN-LAST:event_jMenuItemImportActionPerformed

  private void jMenuItemViewDocumentsActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemViewDocumentsActionPerformed
    if (this.docFrameTableModel == null) {
      this.docFrameTableModel = new DocumentFrameTableModel(theProject.getDocumentCollection());
    }
    DocumentFrame d =
        new DocumentFrame(this.docFrameTableModel, theDesktop, theProject.getDocumentCollection());
    d.setVisible(true);
    theDesktop.add(d);
  } // GEN-LAST:event_jMenuItemViewDocumentsActionPerformed

  private void jMenuViewActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuViewActionPerformed
  } // GEN-LAST:event_jMenuViewActionPerformed

  private void jMenuItemQuitActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemQuitActionPerformed
    this.shutDown();
    System.exit(0);
  } // GEN-LAST:event_jMenuItemQuitActionPerformed

  private void jMenuItemNewProjectActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemNewProjectActionPerformed
    String projectName = JOptionPane.showInputDialog(null, "Please enter a project name");

    if (projectName != null) {
      theProject = new Project(projectName);

      // now we've got a project we need to enalbe the save and close buttons
      jMenuItemSaveProject.setEnabled(true);
      jMenuItemCloseProject.setEnabled(true);
      jMenuItemViewDocuments.setEnabled(true);
      jMenuItemImport.setEnabled(true);

      // set title
      setTitle("LSAT - " + projectName);

      jMenuItemViewDocumentsActionPerformed(null);
    }
  } // GEN-LAST:event_jMenuItemNewProjectActionPerformed

  private void setWindowSizeAndTitle() {
    Dimension screenSize = this.getToolkit().getScreenSize();
    int newHeight = (int) ((float) screenSize.height * 0.7);
    int newWidth = (int) ((float) screenSize.width * 0.7);
    this.setSize(newWidth, newHeight);
    newHeight = (int) ((0.3 * (float) screenSize.height) / 2);
    newWidth = (int) ((0.3 * (float) screenSize.width) / 2);
    this.setLocation(newWidth, newHeight);
    this.setTitle("LSAT");
  }

  private void initUserComponents() {
    // required to attach windows to in the MDI world
    theDesktop = new JDesktopPane();
    setContentPane(theDesktop);

    // create and add the debug frame
    debugFrame = new DebugFrame();
    debugFrame.setVisible(false);
    theDesktop.add(debugFrame);
  }

  private void shutDown() {
    // nothing in here yet
  }

  public void showLSAResults(LSAResults results) {
    this.currentResults = results;
    jMenuItemViewLSAResultsActionPerformed(null);
  }

  // Variables declaration - do not modify//GEN-BEGIN:variables
  private javax.swing.JMenuBar jMenuBar;
  private javax.swing.JMenu jMenuEdit;
  private javax.swing.JMenu jMenuFile;
  private javax.swing.JMenuItem jMenuItemCloseProject;
  private javax.swing.JMenuItem jMenuItemCopy;
  private javax.swing.JMenuItem jMenuItemCut;
  private javax.swing.JMenuItem jMenuItemDumpChunks;
  private javax.swing.JMenuItem jMenuItemImport;
  private javax.swing.JMenuItem jMenuItemLoadLSAResults;
  private javax.swing.JMenuItem jMenuItemLoadProject;
  private javax.swing.JMenuItem jMenuItemLogGraphingStats;
  private javax.swing.JMenuItem jMenuItemLogRSGraphingStats;
  private javax.swing.JMenuItem jMenuItemNewProject;
  private javax.swing.JMenuItem jMenuItemPaste;
  private javax.swing.JMenuItem jMenuItemPerformLSA;
  private javax.swing.JMenuItem jMenuItemQuit;
  private javax.swing.JMenuItem jMenuItemSaveLSAResults;
  private javax.swing.JMenuItem jMenuItemSaveProject;
  private javax.swing.JMenuItem jMenuItemShowPreferences;
  private javax.swing.JMenuItem jMenuItemViewDebug;
  private javax.swing.JMenuItem jMenuItemViewDocuments;
  private javax.swing.JMenuItem jMenuItemViewLSAResults;
  private javax.swing.JMenuItem jMenuItemViewVisualResults;
  private javax.swing.JMenu jMenuLSA;
  private javax.swing.JMenu jMenuPreferences;
  private javax.swing.JMenu jMenuView;
  private javax.swing.JSeparator jSeparator;
  private javax.swing.JSeparator jSeparator1;
  private javax.swing.JSeparator jSeparator2;
  private javax.swing.JSeparator jSeparator3;
  // End of variables declaration//GEN-END:variables

  /** @param args the command line arguments */
  public static void main(String args[]) {
    java.awt.EventQueue.invokeLater(
        new Runnable() {
          public void run() {
            new LSATApplication().setVisible(true);
          }
        });
  }
}
/**
 * ******************************************************** The Secondary NameNode is a helper to
 * the primary NameNode. The Secondary is responsible for supporting periodic checkpoints of the
 * HDFS metadata. The current design allows only one Secondary NameNode per HDFs cluster.
 *
 * <p>The Secondary NameNode is a daemon that periodically wakes up (determined by the schedule
 * specified in the configuration), triggers a periodic checkpoint and then goes back to sleep. The
 * Secondary NameNode uses the ClientProtocol to talk to the primary NameNode.
 *
 * <p>********************************************************
 */
public class SecondaryNameNode implements Runnable {

  public static final Log LOG = LogFactory.getLog(SecondaryNameNode.class.getName());

  private String fsName;
  private CheckpointStorage checkpointImage;
  private FSNamesystem namesystem;

  private NamenodeProtocol namenode;
  private Configuration conf;
  private InetSocketAddress nameNodeAddr;
  private volatile boolean shouldRun;
  private HttpServer infoServer;
  private int infoPort;
  private String infoBindAddress;

  private Collection<File> checkpointDirs;
  private Collection<File> checkpointEditsDirs;
  private long checkpointPeriod; // in seconds
  private long checkpointSize; // size (in MB) of current Edit Log

  /** Utility class to facilitate junit test error simulation. */
  static class ErrorSimulator {
    private static boolean[] simulation = null; // error simulation events

    static void initializeErrorSimulationEvent(int numberOfEvents) {
      simulation = new boolean[numberOfEvents];
      for (int i = 0; i < numberOfEvents; i++) {
        simulation[i] = false;
      }
    }

    static boolean getErrorSimulation(int index) {
      if (simulation == null) return false;
      assert (index < simulation.length);
      return simulation[index];
    }

    static void setErrorSimulation(int index) {
      assert (index < simulation.length);
      simulation[index] = true;
    }

    static void clearErrorSimulation(int index) {
      assert (index < simulation.length);
      simulation[index] = false;
    }
  }

  FSImage getFSImage() {
    return checkpointImage;
  }

  /** Create a connection to the primary namenode. */
  public SecondaryNameNode(Configuration conf) throws IOException {
    try {
      initialize(conf);
    } catch (IOException e) {
      shutdown();
      throw e;
    }
  }

  /** Initialize SecondaryNameNode. */
  private void initialize(Configuration conf) throws IOException {
    // initiate Java VM metrics
    JvmMetrics.init("SecondaryNameNode", conf.get("session.id"));

    // Create connection to the namenode.
    shouldRun = true;
    nameNodeAddr = NameNode.getAddress(conf);

    this.conf = conf;
    this.namenode =
        (NamenodeProtocol)
            RPC.waitForProxy(
                NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, conf);

    // initialize checkpoint directories
    fsName = getInfoServer();
    checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointImage = new CheckpointStorage(conf);
    checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);

    // Initialize other scheduling parameters from the configuration
    checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
    checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);

    // initialize the webserver for uploading files.
    String infoAddr =
        NetUtils.getServerAddress(
            conf,
            "dfs.secondary.info.bindAddress",
            "dfs.secondary.info.port",
            "dfs.secondary.http.address");
    InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
    infoBindAddress = infoSocAddr.getHostName();
    int tmpInfoPort = infoSocAddr.getPort();
    infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf);
    infoServer.setAttribute("name.system.image", checkpointImage);
    this.infoServer.setAttribute("name.conf", conf);
    infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
    infoServer.start();

    // The web-server port can be ephemeral... ensure we have the correct info
    infoPort = infoServer.getPort();
    conf.set("dfs.secondary.http.address", infoBindAddress + ":" + infoPort);
    LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" + infoPort);
    LOG.warn(
        "Checkpoint Period   :"
            + checkpointPeriod
            + " secs "
            + "("
            + checkpointPeriod / 60
            + " min)");
    LOG.warn(
        "Log Size Trigger    :"
            + checkpointSize
            + " bytes "
            + "("
            + checkpointSize / 1024
            + " KB)");
  }

  /** Shut down this instance of the datanode. Returns only after shutdown is complete. */
  public void shutdown() {
    shouldRun = false;
    try {
      if (infoServer != null) infoServer.stop();
    } catch (Exception e) {
      LOG.warn("Exception shutting down SecondaryNameNode", e);
    }
    try {
      if (checkpointImage != null) checkpointImage.close();
    } catch (IOException e) {
      LOG.warn(StringUtils.stringifyException(e));
    }
  }

  //
  // The main work loop
  //
  public void run() {

    //
    // Poll the Namenode (once every 5 minutes) to find the size of the
    // pending edit log.
    //
    long period = 5 * 60; // 5 minutes
    long lastCheckpointTime = 0;
    if (checkpointPeriod < period) {
      period = checkpointPeriod;
    }

    while (shouldRun) {
      try {
        Thread.sleep(1000 * period);
      } catch (InterruptedException ie) {
        // do nothing
      }
      if (!shouldRun) {
        break;
      }
      try {
        long now = System.currentTimeMillis();

        long size = namenode.getEditLogSize();
        if (size >= checkpointSize || now >= lastCheckpointTime + 1000 * checkpointPeriod) {
          doCheckpoint();
          lastCheckpointTime = now;
        }
      } catch (IOException e) {
        LOG.error("Exception in doCheckpoint: ");
        LOG.error(StringUtils.stringifyException(e));
        e.printStackTrace();
        checkpointImage.imageDigest = null;
      } catch (Throwable e) {
        LOG.error("Throwable Exception in doCheckpoint: ");
        LOG.error(StringUtils.stringifyException(e));
        e.printStackTrace();
        Runtime.getRuntime().exit(-1);
      }
    }
  }

  /**
   * Download <code>fsimage</code> and <code>edits</code> files from the name-node.
   *
   * @return true if a new image has been downloaded and needs to be loaded
   * @throws IOException
   */
  private boolean downloadCheckpointFiles(CheckpointSignature sig) throws IOException {

    checkpointImage.cTime = sig.cTime;
    checkpointImage.checkpointTime = sig.checkpointTime;

    boolean downloadImage = true;
    String fileid;
    File[] srcNames;
    if (sig.imageDigest.equals(checkpointImage.imageDigest)) {
      downloadImage = false;
      LOG.info("Image has not changed. Will not download image.");
    } else {
      // get fsimage
      srcNames = checkpointImage.getImageFiles();
      assert srcNames.length > 0 : "No checkpoint targets.";
      fileid = "getimage=1";
      TransferFsImage.getFileClient(fsName, fileid, srcNames, false);
      checkpointImage.imageDigest = sig.imageDigest;
      LOG.info(
          "Downloaded file " + srcNames[0].getName() + " size " + srcNames[0].length() + " bytes.");
    }
    // get edits file
    fileid = "getedit=1";
    srcNames = checkpointImage.getEditsFiles();
    assert srcNames.length > 0 : "No checkpoint targets.";
    TransferFsImage.getFileClient(fsName, fileid, srcNames, false);
    LOG.info(
        "Downloaded file " + srcNames[0].getName() + " size " + srcNames[0].length() + " bytes.");

    checkpointImage.checkpointUploadDone(null);

    return downloadImage;
  }

  /** Copy the new fsimage into the NameNode */
  private void putFSImage(CheckpointSignature sig) throws IOException {
    String fileid =
        "putimage=1&port="
            + infoPort
            + "&machine="
            + InetAddress.getLocalHost().getHostAddress()
            + "&token="
            + sig.toString();
    LOG.info("Posted URL " + fsName + fileid);
    TransferFsImage.getFileClient(fsName, fileid, (File[]) null, false);
  }

  /** Returns the Jetty server that the Namenode is listening on. */
  private String getInfoServer() throws IOException {
    URI fsName = FileSystem.getDefaultUri(conf);
    if (!"hdfs".equals(fsName.getScheme())) {
      throw new IOException("This is not a DFS");
    }
    return NetUtils.getServerAddress(
        conf, "dfs.info.bindAddress", "dfs.info.port", "dfs.http.address");
  }

  /** Create a new checkpoint */
  void doCheckpoint() throws IOException {

    LOG.info("Checkpoint starting");

    // Do the required initialization of the merge work area.
    startCheckpoint();

    // Tell the namenode to start logging transactions in a new edit file
    // Returns a token that would be used to upload the merged image.
    CheckpointSignature sig = (CheckpointSignature) namenode.rollEditLog();

    // error simulation code for junit test
    if (ErrorSimulator.getErrorSimulation(0)) {
      throw new IOException("Simulating error0 " + "after creating edits.new");
    }

    boolean loadImage = downloadCheckpointFiles(sig); // Fetch fsimage and edits
    doMerge(sig, loadImage); // Do the merge

    //
    // Upload the new image into the NameNode. Then tell the Namenode
    // to make this new uploaded image as the most current image.
    //
    putFSImage(sig);

    // error simulation code for junit test
    if (ErrorSimulator.getErrorSimulation(1)) {
      throw new IOException("Simulating error1 " + "after uploading new image to NameNode");
    }

    namenode.rollFsImage(new CheckpointSignature(checkpointImage));
    checkpointImage.endCheckpoint();

    LOG.info("Checkpoint done. New Image Size: " + checkpointImage.getFsImageName().length());
  }

  private void startCheckpoint() throws IOException {
    checkpointImage.unlockAll();
    checkpointImage.getEditLog().close();
    checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);
    checkpointImage.startCheckpoint();
  }

  /** Merge downloaded image and edits and write the new image into current storage directory. */
  private void doMerge(CheckpointSignature sig, boolean loadImage) throws IOException {
    if (loadImage) { // create an empty namespace if new image
      namesystem = new FSNamesystem(checkpointImage, conf);
    }
    assert namesystem.dir.fsImage == checkpointImage;
    checkpointImage.doMerge(sig, loadImage);
  }

  /**
   * @param argv The parameters passed to this program.
   * @exception Exception if the filesystem does not exist.
   * @return 0 on success, non zero on error.
   */
  private int processArgs(String[] argv) throws Exception {

    if (argv.length < 1) {
      printUsage("");
      return -1;
    }

    int exitCode = -1;
    int i = 0;
    String cmd = argv[i++];

    //
    // verify that we have enough command line parameters
    //
    if ("-geteditsize".equals(cmd)) {
      if (argv.length != 1) {
        printUsage(cmd);
        return exitCode;
      }
    } else if ("-checkpoint".equals(cmd)) {
      if (argv.length != 1 && argv.length != 2) {
        printUsage(cmd);
        return exitCode;
      }
      if (argv.length == 2 && !"force".equals(argv[i])) {
        printUsage(cmd);
        return exitCode;
      }
    }

    exitCode = 0;
    try {
      if ("-checkpoint".equals(cmd)) {
        long size = namenode.getEditLogSize();
        if (size >= checkpointSize || argv.length == 2 && "force".equals(argv[i])) {
          doCheckpoint();
        } else {
          System.err.println(
              "EditLog size "
                  + size
                  + " bytes is "
                  + "smaller than configured checkpoint "
                  + "size "
                  + checkpointSize
                  + " bytes.");
          System.err.println("Skipping checkpoint.");
        }
      } else if ("-geteditsize".equals(cmd)) {
        long size = namenode.getEditLogSize();
        System.out.println("EditLog size is " + size + " bytes");
      } else {
        exitCode = -1;
        LOG.error(cmd.substring(1) + ": Unknown command");
        printUsage("");
      }
    } catch (RemoteException e) {
      //
      // This is a error returned by hadoop server. Print
      // out the first line of the error mesage, ignore the stack trace.
      exitCode = -1;
      try {
        String[] content;
        content = e.getLocalizedMessage().split("\n");
        LOG.error(cmd.substring(1) + ": " + content[0]);
      } catch (Exception ex) {
        LOG.error(cmd.substring(1) + ": " + ex.getLocalizedMessage());
      }
    } catch (IOException e) {
      //
      // IO exception encountered locally.
      //
      exitCode = -1;
      LOG.error(cmd.substring(1) + ": " + e.getLocalizedMessage());
    } finally {
      // Does the RPC connection need to be closed?
    }
    return exitCode;
  }

  /**
   * Displays format of commands.
   *
   * @param cmd The command that is being executed.
   */
  private void printUsage(String cmd) {
    if ("-geteditsize".equals(cmd)) {
      System.err.println("Usage: java SecondaryNameNode" + " [-geteditsize]");
    } else if ("-checkpoint".equals(cmd)) {
      System.err.println("Usage: java SecondaryNameNode" + " [-checkpoint [force]]");
    } else {
      System.err.println(
          "Usage: java SecondaryNameNode " + "[-checkpoint [force]] " + "[-geteditsize] ");
    }
  }

  /**
   * main() has some simple utility methods.
   *
   * @param argv Command line parameters.
   * @exception Exception if the filesystem does not exist.
   */
  public static void main(String[] argv) throws Exception {
    StringUtils.startupShutdownMessage(SecondaryNameNode.class, argv, LOG);
    Configuration tconf = new Configuration();
    if (argv.length >= 1) {
      SecondaryNameNode secondary = new SecondaryNameNode(tconf);
      int ret = secondary.processArgs(argv);
      System.exit(ret);
    }

    // Create a never ending deamon
    Daemon checkpointThread = new Daemon(new SecondaryNameNode(tconf));
    checkpointThread.start();
  }

  static class CheckpointStorage extends FSImage {
    /** */
    CheckpointStorage(Configuration conf) throws IOException {
      super(conf);
    }

    @Override
    public boolean isConversionNeeded(StorageDirectory sd) {
      return false;
    }

    /**
     * Analyze checkpoint directories. Create directories if they do not exist. Recover from an
     * unsuccessful checkpoint is necessary.
     *
     * @param dataDirs
     * @param editsDirs
     * @throws IOException
     */
    void recoverCreate(Collection<File> dataDirs, Collection<File> editsDirs) throws IOException {
      Collection<File> tempDataDirs = new ArrayList<File>(dataDirs);
      Collection<File> tempEditsDirs = new ArrayList<File>(editsDirs);
      this.storageDirs = new ArrayList<StorageDirectory>();
      setStorageDirectories(tempDataDirs, tempEditsDirs);
      for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext(); ) {
        StorageDirectory sd = it.next();
        boolean isAccessible = true;
        try { // create directories if don't exist yet
          if (!sd.getRoot().mkdirs()) {
            // do nothing, directory is already created
          }
        } catch (SecurityException se) {
          isAccessible = false;
        }
        if (!isAccessible)
          throw new InconsistentFSStateException(
              sd.getRoot(), "cannot access checkpoint directory.");
        StorageState curState;
        try {
          curState = sd.analyzeStorage(HdfsConstants.StartupOption.REGULAR);
          // sd is locked but not opened
          switch (curState) {
            case NON_EXISTENT:
              // fail if any of the configured checkpoint dirs are inaccessible
              throw new InconsistentFSStateException(
                  sd.getRoot(), "checkpoint directory does not exist or is not accessible.");
            case NOT_FORMATTED:
              break; // it's ok since initially there is no current and VERSION
            case NORMAL:
              break;
            default: // recovery is possible
              sd.doRecover(curState);
          }
        } catch (IOException ioe) {
          sd.unlock();
          throw ioe;
        }
      }
    }

    /**
     * Prepare directories for a new checkpoint.
     *
     * <p>Rename <code>current</code> to <code>lastcheckpoint.tmp</code> and recreate <code>current
     * </code>.
     *
     * @throws IOException
     */
    void startCheckpoint() throws IOException {
      for (StorageDirectory sd : storageDirs) {
        moveCurrent(sd);
      }
    }

    void endCheckpoint() throws IOException {
      for (StorageDirectory sd : storageDirs) {
        moveLastCheckpoint(sd);
      }
    }

    /** Merge image and edits, and verify consistency with the signature. */
    private void doMerge(CheckpointSignature sig, boolean loadImage) throws IOException {
      getEditLog().open();
      StorageDirectory sdName = null;
      StorageDirectory sdEdits = null;
      Iterator<StorageDirectory> it = null;
      if (loadImage) {
        it = dirIterator(NameNodeDirType.IMAGE);
        if (it.hasNext()) sdName = it.next();
        if (sdName == null) throw new IOException("Could not locate checkpoint fsimage");
      }
      it = dirIterator(NameNodeDirType.EDITS);
      if (it.hasNext()) sdEdits = it.next();
      if (sdEdits == null) throw new IOException("Could not locate checkpoint edits");
      if (loadImage) {
        loadFSImage(FSImage.getImageFile(sdName, NameNodeFile.IMAGE));
      }
      loadFSEdits(sdEdits);
      sig.validateStorageInfo(this);
      saveNamespace(false);
    }
  }
}
/** This is <em>not</em> reusable. */
@SuppressWarnings("deprecation")
public class BSONWritable implements BSONObject, WritableComparable {

  /** Constructs a new instance. */
  public BSONWritable() {
    _doc = new BasicBSONObject();
  }

  /**
   * Copy constructor, copies data from an existing BSONWritable
   *
   * @param other The BSONWritable to copy from
   */
  public BSONWritable(BSONWritable other) {
    this();
    copy(other);
  }

  /** Constructs a new instance around an existing BSONObject */
  public BSONWritable(BSONObject doc) {
    this();
    putAll(doc);
  }

  /**
   * {@inheritDoc}
   *
   * @see BSONObject#put(String,Object)
   */
  public Object put(String key, Object value) {
    return _doc.put(key, value);
  }

  /**
   * {@inheritDoc}
   *
   * @see BSONObject#putAll(BSONObject)
   */
  public void putAll(BSONObject otherDoc) {
    _doc.putAll(otherDoc);
  }

  /**
   * {@inheritDoc}
   *
   * @see BSONObject#putAll(Map)
   */
  public void putAll(Map otherMap) {
    _doc.putAll(otherMap);
  }

  /**
   * {@inheritDoc}
   *
   * @see BSONObject#get(String)
   */
  public Object get(String key) {
    return _doc.get(key);
  }

  /**
   * {@inheritDoc}
   *
   * @see BSONObject#toMap()
   */
  public Map toMap() {
    return _doc.toMap();
  }

  /**
   * {@inheritDoc}
   *
   * @see BSONObject#removeField(String)
   */
  public Object removeField(String key) {
    return _doc.removeField(key);
  }

  /**
   * {@inheritDoc}
   *
   * @see BSONObject#containsKey(String)
   */
  public boolean containsKey(String key) {
    return _doc.containsKey(key);
  }

  /**
   * {@inheritDoc}
   *
   * @see BSONObject#containsField(String)
   */
  public boolean containsField(String fieldName) {
    return _doc.containsField(fieldName);
  }

  /**
   * {@inheritDoc}
   *
   * @see BSONObject#keySet()
   */
  public Set<java.lang.String> keySet() {
    return _doc.keySet();
  }

  /**
   * {@inheritDoc}
   *
   * @see Writable#write(DataOutput)
   */
  public void write(DataOutput out) throws IOException {
    BSONEncoder enc = new BSONEncoder();
    BasicOutputBuffer buf = new BasicOutputBuffer();
    enc.set(buf);
    enc.putObject(_doc);
    enc.done();
    out.writeInt(buf.size());
    // For better performance we can copy BasicOutputBuffer.pipe(OutputStream)
    // to have a method signature that works with DataOutput
    out.write(buf.toByteArray());
  }

  /**
   * {@inheritDoc}
   *
   * @see Writable#readFields(DataInput)
   */
  public void readFields(DataInput in) throws IOException {
    BSONDecoder dec = new BSONDecoder();
    BSONCallback cb = new BasicBSONCallback();
    // Read the BSON length from the start of the record
    int dataLen = in.readInt();
    byte[] buf = new byte[dataLen];
    in.readFully(buf);
    dec.decode(buf, cb);
    _doc = (BSONObject) cb.get();
    log.info("Decoded a BSON Object: " + _doc);
  }

  /** {@inheritDoc} */
  @Override
  public String toString() {
    BSONEncoder enc = new BSONEncoder();
    BasicOutputBuffer buf = new BasicOutputBuffer();
    enc.set(buf);
    enc.putObject(_doc);
    enc.done();
    String str = buf.asString();
    log.debug("Output As String: '" + str + "'");
    return str;
  }

  /** Used by child copy constructors. */
  protected synchronized void copy(Writable other) {
    if (other != null) {
      try {
        DataOutputBuffer out = new DataOutputBuffer();
        other.write(out);
        DataInputBuffer in = new DataInputBuffer();
        in.reset(out.getData(), out.getLength());
        readFields(in);

      } catch (IOException e) {
        throw new IllegalArgumentException("map cannot be copied: " + e.getMessage());
      }

    } else {
      throw new IllegalArgumentException("source map cannot be null");
    }
  }

  public static class Comparator extends WritableComparator {
    public Comparator() {
      super(BSONWritable.class);
    }

    public int compare(WritableComparable a, WritableComparable b) {

      if (a instanceof BSONWritable && b instanceof BSONWritable) {
        return ((BSONWritable) a)._doc.toString().compareTo(((BSONWritable) b)._doc.toString());
      } else {
        return -1;
      }
    }
  }

  static { // register this comparator
    WritableComparator.define(BSONWritable.class, new Comparator());
  }

  @Override
  public int compareTo(Object o) {
    return new Comparator().compare(this, o);
  }

  protected BSONObject _doc;

  private static final Log log = LogFactory.getLog(BSONWritable.class);
}
Exemple #14
0
/**
 * Base class that runs a task in a separate process. Tasks are run in a separate process in order
 * to isolate the map/reduce system code from bugs in user supplied map and reduce functions.
 */
abstract class TaskRunner extends Thread {
  public static final Log LOG = LogFactory.getLog(TaskRunner.class);

  volatile boolean killed = false;
  private TaskTracker.TaskInProgress tip;
  private Task t;
  private Object lock = new Object();
  private volatile boolean done = false;
  private int exitCode = -1;
  private boolean exitCodeSet = false;

  private TaskTracker tracker;

  protected JobConf conf;
  JvmManager jvmManager;

  /** for cleaning up old map outputs */
  protected MapOutputFile mapOutputFile;

  public TaskRunner(TaskTracker.TaskInProgress tip, TaskTracker tracker, JobConf conf) {
    this.tip = tip;
    this.t = tip.getTask();
    this.tracker = tracker;
    this.conf = conf;
    this.mapOutputFile = new MapOutputFile(t.getJobID());
    this.mapOutputFile.setConf(conf);
    this.jvmManager = tracker.getJvmManagerInstance();
  }

  public Task getTask() {
    return t;
  }

  public TaskTracker.TaskInProgress getTaskInProgress() {
    return tip;
  }

  public TaskTracker getTracker() {
    return tracker;
  }

  /**
   * Called to assemble this task's input. This method is run in the parent process before the child
   * is spawned. It should not execute user code, only system code.
   */
  public boolean prepare() throws IOException {
    return true;
  }

  /**
   * Called when this task's output is no longer needed. This method is run in the parent process
   * after the child exits. It should not execute user code, only system code.
   */
  public void close() throws IOException {}

  private static String stringifyPathArray(Path[] p) {
    if (p == null) {
      return null;
    }
    StringBuffer str = new StringBuffer(p[0].toString());
    for (int i = 1; i < p.length; i++) {
      str.append(",");
      str.append(p[i].toString());
    }
    return str.toString();
  }

  @Override
  public final void run() {
    try {

      // before preparing the job localize
      // all the archives
      TaskAttemptID taskid = t.getTaskID();
      LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir");
      File jobCacheDir = null;
      if (conf.getJar() != null) {
        jobCacheDir = new File(new Path(conf.getJar()).getParent().toString());
      }
      File workDir =
          new File(
              lDirAlloc
                  .getLocalPathToRead(
                      TaskTracker.getJobCacheSubdir()
                          + Path.SEPARATOR
                          + t.getJobID()
                          + Path.SEPARATOR
                          + t.getTaskID()
                          + Path.SEPARATOR
                          + MRConstants.WORKDIR,
                      conf)
                  .toString());

      URI[] archives = DistributedCache.getCacheArchives(conf);
      URI[] files = DistributedCache.getCacheFiles(conf);
      FileStatus fileStatus;
      FileSystem fileSystem;
      Path localPath;
      String baseDir;

      if ((archives != null) || (files != null)) {
        if (archives != null) {
          String[] archivesTimestamps = DistributedCache.getArchiveTimestamps(conf);
          Path[] p = new Path[archives.length];
          for (int i = 0; i < archives.length; i++) {
            fileSystem = FileSystem.get(archives[i], conf);
            fileStatus = fileSystem.getFileStatus(new Path(archives[i].getPath()));
            String cacheId = DistributedCache.makeRelative(archives[i], conf);
            String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId;
            if (lDirAlloc.ifExists(cachePath, conf)) {
              localPath = lDirAlloc.getLocalPathToRead(cachePath, conf);
            } else {
              localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf);
            }
            baseDir = localPath.toString().replace(cacheId, "");
            p[i] =
                DistributedCache.getLocalCache(
                    archives[i],
                    conf,
                    new Path(baseDir),
                    fileStatus,
                    true,
                    Long.parseLong(archivesTimestamps[i]),
                    new Path(workDir.getAbsolutePath()),
                    false);
          }
          DistributedCache.setLocalArchives(conf, stringifyPathArray(p));
        }
        if ((files != null)) {
          String[] fileTimestamps = DistributedCache.getFileTimestamps(conf);
          Path[] p = new Path[files.length];
          for (int i = 0; i < files.length; i++) {
            fileSystem = FileSystem.get(files[i], conf);
            fileStatus = fileSystem.getFileStatus(new Path(files[i].getPath()));
            String cacheId = DistributedCache.makeRelative(files[i], conf);
            String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId;
            if (lDirAlloc.ifExists(cachePath, conf)) {
              localPath = lDirAlloc.getLocalPathToRead(cachePath, conf);
            } else {
              localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf);
            }
            baseDir = localPath.toString().replace(cacheId, "");
            p[i] =
                DistributedCache.getLocalCache(
                    files[i],
                    conf,
                    new Path(baseDir),
                    fileStatus,
                    false,
                    Long.parseLong(fileTimestamps[i]),
                    new Path(workDir.getAbsolutePath()),
                    false);
          }
          DistributedCache.setLocalFiles(conf, stringifyPathArray(p));
        }
        Path localTaskFile = new Path(t.getJobFile());
        FileSystem localFs = FileSystem.getLocal(conf);
        localFs.delete(localTaskFile, true);
        OutputStream out = localFs.create(localTaskFile);
        try {
          conf.writeXml(out);
        } finally {
          out.close();
        }
      }

      if (!prepare()) {
        return;
      }

      String sep = System.getProperty("path.separator");
      StringBuffer classPath = new StringBuffer();
      // start with same classpath as parent process
      classPath.append(System.getProperty("java.class.path"));
      classPath.append(sep);
      if (!workDir.mkdirs()) {
        if (!workDir.isDirectory()) {
          LOG.fatal("Mkdirs failed to create " + workDir.toString());
        }
      }

      String jar = conf.getJar();
      if (jar != null) {
        // if jar exists, it into workDir
        File[] libs = new File(jobCacheDir, "lib").listFiles();
        if (libs != null) {
          for (int i = 0; i < libs.length; i++) {
            classPath.append(sep); // add libs from jar to classpath
            classPath.append(libs[i]);
          }
        }
        classPath.append(sep);
        classPath.append(new File(jobCacheDir, "classes"));
        classPath.append(sep);
        classPath.append(jobCacheDir);
      }

      // include the user specified classpath

      // archive paths
      Path[] archiveClasspaths = DistributedCache.getArchiveClassPaths(conf);
      if (archiveClasspaths != null && archives != null) {
        Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
        if (localArchives != null) {
          for (int i = 0; i < archives.length; i++) {
            for (int j = 0; j < archiveClasspaths.length; j++) {
              if (archives[i].getPath().equals(archiveClasspaths[j].toString())) {
                classPath.append(sep);
                classPath.append(localArchives[i].toString());
              }
            }
          }
        }
      }
      // file paths
      Path[] fileClasspaths = DistributedCache.getFileClassPaths(conf);
      if (fileClasspaths != null && files != null) {
        Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
        if (localFiles != null) {
          for (int i = 0; i < files.length; i++) {
            for (int j = 0; j < fileClasspaths.length; j++) {
              if (files[i].getPath().equals(fileClasspaths[j].toString())) {
                classPath.append(sep);
                classPath.append(localFiles[i].toString());
              }
            }
          }
        }
      }

      classPath.append(sep);
      classPath.append(workDir);
      //  Build exec child jmv args.
      Vector<String> vargs = new Vector<String>(8);
      File jvm = // use same jvm as parent
          new File(new File(System.getProperty("java.home"), "bin"), "java");

      vargs.add(jvm.toString());

      // Add child (task) java-vm options.
      //
      // The following symbols if present in mapred.child.java.opts value are
      // replaced:
      // + @taskid@ is interpolated with value of TaskID.
      // Other occurrences of @ will not be altered.
      //
      // Example with multiple arguments and substitutions, showing
      // jvm GC logging, and start of a passwordless JVM JMX agent so can
      // connect with jconsole and the likes to watch child memory, threads
      // and get thread dumps.
      //
      //  <property>
      //    <name>mapred.child.java.opts</name>
      //    <value>-verbose:gc -Xloggc:/tmp/@[email protected] \
      //           -Dcom.sun.management.jmxremote.authenticate=false \
      //           -Dcom.sun.management.jmxremote.ssl=false \
      //    </value>
      //  </property>
      //
      String javaOpts = conf.get("mapred.child.java.opts", "-Xmx200m");
      javaOpts = javaOpts.replace("@taskid@", taskid.toString());
      String[] javaOptsSplit = javaOpts.split(" ");

      // Add java.library.path; necessary for loading native libraries.
      //
      // 1. To support native-hadoop library i.e. libhadoop.so, we add the
      //    parent processes' java.library.path to the child.
      // 2. We also add the 'cwd' of the task to it's java.library.path to help
      //    users distribute native libraries via the DistributedCache.
      // 3. The user can also specify extra paths to be added to the
      //    java.library.path via mapred.child.java.opts.
      //
      String libraryPath = System.getProperty("java.library.path");
      if (libraryPath == null) {
        libraryPath = workDir.getAbsolutePath();
      } else {
        libraryPath += sep + workDir;
      }
      boolean hasUserLDPath = false;
      for (int i = 0; i < javaOptsSplit.length; i++) {
        if (javaOptsSplit[i].startsWith("-Djava.library.path=")) {
          javaOptsSplit[i] += sep + libraryPath;
          hasUserLDPath = true;
          break;
        }
      }
      if (!hasUserLDPath) {
        vargs.add("-Djava.library.path=" + libraryPath);
      }
      for (int i = 0; i < javaOptsSplit.length; i++) {
        vargs.add(javaOptsSplit[i]);
      }

      // add java.io.tmpdir given by mapred.child.tmp
      String tmp = conf.get("mapred.child.tmp", "./tmp");
      Path tmpDir = new Path(tmp);

      // if temp directory path is not absolute
      // prepend it with workDir.
      if (!tmpDir.isAbsolute()) {
        tmpDir = new Path(workDir.toString(), tmp);
      }
      FileSystem localFs = FileSystem.getLocal(conf);
      if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
        throw new IOException("Mkdirs failed to create " + tmpDir.toString());
      }
      vargs.add("-Djava.io.tmpdir=" + tmpDir.toString());

      // Add classpath.
      vargs.add("-classpath");
      vargs.add(classPath.toString());

      // Setup the log4j prop
      long logSize = TaskLog.getTaskLogLength(conf);
      vargs.add(
          "-Dhadoop.log.dir=" + new File(System.getProperty("hadoop.log.dir")).getAbsolutePath());
      vargs.add("-Dhadoop.root.logger=INFO,TLA");
      vargs.add("-Dhadoop.tasklog.taskid=" + taskid);
      vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);

      if (conf.getProfileEnabled()) {
        if (conf.getProfileTaskRange(t.isMapTask()).isIncluded(t.getPartition())) {
          File prof = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.PROFILE);
          vargs.add(String.format(conf.getProfileParams(), prof.toString()));
        }
      }

      // Add main class and its arguments
      vargs.add(Child.class.getName()); // main of Child
      // pass umbilical address
      InetSocketAddress address = tracker.getTaskTrackerReportAddress();
      vargs.add(address.getAddress().getHostAddress());
      vargs.add(Integer.toString(address.getPort()));
      vargs.add(taskid.toString()); // pass task identifier

      String pidFile = null;
      if (tracker.isTaskMemoryManagerEnabled()) {
        pidFile =
            lDirAlloc
                .getLocalPathForWrite(
                    (TaskTracker.getPidFilesSubdir() + Path.SEPARATOR + taskid), this.conf)
                .toString();
      }

      // set memory limit using ulimit if feasible and necessary ...
      String[] ulimitCmd = Shell.getUlimitMemoryCommand(conf);
      List<String> setup = null;
      if (ulimitCmd != null) {
        setup = new ArrayList<String>();
        for (String arg : ulimitCmd) {
          setup.add(arg);
        }
      }

      // Set up the redirection of the task's stdout and stderr streams
      File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT);
      File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR);
      stdout.getParentFile().mkdirs();
      tracker.getTaskTrackerInstrumentation().reportTaskLaunch(taskid, stdout, stderr);

      Map<String, String> env = new HashMap<String, String>();
      StringBuffer ldLibraryPath = new StringBuffer();
      ldLibraryPath.append(workDir.toString());
      String oldLdLibraryPath = null;
      oldLdLibraryPath = System.getenv("LD_LIBRARY_PATH");
      if (oldLdLibraryPath != null) {
        ldLibraryPath.append(sep);
        ldLibraryPath.append(oldLdLibraryPath);
      }
      env.put("LD_LIBRARY_PATH", ldLibraryPath.toString());
      jvmManager.launchJvm(
          this,
          jvmManager.constructJvmEnv(
              setup, vargs, stdout, stderr, logSize, workDir, env, pidFile, conf));
      synchronized (lock) {
        while (!done) {
          lock.wait();
        }
      }
      tracker.getTaskTrackerInstrumentation().reportTaskEnd(t.getTaskID());
      if (exitCodeSet) {
        if (!killed && exitCode != 0) {
          if (exitCode == 65) {
            tracker.getTaskTrackerInstrumentation().taskFailedPing(t.getTaskID());
          }
          throw new IOException("Task process exit with nonzero status of " + exitCode + ".");
        }
      }
    } catch (FSError e) {
      LOG.fatal("FSError", e);
      try {
        tracker.fsError(t.getTaskID(), e.getMessage());
      } catch (IOException ie) {
        LOG.fatal(t.getTaskID() + " reporting FSError", ie);
      }
    } catch (Throwable throwable) {
      LOG.warn(t.getTaskID() + " Child Error", throwable);
      ByteArrayOutputStream baos = new ByteArrayOutputStream();
      throwable.printStackTrace(new PrintStream(baos));
      try {
        tracker.reportDiagnosticInfo(t.getTaskID(), baos.toString());
      } catch (IOException e) {
        LOG.warn(t.getTaskID() + " Reporting Diagnostics", e);
      }
    } finally {
      try {
        URI[] archives = DistributedCache.getCacheArchives(conf);
        URI[] files = DistributedCache.getCacheFiles(conf);
        if (archives != null) {
          for (int i = 0; i < archives.length; i++) {
            DistributedCache.releaseCache(archives[i], conf);
          }
        }
        if (files != null) {
          for (int i = 0; i < files.length; i++) {
            DistributedCache.releaseCache(files[i], conf);
          }
        }
      } catch (IOException ie) {
        LOG.warn("Error releasing caches : Cache files might not have been cleaned up");
      }
      tracker.reportTaskFinished(t.getTaskID(), false);
      if (t.isMapTask()) {
        tracker.addFreeMapSlot();
      } else {
        tracker.addFreeReduceSlot();
      }
    }
  }

  // Mostly for setting up the symlinks. Note that when we setup the distributed
  // cache, we didn't create the symlinks. This is done on a per task basis
  // by the currently executing task.
  public static void setupWorkDir(JobConf conf) throws IOException {
    File workDir = new File(".").getAbsoluteFile();
    FileUtil.fullyDelete(workDir);
    if (DistributedCache.getSymlink(conf)) {
      URI[] archives = DistributedCache.getCacheArchives(conf);
      URI[] files = DistributedCache.getCacheFiles(conf);
      Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
      Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
      if (archives != null) {
        for (int i = 0; i < archives.length; i++) {
          String link = archives[i].getFragment();
          if (link != null) {
            link = workDir.toString() + Path.SEPARATOR + link;
            File flink = new File(link);
            if (!flink.exists()) {
              FileUtil.symLink(localArchives[i].toString(), link);
            }
          }
        }
      }
      if (files != null) {
        for (int i = 0; i < files.length; i++) {
          String link = files[i].getFragment();
          if (link != null) {
            link = workDir.toString() + Path.SEPARATOR + link;
            File flink = new File(link);
            if (!flink.exists()) {
              FileUtil.symLink(localFiles[i].toString(), link);
            }
          }
        }
      }
    }
    File jobCacheDir = null;
    if (conf.getJar() != null) {
      jobCacheDir = new File(new Path(conf.getJar()).getParent().toString());
    }

    // create symlinks for all the files in job cache dir in current
    // workingdir for streaming
    try {
      DistributedCache.createAllSymlink(conf, jobCacheDir, workDir);
    } catch (IOException ie) {
      // Do not exit even if symlinks have not been created.
      LOG.warn(StringUtils.stringifyException(ie));
    }
    // add java.io.tmpdir given by mapred.child.tmp
    String tmp = conf.get("mapred.child.tmp", "./tmp");
    Path tmpDir = new Path(tmp);

    // if temp directory path is not absolute
    // prepend it with workDir.
    if (!tmpDir.isAbsolute()) {
      tmpDir = new Path(workDir.toString(), tmp);
      FileSystem localFs = FileSystem.getLocal(conf);
      if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
        throw new IOException("Mkdirs failed to create " + tmpDir.toString());
      }
    }
  }

  /** Kill the child process */
  public void kill() {
    killed = true;
    jvmManager.taskKilled(this);
    signalDone();
  }

  public void signalDone() {
    synchronized (lock) {
      done = true;
      lock.notify();
    }
  }

  public void setExitCode(int exitCode) {
    this.exitCodeSet = true;
    this.exitCode = exitCode;
  }
}
/**
 * Created with IntelliJ IDEA. User: ciobi Date: 2013-06-15 Time: 09:56
 *
 * <p>
 */
public class ReaderHandler extends WebAppContext {

  public static final Log LOG = LogFactory.getLog(ReaderHandler.class);
  // ttt1 option that on http only redirects to https, for all paths

  public static final String ACTION_LOGIN = "******";
  public static final String ACTION_SIGNUP = "signup";
  public static final String ACTION_CHANGE_PASSWORD = "******";
  public static final String ACTION_CHANGE_SETTINGS = "change_settings";
  public static final String ACTION_ADD_FEED = "add_feed";
  public static final String ACTION_REMOVE_FEED = "remove_feed";
  public static final String ACTION_UPDATE_FEED_LIST = "update_feed_list"; // for ordering, //ttt2

  public static final String PATH_LOGIN = "******" + ACTION_LOGIN;
  public static final String PATH_CHANGE_PASSWORD = "******" + ACTION_CHANGE_PASSWORD;
  public static final String PATH_CHANGE_SETTINGS = "/" + ACTION_CHANGE_SETTINGS;
  public static final String PATH_SIGNUP = "/" + ACTION_SIGNUP;
  public static final String PATH_ADD_FEED = "/" + ACTION_ADD_FEED;
  public static final String PATH_REMOVE_FEED = "/" + ACTION_REMOVE_FEED;
  public static final String PATH_UPDATE_FEED_LIST = "/" + ACTION_UPDATE_FEED_LIST;
  public static final String PATH_ERROR = "/error";
  public static final String PATH_LOGOUT = "/logout";
  public static final String PATH_SETTINGS = "/settings";
  public static final String PATH_FEEDS = "/feeds";
  public static final String PATH_FEED = "/feed";
  public static final String PATH_ADMIN = "/admin";
  public static final String PATH_FEED_ADMIN = "/feed_admin";
  public static final String PATH_OPEN_ARTICLE =
      "/open_article/"; // !!! it's easier to end this one with a slash

  // params we use to send strings to the JSPs or to get user input in POST, via
  // request.getParameter(), or both
  public static final String PARAM_USER_ID = "userId";
  public static final String PARAM_USER_NAME = "name";
  public static final String PARAM_EMAIL = "email";
  public static final String PARAM_CURRENT_PASSWORD = "******";
  public static final String PARAM_PASSWORD = "******";
  public static final String PARAM_PASSWORD_CONFIRM = "passwordConfirm";
  public static final String PARAM_PATH = "path";
  // public static final String PARAM_ERROR = "error";
  public static final String PARAM_REMEMBER_ACCOUNT = "rememberAccount";
  public static final String PARAM_NEW_FEED_URL = "feedUrl";
  public static final String PARAM_FEED_ID = "feedId";
  public static final String PARAM_ITEMS_PER_PAGE = "itemsPerPage";
  public static final String PARAM_STYLE = "style";
  public static final String PARAM_FEED_DATE_FORMAT = "feedDateFormat";

  // variable names, used to give JSPs access to Java objects in the handler via
  // request.getAttribute(()
  public static final String VAR_USER = "******";
  public static final String VAR_LOGIN_INFO = "loginInfo";
  public static final String VAR_USER_DB = "userDb";
  public static final String VAR_FEED_DB = "feedDb";
  public static final String VAR_ARTICLE_DB = "articleDb";
  public static final String VAR_READ_ARTICLES_COLL_DB = "readArticlesCollDb";

  public static final String BROWSER_ID = "browserId";
  public static final String SESSION_ID = "sessionId";

  private LoginInfo.DB loginInfoDb;
  private User.DB userDb;
  private Feed.DB feedDb;
  private Article.DB articleDb;
  private ReadArticlesColl.DB readArticlesCollDb;

  private UserHelpers userHelpers;

  private boolean isInJar = Utils.isInJar();

  private static class ReaderErrorHandler extends ErrorHandler {
    @Override // !!! note that this gets called for missing pages, but not if exceptions are thrown;
              // exceptions are handled separately
    public void handle(
        String target,
        Request request,
        HttpServletRequest httpServletRequest,
        HttpServletResponse httpServletResponse)
        throws IOException {
      request.setHandled(true);
      httpServletResponse
          .getWriter()
          .println(
              String.format("<h1>Page doesn't exist: %s</h1>", request.getUri().getDecodedPath()));
    }
  }

  private static HashMap<String, String> PATH_MAPPING = new HashMap<>();

  static {
    PATH_MAPPING.put("", "home_page");
    PATH_MAPPING.put(PATH_LOGIN, "login");
    PATH_MAPPING.put(PATH_LOGOUT, "login"); // !!! after logout we get redirected to /login
    PATH_MAPPING.put(PATH_SIGNUP, "signup");
    PATH_MAPPING.put(PATH_ERROR, "error");
    PATH_MAPPING.put(PATH_FEED_ADMIN, "feed_admin");
    PATH_MAPPING.put(PATH_SETTINGS, "settings");
    PATH_MAPPING.put(PATH_FEEDS, "feeds");
    PATH_MAPPING.put(PATH_FEED + "/*", "feed");
    PATH_MAPPING.put(PATH_ADMIN, "admin");
  }

  public ReaderHandler(LowLevelDbAccess lowLevelDbAccess, String webDir) {

    loginInfoDb = new LoginInfo.DB(lowLevelDbAccess);
    userDb = new User.DB(lowLevelDbAccess);
    feedDb = new Feed.DB(lowLevelDbAccess);
    articleDb = new Article.DB(lowLevelDbAccess);
    readArticlesCollDb = new ReadArticlesColl.DB(lowLevelDbAccess);
    userHelpers = new UserHelpers(loginInfoDb, userDb);

    setContextPath("/");

    File warPath = new File(webDir);
    setWar(warPath.getAbsolutePath());

    if (isInJar) {
      for (Map.Entry<String, String> entry : PATH_MAPPING.entrySet()) {
        addPrebuiltJsp(entry.getKey(), "jsp." + entry.getValue().replaceAll("_", "_005f") + "_jsp");
      }
    } else {
      for (Map.Entry<String, String> entry : PATH_MAPPING.entrySet()) {
        addServlet(
            new ServletHolder(new RedirectServlet("/" + entry.getValue() + ".jsp")),
            entry.getKey());
      }
    }

    setErrorHandler(new ReaderErrorHandler());
  }

  private void addPrebuiltJsp(String path, String className) {
    try {
      Class clazz =
          Class.forName(
              className); // ttt2 see if possible to not use this, preferably without doing
                          // redirections like RedirectServlet
      Object obj = clazz.newInstance();
      addServlet(new ServletHolder((Servlet) obj), path);
      LOG.info("Added prebuilt JSP: " + obj.toString());
    } catch (Exception e) {
      LOG.fatal(String.format("Failed to load prebuilt JSP for %s and %s", path, className), e);
    }
  }

  @Override
  public void doHandle(
      String target,
      Request request,
      HttpServletRequest httpServletRequest,
      HttpServletResponse httpServletResponse)
      throws IOException, ServletException {

    LOG.info("handling " + target);

    // !!! doHandle() is called twice for a request when using redirectiion, first time with
    // request.getPathInfo()
    // set to the URI and target set to the path, then with request.getPathInfo() set to null and
    // target set to the .jsp
    try {
      // request.setHandled(true);
      boolean secured;
      if (request.getScheme().equals("https")) {
        secured = true;
      } else if (request.getScheme().equals("http")) {
        secured = false;
      } else {
        httpServletResponse
            .getWriter()
            .println(
                String.format(
                    "<h1>Unknown scheme %s at %s</h1>",
                    request.getScheme(), request.getUri().getDecodedPath()));
        return;
      }

      if (request.getMethod().equals("GET")) {
        if (isInJar || target.endsWith(".jsp")) {
          // !!! when not in jar there's no need to do anything about params if it's not a .jsp,
          // as this will get called again for the corresponding .jsp
          if (prepareForJspGet(target, request, httpServletResponse, secured)) {
            return;
          }
        }
        if (target.startsWith(PATH_OPEN_ARTICLE)) {
          handleOpenArticle(request, httpServletResponse, target);
          return;
        }
        super.doHandle(target, request, httpServletRequest, httpServletResponse);
        LOG.info("handling of " + target + " went to super");

        // httpServletResponse.setDateHeader("Date", System.currentTimeMillis());     //ttt2 review
        // these, probably not use
        // httpServletResponse.setDateHeader("Expires", System.currentTimeMillis() + 60000);

        return;
      }

      if (request.getMethod().equals("POST")) {
        if (request.getUri().getDecodedPath().equals(PATH_LOGIN)) {
          handleLoginPost(request, httpServletResponse, secured);
        } else if (request.getUri().getDecodedPath().equals(PATH_SIGNUP)) {
          handleSignupPost(request, httpServletResponse);
        } else if (request.getUri().getDecodedPath().equals(PATH_CHANGE_PASSWORD)) {
          handleChangePasswordPost(request, httpServletResponse);
        } else if (request.getUri().getDecodedPath().equals(PATH_UPDATE_FEED_LIST)) {
          handleUpdateFeedListPost(request, httpServletResponse);
        } else if (request.getUri().getDecodedPath().equals(PATH_ADD_FEED)) {
          handleAddFeedPost(request, httpServletResponse);
        } else if (request.getUri().getDecodedPath().equals(PATH_REMOVE_FEED)) {
          handleRemoveFeedPost(request, httpServletResponse);
        } else if (request.getUri().getDecodedPath().equals(PATH_CHANGE_SETTINGS)) {
          handleChangeSettingsPost(request, httpServletResponse);
        }
      }

      /*{ // for tests only;
          httpServletResponse.getWriter().println(String.format("<h1>Unable to process request %s</h1>",
                  request.getUri().getDecodedPath()));
          request.setHandled(true);
      }*/
    } catch (Exception e) {
      LOG.error("Error processing request", e);
      try {
        // redirectToError(e.toString(), request, httpServletResponse); //!!! redirectToError leads
        // to infinite loop, probably related to
        // the fact that we get 2 calls for a regular request when redirecting
        httpServletResponse
            .getWriter()
            .println(
                String.format(
                    "<h1>Unable to process request %s</h1>", // ttt1 generate some HTML
                    request.getUri().getDecodedPath()));
        request.setHandled(true);
      } catch (Exception e1) {
        LOG.error("Error redirecting", e1);
      }
    }
  }

  /**
   * Normally sets the path and a few attributes that the JSPs are likely to need. Also verifies the
   * login information. If necessary, just redirects to the login page.
   *
   * @param target
   * @param request
   * @param httpServletResponse
   * @param secured
   * @return true if the request is already handled so the .jsp shouldn't get called
   * @throws Exception
   */
  private boolean prepareForJspGet(
      String target, Request request, HttpServletResponse httpServletResponse, boolean secured)
      throws Exception {

    LoginInfo.SessionInfo sessionInfo = UserHelpers.getSessionInfo(request);

    LOG.info(
        String.format(
            "hndl - %s ; %s; %s ; %s",
            target,
            request.getPathInfo(),
            request.getMethod(),
            secured ? "secured" : "not secured"));

    String path = request.getUri().getDecodedPath();

    boolean redirectToLogin = path.equals(PATH_LOGOUT);
    LoginInfo loginInfo = null;
    if (sessionInfo.isNull()) {
      redirectToLogin = true;
      LOG.info("Null session info. Logging in again.");
    } else {
      loginInfo =
          loginInfoDb.get(
              sessionInfo.browserId,
              sessionInfo.sessionId); // ttt2 use a cache, to avoid going to DB
      if (loginInfo == null || loginInfo.expiresOn < System.currentTimeMillis()) {
        LOG.info("Session has expired. Logging in again. Info: " + loginInfo);
        redirectToLogin = true;
      }
    }

    if (!path.equals(PATH_LOGIN) && !path.equals(PATH_SIGNUP) && !path.equals(PATH_ERROR)) {

      if (redirectToLogin) {
        // ttt2 perhaps store URI, to return to it after login
        logOut(sessionInfo.browserId);
        addLoginParams(request, loginInfo);
        httpServletResponse.sendRedirect(PATH_LOGIN);
        return true;
      }

      User user = userDb.get(loginInfo.userId);
      if (user == null) {
        WebUtils.redirectToError("Unknown user", request, httpServletResponse);
        return true;
      }
      if (!user.active) {
        WebUtils.redirectToError("Account is not active", request, httpServletResponse);
        return true;
      }
      request.setAttribute(VAR_FEED_DB, feedDb);
      request.setAttribute(VAR_USER_DB, userDb);
      request.setAttribute(VAR_ARTICLE_DB, articleDb);
      request.setAttribute(VAR_READ_ARTICLES_COLL_DB, readArticlesCollDb);

      request.setAttribute(VAR_USER, user);
      request.setAttribute(VAR_LOGIN_INFO, loginInfo);

      MultiMap<String> params = new MultiMap<>();
      params.put(PARAM_PATH, path);
      request.setParameters(params);
    }

    if (path.equals(PATH_LOGIN)) {
      addLoginParams(request, loginInfo);
    }
    return false;
  }

  private void handleOpenArticle(
      Request request, HttpServletResponse httpServletResponse, String target) throws Exception {
    try {
      int k1 = target.indexOf('/', 1);
      int k2 = target.indexOf('/', k1 + 1);
      String feedId = target.substring(k1 + 1, k2);
      String strSeq = target.substring(k2 + 1);
      int seq = Integer.parseInt(strSeq);
      Article article = articleDb.get(feedId, seq);
      LoginInfo loginInfo = userHelpers.getLoginInfo(request);
      // ttt2 using the link from a non-authenticated browser causes a NPE; maybe do something
      // better, e.g. sign up
      ReadArticlesColl readArticlesColl = readArticlesCollDb.get(loginInfo.userId, feedId);
      if (readArticlesColl == null) {
        readArticlesColl = new ReadArticlesColl(loginInfo.userId, feedId);
      }
      if (!readArticlesColl.isRead(seq)) {
        readArticlesColl.markRead(seq, Config.getConfig().maxSizeForReadArticles);
        readArticlesCollDb.add(readArticlesColl);
      }
      String s =
          URIUtil.encodePath(article.url)
              .replace("%3F", "?")
              .replace("%23", "#"); // ttt2 see how to do this right
      httpServletResponse.sendRedirect(s);
    } catch (Exception e) {
      WebUtils.showResult(
          String.format("Failed to get article for path %s. %s", target, e),
          "/",
          request,
          httpServletResponse);
    }
  }

  private void handleSignupPost(Request request, HttpServletResponse httpServletResponse)
      throws Exception {
    String userId = request.getParameter(PARAM_USER_ID);
    String userName = request.getParameter(PARAM_USER_NAME);
    String email = request.getParameter(PARAM_EMAIL);
    String stringPassword = request.getParameter(PARAM_PASSWORD);
    String stringPasswordConfirm = request.getParameter(PARAM_PASSWORD_CONFIRM);

    if (!stringPassword.equals(stringPasswordConfirm)) {
      WebUtils.redirectToError(
          "Mismatch between password and password confirmation", request, httpServletResponse);
      return;
    }

    SecureRandom secureRandom = new SecureRandom();
    String salt = "" + secureRandom.nextLong();
    byte[] password = User.computeHashedPassword(stringPassword, salt);
    User user = userDb.get(userId);
    if (user != null) {
      WebUtils.redirectToError(
          "There already exists a user with the ID " + userId, request, httpServletResponse);
      return;
    }

    user =
        new User(
            userId,
            userName,
            password,
            salt,
            email,
            new ArrayList<String>(),
            Config.getConfig().activateAccountsAtCreation,
            false);
    // ttt2 add confirmation by email, captcha, ...
    List<String> fieldErrors = user.checkFields();
    if (!fieldErrors.isEmpty()) {
      StringBuilder bld =
          new StringBuilder("Invalid values when trying to create user with ID ")
              .append(userId)
              .append("<br/>");
      for (String s : fieldErrors) {
        bld.append(s).append("<br/>");
      }
      WebUtils.redirectToError(bld.toString(), request, httpServletResponse);
      return;
    }

    // ttt2 2 clients can add the same userId simultaneously
    userDb.add(user);

    httpServletResponse.sendRedirect("/");
  }

  private void handleChangePasswordPost(Request request, HttpServletResponse httpServletResponse)
      throws Exception {

    LoginInfo loginInfo = userHelpers.getLoginInfo(request);
    if (loginInfo == null) {
      WebUtils.redirectToError("Couldn't determine the current user", request, httpServletResponse);
      return;
    }

    String userId = loginInfo.userId;
    String stringCrtPassword = request.getParameter(PARAM_CURRENT_PASSWORD);
    String stringNewPassword = request.getParameter(PARAM_PASSWORD);
    String stringNewPasswordConfirm = request.getParameter(PARAM_PASSWORD_CONFIRM);

    if (!stringNewPassword.equals(stringNewPasswordConfirm)) {
      showResult(
          "Mismatch between password and password confirmation",
          PATH_SETTINGS,
          request,
          httpServletResponse);
      return;
    }

    User user =
        userDb.get(
            userId); // ttt1 crashes for wrong ID; 2013.07.20 - no longer have an idea what this is
                     // about
    if (user == null) {
      WebUtils.redirectToError("Couldn't find the current user", request, httpServletResponse);
      return;
    }

    if (!user.checkPassword(stringCrtPassword)) {
      showResult("Incorrect current password", PATH_SETTINGS, request, httpServletResponse);
      return;
    }

    SecureRandom secureRandom = new SecureRandom();
    String salt = "" + secureRandom.nextLong();
    byte[] password = User.computeHashedPassword(stringNewPassword, salt);
    user.salt = salt;
    user.password = password;

    // ttt3 2 clients can change the password simultaneously
    userDb.add(user);

    // httpServletResponse.sendRedirect(PATH_SETTINGS);
    showResult("Password changed", PATH_SETTINGS, request, httpServletResponse);
  }

  private void handleChangeSettingsPost(Request request, HttpServletResponse httpServletResponse)
      throws Exception {

    LoginInfo loginInfo = userHelpers.getLoginInfo(request);
    if (loginInfo == null) {
      WebUtils.redirectToError("Couldn't determine the current user", request, httpServletResponse);
      return;
    }

    String stringItemsPerPage = request.getParameter(PARAM_ITEMS_PER_PAGE);
    try {
      loginInfo.itemsPerPage = Integer.parseInt(stringItemsPerPage);
    } catch (Exception e) {
      showResult(
          "Error trying to set the items per page. Expected integer value but got "
              + stringItemsPerPage,
          PATH_SETTINGS,
          request,
          httpServletResponse);
      return;
    }
    loginInfo.style = request.getParameter(PARAM_STYLE);
    loginInfo.feedDateFormat =
        request.getParameter(PARAM_FEED_DATE_FORMAT); // ttt2 validate, better in JSP

    loginInfoDb.add(loginInfo);

    // httpServletResponse.sendRedirect(PATH_SETTINGS);
    showResult("Settings changed", "/", request, httpServletResponse);
  }

  private void handleUpdateFeedListPost(Request request, HttpServletResponse httpServletResponse)
      throws Exception {
    LOG.info("updating feed list"); // ttt2 implement
    httpServletResponse.sendRedirect(PATH_FEED_ADMIN);
  }

  private void handleAddFeedPost(Request request, HttpServletResponse httpServletResponse)
      throws Exception {
    LOG.info("adding feed");
    User user = userHelpers.getUser(request);

    try {
      if (user == null) {
        LOG.error("User not found");
        return;
      }

      String url = request.getParameter(PARAM_NEW_FEED_URL);
      // ttt1 add some validation; probably best try to actually get data, set the title, ...
      if (url == null || url.equals("")) {
        LOG.error("New feed not specified");
        // ttt1 show some error
        return;
      }

      MessageDigest digest = MessageDigest.getInstance("MD5");
      String feedId = PrintUtils.byteArrayAsUrlString(digest.digest(url.getBytes("UTF-8")));
      feedId = feedId.substring(0, Config.getConfig().feedIdSize);

      Feed feed = feedDb.get(feedId);
      if (feed == null) {
        feed = new Feed(feedId, url);
        feedDb.add(feed);
      }

      if (user.feedIds.contains(feedId)) {
        LOG.error(String.format("Trying to add existing feed %s to user %s", feedId, user));
      } else {
        user.feedIds.add(feedId);
        userDb.updateFeeds(user);
      }
    } finally {
      httpServletResponse.sendRedirect(PATH_FEED_ADMIN);
    }
  }

  private void handleRemoveFeedPost(Request request, HttpServletResponse httpServletResponse)
      throws Exception {
    LOG.info("removing feed");
    User user = userHelpers.getUser(request);

    try {
      if (user == null) {
        LOG.error("User not found");
        return;
      }

      String feedId = request.getParameter(PARAM_FEED_ID);

      LOG.info(String.format("Removing feed %s for user %s", feedId, user));

      // ttt1 add some validation; probably best try to actually get data, set the title, ...
      if (feedId == null || feedId.equals("")) {
        LOG.error("feed not specified");
        // ttt1 show some error
        return;
      }

      if (user.feedIds.remove(
          feedId)) { // ttt2 clean up the global feed table; that's probably better done if nobody
                     // accesses a feed for 3 months or so
        userDb.updateFeeds(user);
        LOG.info(String.format("Removed feed %s for user %s", feedId, user));
      } else {
        LOG.info(String.format("No feed found with ID %s for user %s", feedId, user));
      }
    } finally {
      httpServletResponse.sendRedirect(PATH_FEED_ADMIN);
    }
  }

  private void handleLoginPost(
      Request request, HttpServletResponse httpServletResponse, boolean secured) throws Exception {
    String userId = request.getParameter(PARAM_USER_ID);
    String password = request.getParameter(PARAM_PASSWORD);
    String rememberAccountStr = request.getParameter(PARAM_REMEMBER_ACCOUNT);
    boolean rememberAccount = Boolean.parseBoolean(rememberAccountStr);
    LoginInfo.SessionInfo sessionInfo = UserHelpers.getSessionInfo(request);

    logOut(sessionInfo.browserId);

    User user = userDb.get(userId);
    if (user == null) {
      WebUtils.redirectToError("User " + userId + " not found", request, httpServletResponse);
      return;
    }

    if (!user.checkPassword(password)) {
      WebUtils.redirectToError("Invalid password", request, httpServletResponse);
      return;
    }

    if (!user.active) {
      WebUtils.redirectToError(
          "Account for User " + userId + " needs to be activated", request, httpServletResponse);
      return;
    }

    LOG.info("Logged in user " + userId);

    sessionInfo.sessionId = null;
    if (sessionInfo.browserId == null) {
      sessionInfo.browserId = getRandomId();
    } else {
      for (LoginInfo loginInfo : loginInfoDb.getLoginsForBrowser(sessionInfo.browserId)) {
        if (userId.equals(loginInfo.userId)) {
          sessionInfo.sessionId = loginInfo.sessionId;
          break;
        }
      }
    }

    long expireOn = System.currentTimeMillis() + Config.getConfig().loginExpireInterval;
    if (sessionInfo.sessionId == null) {
      sessionInfo.sessionId = getRandomId();
      Config config = Config.getConfig();
      loginInfoDb.add(
          new LoginInfo(
              sessionInfo.browserId,
              sessionInfo.sessionId,
              userId,
              expireOn,
              rememberAccount,
              config.defaultStyle,
              config.defaultItemsPerPage,
              config.defaultFeedDateFormat));
      LOG.info(String.format("Logging in in a new session. User: %s", user));
    } else {
      loginInfoDb.updateExpireTime(sessionInfo.browserId, sessionInfo.sessionId, expireOn);
      LOG.info(String.format("Logging in in an existing session. User: %s", user));
    }

    WebUtils.saveCookies(
        httpServletResponse, secured, sessionInfo.browserId, sessionInfo.sessionId);

    httpServletResponse.sendRedirect("/");
  }

  private String getRandomId() {
    SecureRandom secureRandom = new SecureRandom();
    return "" + secureRandom.nextLong();
  }

  private void addLoginParams(Request request, LoginInfo loginInfo) {
    MultiMap<String> params = new MultiMap<>();
    if (loginInfo != null && loginInfo.rememberAccount) {
      params.put(PARAM_USER_ID, loginInfo.userId);
    }
    request.setParameters(params);
  }

  private void logOut(String browserId) throws Exception {
    // ttt2 the right way to do it is to go through all the sessions of the current browser, which
    // would require a new field and a new index;
    // not sure if it's worth it, but this would work: A logs in, forgets to log out, B delets the
    // cookies, logs in, A sees B is logged in, then B
    // restores the cookies and uses A's account
    if (browserId == null) {
      return;
    }

    List<LoginInfo> loginInfos = loginInfoDb.getLoginsForBrowser(browserId);
    long expireTarget = System.currentTimeMillis() - Utils.ONE_DAY;
    for (LoginInfo loginInfo : loginInfos) {
      if (loginInfo.expiresOn <= expireTarget) {
        LOG.info(String.format("LoginInfo %s is enough in the past", loginInfo));
      } else {
        LOG.info(String.format("Logging out: %s", loginInfo));
        loginInfoDb.updateExpireTime(browserId, loginInfo.sessionId, expireTarget);
      }
    }
  }

  public static class FeedInfo {
    public String feedId;
    public int maxSeq;

    public FeedInfo(String feedId, int maxSeq) {
      this.feedId = feedId;
      this.maxSeq = maxSeq;
    }
  }

  // !!! IDEA reports this as unused, but it is called from JSP
  public static FeedInfo getFeedInfo(String feedPath) {
    if (feedPath.startsWith(PATH_FEED + "/")) {
      try {
        if (feedPath.endsWith("/")) {
          feedPath = feedPath.substring(0, feedPath.length() - 1);
        }
        int k = PATH_FEED.length() + 1;
        int p = feedPath.indexOf('/', k);
        return p >= 0
            ? new FeedInfo(feedPath.substring(k, p), Integer.parseInt(feedPath.substring(p + 1)))
            : new FeedInfo(feedPath.substring(k), -1);
      } catch (Exception e) {
        LOG.error("Exception trying to parse the feed info", e);
      }
    }

    LOG.error("Invalid path from feed: " + feedPath);
    return new FeedInfo("INVALID", -1);
  }

  // !!! IDEA reports this as unused, but it is called from JSP
  public static String getStyle(LoginInfo loginInfo) {
    StringBuilder bld = new StringBuilder();
    bld.append("<style media=\"screen\" type=\"text/css\">\n\n");
    if (loginInfo == null) {
      bld.append(Config.getConfig().defaultStyle);
    } else {
      bld.append(loginInfo.style); // ttt3 detect broken styles and return default
    }
    bld.append("</style>\n");
    return bld.toString();
  }

  /*    private void jspCodeCheck() throws Exception {
      Article.DB articleDb;
      Request request;
      String path = "";

      String feedId = ReaderHandler.getFeedId(path);
      int maxSeq = ReaderHandler.getSeq(path);

      Feed.DB feedDb = (Feed.DB)request.getAttribute(ReaderHandler.VAR_FEED_DB);

      Feed feed = feedDb.get(feedId);
      if (feed == null) {
          out.println("Feed " + feedId + " not found");
      } else {
          if (maxSeq == -1) {
              maxSeq = feed.maxSeq;
          }
          if (maxSeq < 0) {
              out.println("Feed " + feedId + " is empty");
          } else {
              ++maxSeq;
              LoginInfo loginInfo = (LoginInfo)request.getAttribute(ReaderHandler.VAR_LOGIN_INFO);
              int minSeq = Math.max(maxSeq - loginInfo.itemsPerPage, 0);
              List<Article> articles = articleDb.get(feedId, minSeq, maxSeq);
              for (Article article : articles) {
                  out.println("<a href=\"" + article.url + "\">" + article.title + "</a><br/>");
              }
          }
      }

  }
  //*/
}
public class IdentitySchema {

  private static final String IDENTITY_TABLE_PREFIX = "JBPM_ID_";

  Configuration configuration = null;
  Properties properties = null;
  Dialect dialect = null;
  Mapping mapping = null;
  String[] createSql = null;
  String[] dropSql = null;
  String[] cleanSql = null;

  ConnectionProvider connectionProvider = null;
  Connection connection = null;
  Statement statement = null;

  public IdentitySchema(Configuration configuration) {
    this.configuration = configuration;
    this.properties = configuration.getProperties();
    this.dialect = Dialect.getDialect(properties);
    try {
      // get the mapping field via reflection :-(
      Field mappingField = Configuration.class.getDeclaredField("mapping");
      mappingField.setAccessible(true);
      this.mapping = (Mapping) mappingField.get(configuration);
    } catch (Exception e) {
      throw new RuntimeException("couldn't get the hibernate mapping", e);
    }
  }

  // scripts lazy initializations /////////////////////////////////////////////

  public String[] getCreateSql() {
    if (createSql == null) {
      createSql = configuration.generateSchemaCreationScript(dialect);
    }
    return createSql;
  }

  public String[] getDropSql() {
    if (dropSql == null) {
      dropSql = configuration.generateDropSchemaScript(dialect);
    }
    return dropSql;
  }

  public String[] getCleanSql() {
    if (cleanSql == null) {
      // loop over all foreign key constraints
      List dropForeignKeysSql = new ArrayList();
      List createForeignKeysSql = new ArrayList();
      Iterator iter = configuration.getTableMappings();
      while (iter.hasNext()) {
        Table table = (Table) iter.next();
        if (table.isPhysicalTable()) {
          Iterator subIter = table.getForeignKeyIterator();
          while (subIter.hasNext()) {
            ForeignKey fk = (ForeignKey) subIter.next();
            if (fk.isPhysicalConstraint()) {
              // collect the drop key constraint
              dropForeignKeysSql.add(
                  fk.sqlDropString(
                      dialect,
                      properties.getProperty(Environment.DEFAULT_CATALOG),
                      properties.getProperty(Environment.DEFAULT_SCHEMA)));
              createForeignKeysSql.add(
                  fk.sqlCreateString(
                      dialect,
                      mapping,
                      properties.getProperty(Environment.DEFAULT_CATALOG),
                      properties.getProperty(Environment.DEFAULT_SCHEMA)));
            }
          }
        }
      }

      List deleteSql = new ArrayList();
      iter = configuration.getTableMappings();
      while (iter.hasNext()) {
        Table table = (Table) iter.next();
        deleteSql.add("delete from " + table.getName());
      }

      List cleanSqlList = new ArrayList();
      cleanSqlList.addAll(dropForeignKeysSql);
      cleanSqlList.addAll(deleteSql);
      cleanSqlList.addAll(createForeignKeysSql);

      cleanSql = (String[]) cleanSqlList.toArray(new String[cleanSqlList.size()]);
    }
    return cleanSql;
  }

  // runtime table detection //////////////////////////////////////////////////

  public boolean hasIdentityTables() {
    return (getIdentityTables().size() > 0);
  }

  public List getIdentityTables() {
    // delete all the data in the jbpm tables
    List jbpmTableNames = new ArrayList();
    try {
      createConnection();
      ResultSet resultSet = connection.getMetaData().getTables("", "", null, null);
      while (resultSet.next()) {
        String tableName = resultSet.getString("TABLE_NAME");
        if ((tableName != null)
            && (tableName.length() > 5)
            && (IDENTITY_TABLE_PREFIX.equalsIgnoreCase(tableName.substring(0, 5)))) {
          jbpmTableNames.add(tableName);
        }
      }
    } catch (SQLException e) {
      throw new RuntimeException("couldn't get the jbpm table names");
    } finally {
      closeConnection();
    }
    return jbpmTableNames;
  }

  // script execution methods /////////////////////////////////////////////////

  public void dropSchema() {
    execute(getDropSql());
  }

  public void createSchema() {
    execute(getCreateSql());
  }

  public void cleanSchema() {
    execute(getCleanSql());
  }

  public void saveSqlScripts(String dir, String prefix) {
    try {
      new File(dir).mkdirs();
      saveSqlScript(dir + "/" + prefix + ".drop.sql", getDropSql());
      saveSqlScript(dir + "/" + prefix + ".create.sql", getCreateSql());
      saveSqlScript(dir + "/" + prefix + ".clean.sql", getCleanSql());
      new SchemaExport(configuration)
          .setDelimiter(getSqlDelimiter())
          .setOutputFile(dir + "/" + prefix + ".drop.create.sql")
          .create(true, false);
    } catch (Exception e) {
      throw new RuntimeException("couldn't generate scripts", e);
    }
  }

  // main /////////////////////////////////////////////////////////////////////

  public static void main(String[] args) {
    try {
      if ((args != null) && (args.length == 1) && ("create".equalsIgnoreCase(args[0]))) {
        new IdentitySchema(IdentitySessionFactory.createConfiguration()).createSchema();
      } else if ((args != null) && (args.length == 1) && ("drop".equalsIgnoreCase(args[0]))) {
        new IdentitySchema(IdentitySessionFactory.createConfiguration()).dropSchema();
      } else if ((args != null) && (args.length == 1) && ("clean".equalsIgnoreCase(args[0]))) {
        new IdentitySchema(IdentitySessionFactory.createConfiguration()).cleanSchema();
      } else if ((args != null) && (args.length == 3) && ("scripts".equalsIgnoreCase(args[0]))) {
        new IdentitySchema(IdentitySessionFactory.createConfiguration())
            .saveSqlScripts(args[1], args[2]);
      } else {
        System.err.println("syntax: JbpmSchema create");
        System.err.println("syntax: JbpmSchema drop");
        System.err.println("syntax: JbpmSchema clean");
        System.err.println("syntax: JbpmSchema scripts <dir> <prefix>");
      }
    } catch (Exception e) {
      e.printStackTrace();
      throw new RuntimeException(e);
    }
  }

  private void saveSqlScript(String fileName, String[] sql) throws FileNotFoundException {
    FileOutputStream fileOutputStream = new FileOutputStream(fileName);
    PrintStream printStream = new PrintStream(fileOutputStream);
    for (int i = 0; i < sql.length; i++) {
      printStream.println(sql[i] + getSqlDelimiter());
    }
  }

  // sql script execution /////////////////////////////////////////////////////

  public void execute(String[] sqls) {
    String sql = null;
    String showSqlText = properties.getProperty("hibernate.show_sql");
    boolean showSql = ("true".equalsIgnoreCase(showSqlText));

    try {
      createConnection();
      statement = connection.createStatement();

      for (int i = 0; i < sqls.length; i++) {
        sql = sqls[i];
        String delimitedSql = sql + getSqlDelimiter();

        if (showSql) log.debug(delimitedSql);
        statement.executeUpdate(delimitedSql);
      }

    } catch (SQLException e) {
      e.printStackTrace();
      throw new RuntimeException("couldn't execute sql '" + sql + "'", e);
    } finally {
      closeConnection();
    }
  }

  private void closeConnection() {
    try {
      if (statement != null) statement.close();
      if (connection != null) {
        JDBCExceptionReporter.logWarnings(connection.getWarnings());
        connection.clearWarnings();
        connectionProvider.closeConnection(connection);
        connectionProvider.close();
      }
    } catch (Exception e) {
      System.err.println("Could not close connection");
      e.printStackTrace();
    }
  }

  private void createConnection() throws SQLException {
    connectionProvider = ConnectionProviderFactory.newConnectionProvider(properties);
    connection = connectionProvider.getConnection();
    if (!connection.getAutoCommit()) {
      connection.commit();
      connection.setAutoCommit(true);
    }
  }

  public Properties getProperties() {
    return properties;
  }

  // sql delimiter ////////////////////////////////////////////////////////////

  private static String sqlDelimiter = null;

  private synchronized String getSqlDelimiter() {
    if (sqlDelimiter == null) {
      sqlDelimiter = properties.getProperty("jbpm.sql.delimiter", ";");
    }
    return sqlDelimiter;
  }

  // logger ///////////////////////////////////////////////////////////////////

  private static final Log log = LogFactory.getLog(IdentitySchema.class);
}
Exemple #17
0
/**
 * MDC Connector
 *
 * @author yjiang
 */
public abstract class MDCConnector extends IoHandlerAdapter {

  static final Log log = LogFactory.getLog(MDCConnector.class);

  /** the the max size of a packet, 32KB */
  static int MAX_SIZE = MDCServer.MAX_SIZE;

  protected Selector selector;
  protected IoConnector connector;
  protected static Configuration _conf;

  protected static boolean inited = false;

  /** Close. */
  public void close() {
    if (selector != null) {
      selector.wakeup();
      try {
        selector.close();
      } catch (IOException e1) {
        log.warn("close selector fails", e1);
      } finally {
        selector = null;
      }
    }

    if (connector != null) {
      connector.dispose();
      connector = null;
    }
  }

  /** Instantiates a new MDC connector. */
  protected MDCConnector() {}

  /** Inits the. */
  public static synchronized void init() {
    if (inited) {
      return;
    }

    _conf = Config.getConfig();

    /** initialize app command */
    Command.init();

    /** initialize the RSA key, hardcode 2048 bits */
    TConn.pub_key = SystemConfig.s("pub_key", null);
    if (TConn.pub_key == null) {
      Key k = RSA.generate(2048);
      TConn.pri_key = k.pri_key;
      TConn.pub_key = k.pub_key;

      /** set back in database */
      SystemConfig.setConfig("pri_key", TConn.pri_key);
      SystemConfig.setConfig("pub_key", TConn.pub_key);
    } else {

      /** get from the database */
      TConn.pri_key = SystemConfig.s("pri_key", null);
    }

    inited = true;
  }

  /**
   * Service.
   *
   * @param o the o
   * @param session the session
   */
  void service(IoBuffer o, IoSession session) {
    try {
      // System.out.println(o.remaining() + "/" + o.capacity());

      session.setAttribute("last", System.currentTimeMillis());

      SimpleIoBuffer in = (SimpleIoBuffer) session.getAttribute("buf");
      if (in == null) {
        in = SimpleIoBuffer.create(4096);
        session.setAttribute("buf", in);
      }
      byte[] data = new byte[o.remaining()];
      o.get(data);
      in.append(data);

      // log.debug("recv: " + data.length + ", " +
      // session.getRemoteAddress());

      while (in.length() > 5) {
        in.mark();
        /**
         * Byte 1: head of the package<br>
         * bit 7-6: "01", indicator of MDC<br>
         * bit 5: encrypt indicator, "0": no; "1": encrypted<br>
         * bit 4: zip indicator, "0": no, "1": ziped<br>
         * bit 0-3: reserved<br>
         * Byte 2-5: length of data<br>
         * Byte[…]: data array<br>
         */
        byte head = in.read();
        /** test the head indicator, if not correct close it */
        if ((head & 0xC0) != 0x40) {
          log.info("flag is not correct! flag:" + head + ",from: " + session.getRemoteAddress());

          session.close(true);
          return;
        }

        int len = in.getInt();

        if (len <= 0 || len > MAX_SIZE) {
          log.error(
              "mdcconnector.Wrong lendth: "
                  + len
                  + "/"
                  + MAX_SIZE
                  + " - "
                  + session.getRemoteAddress());
          session.close(true);
          break;
        }

        if (in.length() < len) {
          in.reset();
          break;
        } else {
          // do it
          // log.info("stub.package.size: " + len);

          byte[] b = new byte[len];
          in.read(b);

          if (TConn.DEBUG) {
            log.debug("recv: " + Bean.toString(b));
          }

          /** test the zip flag */
          if ((head & 0x10) > 0) {
            b = Zip.unzip(b);
          }

          final TConn d = (TConn) session.getAttribute("conn");
          if (d != null) {
            /** test the encrypted flag */
            if ((head & 0x20) > 0) {
              b = DES.decode(b, d.deskey);
            }

            final byte[] bb = b;
            /** test if the packet is for mdc or app */
            new WorkerTask() {

              @Override
              public void onExecute() {
                d.process(bb);
              }
            }.schedule(0);

            session.setAttribute("last", System.currentTimeMillis());
          }
        }
      }
    } catch (Throwable e) {
      log.error("closing stub: " + session.getRemoteAddress(), e);
      session.close(true);
    }
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#sessionCreated(org.apache
   * .mina.core.session.IoSession)
   */
  public void sessionCreated(IoSession session) throws Exception {
    String remote = session.getRemoteAddress().toString();
    log.info("stub created:" + remote);

    /** check the allow ip */
    if (TConn.ALLOW_IP == null || "*".equals(TConn.ALLOW_IP) || remote.matches(TConn.ALLOW_IP)) {
      TConn d = new TConn(session);
      session.setAttribute("conn", d);
    } else {
      log.warn("deny the connection:" + remote + ", allow ip:" + TConn.ALLOW_IP);
      session.close(true);
    }
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#sessionClosed(org.apache
   * .mina.core.session.IoSession)
   */
  public void sessionClosed(IoSession session) throws Exception {
    log.debug("closed stub: " + session.getRemoteAddress());
    TConn d = (TConn) session.getAttribute("conn");
    if (d != null) {
      d.close();
    }
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#sessionIdle(org.apache.
   * mina.core.session.IoSession, org.apache.mina.core.session.IdleStatus)
   */
  public void sessionIdle(IoSession session, IdleStatus status) throws Exception {
    if (IdleStatus.BOTH_IDLE.equals(status)) {
      Long l = (Long) session.getAttribute("last");
      if (l != null && System.currentTimeMillis() - l > 60 * 1000) {
        session.close(true);
      }
    }
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#messageReceived(org.apache
   * .mina.core.session.IoSession, java.lang.Object)
   */
  public void messageReceived(IoSession session, Object message) throws Exception {
    // System.out.println(message);
    if (message instanceof IoBuffer) {
      service((IoBuffer) message, session);
    }
  }

  private static MDCConnector tcpconnector;
  private static MDCConnector udpconnector;

  /**
   * @param host
   * @param port
   * @return TConn
   */
  public static synchronized TConn connectByTcp(String host, int port) {
    return connectByTcp(host, port, X.AMINUTE);
  }

  /**
   * Connect by tcp.
   *
   * @param host the host
   * @param port the port
   * @return the t conn
   */
  public static synchronized TConn connectByTcp(String host, int port, long timeout) {

    TimeStamp t = TimeStamp.create();

    try {
      if (tcpconnector == null) {
        tcpconnector = new TDCConnector();
      }

      tcpconnector.connector.setConnectTimeoutMillis(timeout);

      ConnectFuture connFuture = tcpconnector.connector.connect(new InetSocketAddress(host, port));

      connFuture.awaitUninterruptibly(timeout);
      IoSession session = connFuture.getSession();

      TConn c = new TConn(session);

      session.setAttribute("conn", c);
      return c;
    } catch (Exception e) {
      log.error(
          "error, [" + host + ":" + port + "], cost: " + t.past() + "ms, timeout=" + timeout, e);
    }

    return null;
  }

  /**
   * Connect by udp.
   *
   * @param host the host
   * @param port the port
   * @return the t conn
   */
  public static synchronized TConn connectByUdp(String host, int port) {
    try {
      if (udpconnector == null) {
        udpconnector = new UDCConnector();
      }

      ConnectFuture connFuture = udpconnector.connector.connect(new InetSocketAddress(host, port));
      connFuture.awaitUninterruptibly();
      IoSession session = connFuture.getSession();

      TConn c = new TConn(session);

      session.setAttribute("conn", c);
      return c;
    } catch (Exception e) {
      log.error("[" + host + ":" + port + "]", e);
    }
    return null;
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#exceptionCaught(org.apache
   * .mina.core.session.IoSession, java.lang.Throwable)
   */
  @Override
  public void exceptionCaught(IoSession session, Throwable cause) throws Exception {
    log.error(cause.getMessage(), cause);
  }
}
/** @author Johan Lindquist */
public class BaseController {
  private Log _log = LogFactory.getLog(BaseController.class);

  @Autowired protected JahSpotifyService _jahSpotifyService;

  @Value(value = "${jahspotify.web.controller.default-media-expires-duration}")
  private int _defaultMediaExpirationTime;

  protected void writeResponse(
      final HttpServletResponse httpServletResponse,
      final SimpleStatusResponse simpleStatusResponse) {
    Gson gson = new Gson();
    try {
      httpServletResponse.setContentType("application/json; charset=utf-8");
      _log.debug("Serializing: " + simpleStatusResponse);
      final PrintWriter writer = httpServletResponse.getWriter();
      gson.toJson(simpleStatusResponse.getResponseStatus(), writer);
      writer.flush();
      writer.close();
    } catch (Exception e) {
      _log.error("Error while writing response: " + e.getMessage(), e);
    }
  }

  protected void writeMediaNotReadable(final HttpServletResponse httpServletResponse) {
    SimpleStatusResponse simpleStatusResponse = new SimpleStatusResponse();
    simpleStatusResponse.setResponseStatus(ResponseStatus.RESOURCE_NOT_FOUND);
    writeResponse(httpServletResponse, simpleStatusResponse);
  }

  protected void writeResponseGeneric(
      final HttpServletResponse httpServletResponse, final Object object) {
    this.writeResponseGenericWithDate(httpServletResponse, null, object);
  }

  protected void writeResponseGenericWithDate(
      final HttpServletResponse httpServletResponse, final Date lastModified, final Object object) {
    writeResponseGenericWithDate(
        httpServletResponse, lastModified, _defaultMediaExpirationTime, object);
  }

  protected void writeResponseGenericWithDate(
      final HttpServletResponse httpServletResponse,
      final Date lastModified,
      final int expirationTime,
      final Object object) {
    Gson gson = new Gson();
    try {
      httpServletResponse.setContentType("application/json; charset=utf-8");
      if (lastModified != null) {
        httpServletResponse.addHeader("Expires", createDateHeader(expirationTime));
        httpServletResponse.addHeader("Last-Modified", toHttpDate(lastModified));
      }
      _log.debug("Serializing: " + object);
      final PrintWriter writer = httpServletResponse.getWriter();
      gson.toJson(object, writer);
      writer.flush();
      writer.close();
    } catch (Exception e) {
      _log.error("Error while writing response: " + e.getMessage(), e);
    }
  }

  protected String createDateHeader(int expires) {
    final Calendar utc = GregorianCalendar.getInstance(TimeZone.getTimeZone("UTC"));
    utc.add(Calendar.SECOND, expires);
    return toHttpDate(utc.getTime());
  }

  protected String toHttpDate(Date date) {
    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z");
    return simpleDateFormat.format(date);
  }

  protected Link retrieveLink(final HttpServletRequest httpServletRequest) {
    String uri =
        httpServletRequest
            .getRequestURI()
            .substring(httpServletRequest.getRequestURI().lastIndexOf("/") + 1);
    _log.debug("Extracted URI: " + uri);
    return Link.create(uri);
  }

  protected <T> T readRequest(final HttpServletRequest httpServletRequest, final Class<T> classOfT)
      throws IOException {
    final BufferedReader br = new BufferedReader(httpServletRequest.getReader());
    Gson gson = new Gson();
    return gson.fromJson(br, classOfT);
  }

  public void writeErrorResponse(final HttpServletResponse httpServletResponse, final Exception e) {
    SimpleStatusResponse simpleStatusResponse = new SimpleStatusResponse();
    simpleStatusResponse.setResponseStatus(ResponseStatus.INTERNAL_ERROR);
    writeResponse(httpServletResponse, simpleStatusResponse);
  }
}
Exemple #19
0
/**
 * A simple RPC mechanism.
 *
 * <p>A <i>protocol</i> is a Java interface. All parameters and return types must be one of:
 *
 * <ul>
 *   <li>a primitive type, <code>boolean</code>, <code>byte</code>, <code>char</code>, <code>short
 *       </code>, <code>int</code>, <code>long</code>, <code>float</code>, <code>double</code>, or
 *       <code>void</code>; or
 *   <li>a {@link String}; or
 *   <li>a {@link Writable}; or
 *   <li>an array of the above types
 * </ul>
 *
 * All methods in the protocol should throw only IOException. No field data of the protocol instance
 * is transmitted.
 */
@InterfaceAudience.LimitedPrivate(value = {"Common", "HDFS", "MapReduce", "Yarn"})
@InterfaceStability.Evolving
public class RPC {
  static final int RPC_SERVICE_CLASS_DEFAULT = 0;

  public enum RpcKind {
    RPC_BUILTIN((short) 1), // Used for built in calls by tests
    RPC_WRITABLE((short) 2), // Use WritableRpcEngine
    RPC_PROTOCOL_BUFFER((short) 3); // Use ProtobufRpcEngine
    static final short MAX_INDEX = RPC_PROTOCOL_BUFFER.value; // used for array size
    public final short value; // TODO make it private

    RpcKind(short val) {
      this.value = val;
    }
  }

  interface RpcInvoker {
    /**
     * Process a client call on the server side
     *
     * @param server the server within whose context this rpc call is made
     * @param protocol - the protocol name (the class of the client proxy used to make calls to the
     *     rpc server.
     * @param rpcRequest - deserialized
     * @param receiveTime time at which the call received (for metrics)
     * @return the call's return
     * @throws IOException
     */
    public Writable call(Server server, String protocol, Writable rpcRequest, long receiveTime)
        throws Exception;
  }

  static final Log LOG = LogFactory.getLog(RPC.class);

  /**
   * Get all superInterfaces that extend VersionedProtocol
   *
   * @param childInterfaces
   * @return the super interfaces that extend VersionedProtocol
   */
  static Class<?>[] getSuperInterfaces(Class<?>[] childInterfaces) {
    List<Class<?>> allInterfaces = new ArrayList<Class<?>>();

    for (Class<?> childInterface : childInterfaces) {
      if (VersionedProtocol.class.isAssignableFrom(childInterface)) {
        allInterfaces.add(childInterface);
        allInterfaces.addAll(Arrays.asList(getSuperInterfaces(childInterface.getInterfaces())));
      } else {
        LOG.warn(
            "Interface "
                + childInterface
                + " ignored because it does not extend VersionedProtocol");
      }
    }
    return allInterfaces.toArray(new Class[allInterfaces.size()]);
  }

  /**
   * Get all interfaces that the given protocol implements or extends which are assignable from
   * VersionedProtocol.
   */
  static Class<?>[] getProtocolInterfaces(Class<?> protocol) {
    Class<?>[] interfaces = protocol.getInterfaces();
    return getSuperInterfaces(interfaces);
  }

  /**
   * Get the protocol name. If the protocol class has a ProtocolAnnotation, then get the protocol
   * name from the annotation; otherwise the class name is the protocol name.
   */
  public static String getProtocolName(Class<?> protocol) {
    if (protocol == null) {
      return null;
    }
    ProtocolInfo anno = protocol.getAnnotation(ProtocolInfo.class);
    return (anno == null) ? protocol.getName() : anno.protocolName();
  }

  /**
   * Get the protocol version from protocol class. If the protocol class has a ProtocolAnnotation,
   * then get the protocol name from the annotation; otherwise the class name is the protocol name.
   */
  public static long getProtocolVersion(Class<?> protocol) {
    if (protocol == null) {
      throw new IllegalArgumentException("Null protocol");
    }
    long version;
    ProtocolInfo anno = protocol.getAnnotation(ProtocolInfo.class);
    if (anno != null) {
      version = anno.protocolVersion();
      if (version != -1) return version;
    }
    try {
      Field versionField = protocol.getField("versionID");
      versionField.setAccessible(true);
      return versionField.getLong(protocol);
    } catch (NoSuchFieldException ex) {
      throw new RuntimeException(ex);
    } catch (IllegalAccessException ex) {
      throw new RuntimeException(ex);
    }
  }

  private RPC() {} // no public ctor

  // cache of RpcEngines by protocol
  private static final Map<Class<?>, RpcEngine> PROTOCOL_ENGINES =
      new HashMap<Class<?>, RpcEngine>();

  private static final String ENGINE_PROP = "rpc.engine";

  /**
   * Set a protocol to use a non-default RpcEngine.
   *
   * @param conf configuration to use
   * @param protocol the protocol interface
   * @param engine the RpcEngine impl
   */
  public static void setProtocolEngine(Configuration conf, Class<?> protocol, Class<?> engine) {
    conf.setClass(ENGINE_PROP + "." + protocol.getName(), engine, RpcEngine.class);
  }

  // return the RpcEngine configured to handle a protocol
  static synchronized RpcEngine getProtocolEngine(Class<?> protocol, Configuration conf) {
    RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
    if (engine == null) {
      Class<?> impl =
          conf.getClass(ENGINE_PROP + "." + protocol.getName(), WritableRpcEngine.class);
      engine = (RpcEngine) ReflectionUtils.newInstance(impl, conf);
      PROTOCOL_ENGINES.put(protocol, engine);
    }
    return engine;
  }

  /** A version mismatch for the RPC protocol. */
  public static class VersionMismatch extends RpcServerException {
    private static final long serialVersionUID = 0;

    private String interfaceName;
    private long clientVersion;
    private long serverVersion;

    /**
     * Create a version mismatch exception
     *
     * @param interfaceName the name of the protocol mismatch
     * @param clientVersion the client's version of the protocol
     * @param serverVersion the server's version of the protocol
     */
    public VersionMismatch(String interfaceName, long clientVersion, long serverVersion) {
      super(
          "Protocol "
              + interfaceName
              + " version mismatch. (client = "
              + clientVersion
              + ", server = "
              + serverVersion
              + ")");
      this.interfaceName = interfaceName;
      this.clientVersion = clientVersion;
      this.serverVersion = serverVersion;
    }

    /**
     * Get the interface name
     *
     * @return the java class name (eg. org.apache.hadoop.mapred.InterTrackerProtocol)
     */
    public String getInterfaceName() {
      return interfaceName;
    }

    /** Get the client's preferred version */
    public long getClientVersion() {
      return clientVersion;
    }

    /** Get the server's agreed to version. */
    public long getServerVersion() {
      return serverVersion;
    }
    /** get the rpc status corresponding to this exception */
    public RpcStatusProto getRpcStatusProto() {
      return RpcStatusProto.ERROR;
    }

    /** get the detailed rpc status corresponding to this exception */
    public RpcErrorCodeProto getRpcErrorCodeProto() {
      return RpcErrorCodeProto.ERROR_RPC_VERSION_MISMATCH;
    }
  }

  /**
   * Get a proxy connection to a remote server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @return the proxy
   * @throws IOException if the far end through a RemoteException
   */
  public static <T> T waitForProxy(
      Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf)
      throws IOException {
    return waitForProtocolProxy(protocol, clientVersion, addr, conf).getProxy();
  }

  /**
   * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods
   * that are supported by the server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @return the protocol proxy
   * @throws IOException if the far end through a RemoteException
   */
  public static <T> ProtocolProxy<T> waitForProtocolProxy(
      Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf)
      throws IOException {
    return waitForProtocolProxy(protocol, clientVersion, addr, conf, Long.MAX_VALUE);
  }

  /**
   * Get a proxy connection to a remote server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @param connTimeout time in milliseconds before giving up
   * @return the proxy
   * @throws IOException if the far end through a RemoteException
   */
  public static <T> T waitForProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      long connTimeout)
      throws IOException {
    return waitForProtocolProxy(protocol, clientVersion, addr, conf, connTimeout).getProxy();
  }

  /**
   * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods
   * that are supported by the server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @param connTimeout time in milliseconds before giving up
   * @return the protocol proxy
   * @throws IOException if the far end through a RemoteException
   */
  public static <T> ProtocolProxy<T> waitForProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      long connTimeout)
      throws IOException {
    return waitForProtocolProxy(protocol, clientVersion, addr, conf, 0, null, connTimeout);
  }

  /**
   * Get a proxy connection to a remote server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @param rpcTimeout timeout for each RPC
   * @param timeout time in milliseconds before giving up
   * @return the proxy
   * @throws IOException if the far end through a RemoteException
   */
  public static <T> T waitForProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      int rpcTimeout,
      long timeout)
      throws IOException {
    return waitForProtocolProxy(protocol, clientVersion, addr, conf, rpcTimeout, null, timeout)
        .getProxy();
  }

  /**
   * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods
   * that are supported by the server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @param rpcTimeout timeout for each RPC
   * @param timeout time in milliseconds before giving up
   * @return the proxy
   * @throws IOException if the far end through a RemoteException
   */
  public static <T> ProtocolProxy<T> waitForProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      int rpcTimeout,
      RetryPolicy connectionRetryPolicy,
      long timeout)
      throws IOException {
    long startTime = Time.now();
    IOException ioe;
    while (true) {
      try {
        return getProtocolProxy(
            protocol,
            clientVersion,
            addr,
            UserGroupInformation.getCurrentUser(),
            conf,
            NetUtils.getDefaultSocketFactory(conf),
            rpcTimeout,
            connectionRetryPolicy);
      } catch (ConnectException se) { // namenode has not been started
        LOG.info("Server at " + addr + " not available yet, Zzzzz...");
        ioe = se;
      } catch (SocketTimeoutException te) { // namenode is busy
        LOG.info("Problem connecting to server: " + addr);
        ioe = te;
      } catch (NoRouteToHostException nrthe) { // perhaps a VIP is failing over
        LOG.info("No route to host for server: " + addr);
        ioe = nrthe;
      }
      // check if timed out
      if (Time.now() - timeout >= startTime) {
        throw ioe;
      }

      // wait for retry
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
        // IGNORE
      }
    }
  }

  /**
   * Construct a client-side proxy object that implements the named protocol, talking to a server at
   * the named address.
   *
   * @param <T>
   */
  public static <T> T getProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      SocketFactory factory)
      throws IOException {
    return getProtocolProxy(protocol, clientVersion, addr, conf, factory).getProxy();
  }

  /**
   * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods
   * that are supported by the server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @param factory socket factory
   * @return the protocol proxy
   * @throws IOException if the far end through a RemoteException
   */
  public static <T> ProtocolProxy<T> getProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      SocketFactory factory)
      throws IOException {
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    return getProtocolProxy(protocol, clientVersion, addr, ugi, conf, factory);
  }

  /**
   * Construct a client-side proxy object that implements the named protocol, talking to a server at
   * the named address.
   *
   * @param <T>
   */
  public static <T> T getProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      UserGroupInformation ticket,
      Configuration conf,
      SocketFactory factory)
      throws IOException {
    return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory).getProxy();
  }

  /**
   * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods
   * that are supported by the server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param ticket user group information
   * @param conf configuration to use
   * @param factory socket factory
   * @return the protocol proxy
   * @throws IOException if the far end through a RemoteException
   */
  public static <T> ProtocolProxy<T> getProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      UserGroupInformation ticket,
      Configuration conf,
      SocketFactory factory)
      throws IOException {
    return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory, 0, null);
  }

  /**
   * Construct a client-side proxy that implements the named protocol, talking to a server at the
   * named address.
   *
   * @param <T>
   * @param protocol protocol
   * @param clientVersion client's version
   * @param addr server address
   * @param ticket security ticket
   * @param conf configuration
   * @param factory socket factory
   * @param rpcTimeout max time for each rpc; 0 means no timeout
   * @return the proxy
   * @throws IOException if any error occurs
   */
  public static <T> T getProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      UserGroupInformation ticket,
      Configuration conf,
      SocketFactory factory,
      int rpcTimeout)
      throws IOException {
    return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout, null)
        .getProxy();
  }

  /**
   * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods
   * that are supported by the server
   *
   * @param protocol protocol
   * @param clientVersion client's version
   * @param addr server address
   * @param ticket security ticket
   * @param conf configuration
   * @param factory socket factory
   * @param rpcTimeout max time for each rpc; 0 means no timeout
   * @return the proxy
   * @throws IOException if any error occurs
   */
  public static <T> ProtocolProxy<T> getProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      UserGroupInformation ticket,
      Configuration conf,
      SocketFactory factory,
      int rpcTimeout,
      RetryPolicy connectionRetryPolicy)
      throws IOException {
    if (UserGroupInformation.isSecurityEnabled()) {
      SaslRpcServer.init(conf);
    }
    return getProtocolEngine(protocol, conf)
        .getProxy(
            protocol,
            clientVersion,
            addr,
            ticket,
            conf,
            factory,
            rpcTimeout,
            connectionRetryPolicy);
  }

  /**
   * Construct a client-side proxy object with the default SocketFactory
   *
   * @param <T>
   * @param protocol
   * @param clientVersion
   * @param addr
   * @param conf
   * @return a proxy instance
   * @throws IOException
   */
  public static <T> T getProxy(
      Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf)
      throws IOException {

    return getProtocolProxy(protocol, clientVersion, addr, conf).getProxy();
  }

  /** Returns the server address for a given proxy. */
  public static InetSocketAddress getServerAddress(Object proxy) {
    return getConnectionIdForProxy(proxy).getAddress();
  }

  /**
   * Return the connection ID of the given object. If the provided object is in fact a protocol
   * translator, we'll get the connection ID of the underlying proxy object.
   *
   * @param proxy the proxy object to get the connection ID of.
   * @return the connection ID for the provided proxy object.
   */
  public static ConnectionId getConnectionIdForProxy(Object proxy) {
    if (proxy instanceof ProtocolTranslator) {
      proxy = ((ProtocolTranslator) proxy).getUnderlyingProxyObject();
    }
    RpcInvocationHandler inv = (RpcInvocationHandler) Proxy.getInvocationHandler(proxy);
    return inv.getConnectionId();
  }

  /**
   * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods
   * that are supported by the server
   *
   * @param protocol
   * @param clientVersion
   * @param addr
   * @param conf
   * @return a protocol proxy
   * @throws IOException
   */
  public static <T> ProtocolProxy<T> getProtocolProxy(
      Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf)
      throws IOException {

    return getProtocolProxy(
        protocol, clientVersion, addr, conf, NetUtils.getDefaultSocketFactory(conf));
  }

  /**
   * Stop the proxy. Proxy must either implement {@link Closeable} or must have associated {@link
   * RpcInvocationHandler}.
   *
   * @param proxy the RPC proxy object to be stopped
   * @throws HadoopIllegalArgumentException if the proxy does not implement {@link Closeable}
   *     interface or does not have closeable {@link InvocationHandler}
   */
  public static void stopProxy(Object proxy) {
    if (proxy == null) {
      throw new HadoopIllegalArgumentException("Cannot close proxy since it is null");
    }
    try {
      if (proxy instanceof Closeable) {
        ((Closeable) proxy).close();
        return;
      } else {
        InvocationHandler handler = Proxy.getInvocationHandler(proxy);
        if (handler instanceof Closeable) {
          ((Closeable) handler).close();
          return;
        }
      }
    } catch (IOException e) {
      LOG.error("Closing proxy or invocation handler caused exception", e);
    } catch (IllegalArgumentException e) {
      LOG.error("RPC.stopProxy called on non proxy: class=" + proxy.getClass().getName(), e);
    }

    // If you see this error on a mock object in a unit test you're
    // developing, make sure to use MockitoUtil.mockProtocol() to
    // create your mock.
    throw new HadoopIllegalArgumentException(
        "Cannot close proxy - is not Closeable or "
            + "does not provide closeable invocation handler "
            + proxy.getClass());
  }

  /** Class to construct instances of RPC server with specific options. */
  public static class Builder {
    private Class<?> protocol = null;
    private Object instance = null;
    private String bindAddress = "0.0.0.0";
    private int port = 0;
    private int numHandlers = 1;
    private int numReaders = -1;
    private int queueSizePerHandler = -1;
    private boolean verbose = false;
    private final Configuration conf;
    private SecretManager<? extends TokenIdentifier> secretManager = null;
    private String portRangeConfig = null;

    public Builder(Configuration conf) {
      this.conf = conf;
    }

    /** Mandatory field */
    public Builder setProtocol(Class<?> protocol) {
      this.protocol = protocol;
      return this;
    }

    /** Mandatory field */
    public Builder setInstance(Object instance) {
      this.instance = instance;
      return this;
    }

    /** Default: 0.0.0.0 */
    public Builder setBindAddress(String bindAddress) {
      this.bindAddress = bindAddress;
      return this;
    }

    /** Default: 0 */
    public Builder setPort(int port) {
      this.port = port;
      return this;
    }

    /** Default: 1 */
    public Builder setNumHandlers(int numHandlers) {
      this.numHandlers = numHandlers;
      return this;
    }

    /** Default: -1 */
    public Builder setnumReaders(int numReaders) {
      this.numReaders = numReaders;
      return this;
    }

    /** Default: -1 */
    public Builder setQueueSizePerHandler(int queueSizePerHandler) {
      this.queueSizePerHandler = queueSizePerHandler;
      return this;
    }

    /** Default: false */
    public Builder setVerbose(boolean verbose) {
      this.verbose = verbose;
      return this;
    }

    /** Default: null */
    public Builder setSecretManager(SecretManager<? extends TokenIdentifier> secretManager) {
      this.secretManager = secretManager;
      return this;
    }

    /** Default: null */
    public Builder setPortRangeConfig(String portRangeConfig) {
      this.portRangeConfig = portRangeConfig;
      return this;
    }

    /**
     * Build the RPC Server.
     *
     * @throws IOException on error
     * @throws HadoopIllegalArgumentException when mandatory fields are not set
     */
    public Server build() throws IOException, HadoopIllegalArgumentException {
      if (this.conf == null) {
        throw new HadoopIllegalArgumentException("conf is not set");
      }
      if (this.protocol == null) {
        throw new HadoopIllegalArgumentException("protocol is not set");
      }
      if (this.instance == null) {
        throw new HadoopIllegalArgumentException("instance is not set");
      }

      return getProtocolEngine(this.protocol, this.conf)
          .getServer(
              this.protocol,
              this.instance,
              this.bindAddress,
              this.port,
              this.numHandlers,
              this.numReaders,
              this.queueSizePerHandler,
              this.verbose,
              this.conf,
              this.secretManager,
              this.portRangeConfig);
    }
  }

  /** An RPC Server. */
  public abstract static class Server extends org.apache.hadoop.ipc.Server {
    boolean verbose;

    static String classNameBase(String className) {
      String[] names = className.split("\\.", -1);
      if (names == null || names.length == 0) {
        return className;
      }
      return names[names.length - 1];
    }

    /** Store a map of protocol and version to its implementation */
    /** The key in Map */
    static class ProtoNameVer {
      final String protocol;
      final long version;

      ProtoNameVer(String protocol, long ver) {
        this.protocol = protocol;
        this.version = ver;
      }

      @Override
      public boolean equals(Object o) {
        if (o == null) return false;
        if (this == o) return true;
        if (!(o instanceof ProtoNameVer)) return false;
        ProtoNameVer pv = (ProtoNameVer) o;
        return ((pv.protocol.equals(this.protocol)) && (pv.version == this.version));
      }

      @Override
      public int hashCode() {
        return protocol.hashCode() * 37 + (int) version;
      }
    }

    /** The value in map */
    static class ProtoClassProtoImpl {
      final Class<?> protocolClass;
      final Object protocolImpl;

      ProtoClassProtoImpl(Class<?> protocolClass, Object protocolImpl) {
        this.protocolClass = protocolClass;
        this.protocolImpl = protocolImpl;
      }
    }

    ArrayList<Map<ProtoNameVer, ProtoClassProtoImpl>> protocolImplMapArray =
        new ArrayList<Map<ProtoNameVer, ProtoClassProtoImpl>>(RpcKind.MAX_INDEX);

    Map<ProtoNameVer, ProtoClassProtoImpl> getProtocolImplMap(RPC.RpcKind rpcKind) {
      if (protocolImplMapArray.size() == 0) { // initialize for all rpc kinds
        for (int i = 0; i <= RpcKind.MAX_INDEX; ++i) {
          protocolImplMapArray.add(new HashMap<ProtoNameVer, ProtoClassProtoImpl>(10));
        }
      }
      return protocolImplMapArray.get(rpcKind.ordinal());
    }

    // Register  protocol and its impl for rpc calls
    void registerProtocolAndImpl(RpcKind rpcKind, Class<?> protocolClass, Object protocolImpl) {
      String protocolName = RPC.getProtocolName(protocolClass);
      long version;

      try {
        version = RPC.getProtocolVersion(protocolClass);
      } catch (Exception ex) {
        LOG.warn("Protocol " + protocolClass + " NOT registered as cannot get protocol version ");
        return;
      }

      getProtocolImplMap(rpcKind)
          .put(
              new ProtoNameVer(protocolName, version),
              new ProtoClassProtoImpl(protocolClass, protocolImpl));
      LOG.debug(
          "RpcKind = "
              + rpcKind
              + " Protocol Name = "
              + protocolName
              + " version="
              + version
              + " ProtocolImpl="
              + protocolImpl.getClass().getName()
              + " protocolClass="
              + protocolClass.getName());
    }

    static class VerProtocolImpl {
      final long version;
      final ProtoClassProtoImpl protocolTarget;

      VerProtocolImpl(long ver, ProtoClassProtoImpl protocolTarget) {
        this.version = ver;
        this.protocolTarget = protocolTarget;
      }
    }

    VerProtocolImpl[] getSupportedProtocolVersions(RPC.RpcKind rpcKind, String protocolName) {
      VerProtocolImpl[] resultk = new VerProtocolImpl[getProtocolImplMap(rpcKind).size()];
      int i = 0;
      for (Map.Entry<ProtoNameVer, ProtoClassProtoImpl> pv :
          getProtocolImplMap(rpcKind).entrySet()) {
        if (pv.getKey().protocol.equals(protocolName)) {
          resultk[i++] = new VerProtocolImpl(pv.getKey().version, pv.getValue());
        }
      }
      if (i == 0) {
        return null;
      }
      VerProtocolImpl[] result = new VerProtocolImpl[i];
      System.arraycopy(resultk, 0, result, 0, i);
      return result;
    }

    VerProtocolImpl getHighestSupportedProtocol(RpcKind rpcKind, String protocolName) {
      Long highestVersion = 0L;
      ProtoClassProtoImpl highest = null;
      if (LOG.isDebugEnabled()) {
        LOG.debug("Size of protoMap for " + rpcKind + " =" + getProtocolImplMap(rpcKind).size());
      }
      for (Map.Entry<ProtoNameVer, ProtoClassProtoImpl> pv :
          getProtocolImplMap(rpcKind).entrySet()) {
        if (pv.getKey().protocol.equals(protocolName)) {
          if ((highest == null) || (pv.getKey().version > highestVersion)) {
            highest = pv.getValue();
            highestVersion = pv.getKey().version;
          }
        }
      }
      if (highest == null) {
        return null;
      }
      return new VerProtocolImpl(highestVersion, highest);
    }

    protected Server(
        String bindAddress,
        int port,
        Class<? extends Writable> paramClass,
        int handlerCount,
        int numReaders,
        int queueSizePerHandler,
        Configuration conf,
        String serverName,
        SecretManager<? extends TokenIdentifier> secretManager,
        String portRangeConfig)
        throws IOException {
      super(
          bindAddress,
          port,
          paramClass,
          handlerCount,
          numReaders,
          queueSizePerHandler,
          conf,
          serverName,
          secretManager,
          portRangeConfig);
      initProtocolMetaInfo(conf);
    }

    private void initProtocolMetaInfo(Configuration conf) {
      RPC.setProtocolEngine(conf, ProtocolMetaInfoPB.class, ProtobufRpcEngine.class);
      ProtocolMetaInfoServerSideTranslatorPB xlator =
          new ProtocolMetaInfoServerSideTranslatorPB(this);
      BlockingService protocolInfoBlockingService =
          ProtocolInfoService.newReflectiveBlockingService(xlator);
      addProtocol(
          RpcKind.RPC_PROTOCOL_BUFFER, ProtocolMetaInfoPB.class, protocolInfoBlockingService);
    }

    /**
     * Add a protocol to the existing server.
     *
     * @param protocolClass - the protocol class
     * @param protocolImpl - the impl of the protocol that will be called
     * @return the server (for convenience)
     */
    public Server addProtocol(RpcKind rpcKind, Class<?> protocolClass, Object protocolImpl) {
      registerProtocolAndImpl(rpcKind, protocolClass, protocolImpl);
      return this;
    }

    @Override
    public Writable call(
        RPC.RpcKind rpcKind, String protocol, Writable rpcRequest, long receiveTime)
        throws Exception {
      return getRpcInvoker(rpcKind).call(this, protocol, rpcRequest, receiveTime);
    }
  }
}
public class MilterRequestHandler implements RequestHandler, JilterHandler, StopBlockTarget {

  protected static Log logger = LogFactory.getLog(MilterRequestHandler.class.getName());
  protected SocketChannel socket = null;
  protected ArrayList<String> rcpts = null;
  protected FetchMessageCallback callback;
  protected String host = "";
  protected JilterStatus status = null;
  protected ByteArrayOutputStream bos = new ByteArrayOutputStream();
  protected static Pattern headerPattern1 = Pattern.compile("^cc|^to|^bcc");
  protected static Pattern headerPattern2 = Pattern.compile(".*<([-.+_\\d\\w]*@[-.+_\\d\\w]*)>");
  protected static Pattern headerPattern3 = Pattern.compile("([-.+_\\d\\w]*@[-.+_\\d\\w]*)");
  private static final int IDLE_TIMEOUT = 300000; // 5 minutes
  protected boolean includeBCC = false;

  public void handleRequest(SocketChannel socket, FetchMessageCallback callback) {
    this.socket = socket;
    this.callback = callback;
    includeBCC = false;
    rcpts = new ArrayList<String>();
    bos = new ByteArrayOutputStream();

    InetAddress address = socket.socket().getInetAddress();
    boolean isAllowed = Config.getConfig().getAgent().isAllowed(address);
    if (!isAllowed) {
      logger.debug(
          "attempted milter connection from disallowed address. force disconnect {address='"
              + address.getHostAddress()
              + "'}");
      try {
        socket.close();
      } catch (IOException io) {
        logger.error("failed to close milter socket.", io);
      }
      return;
    }

    ByteBuffer dataBuffer = ByteBuffer.allocateDirect(4096);
    JilterProcessor processor = new JilterProcessor(this);
    try {
      while (processor.process(socket, (ByteBuffer) dataBuffer.flip())) {
        dataBuffer.compact();
        if (this.socket.read(dataBuffer) == -1) {
          logger.debug("socket reports EOF, exiting read loop");
          break;
        }
      }
    } catch (IOException e) {
      logger.debug("Unexpected exception, connection will be closed", e);
    } finally {
      logger.debug("closing processor");
      processor.close();
      logger.debug("processor closed");
      try {
        logger.debug("closing socket");
        this.socket.close();
        logger.debug("socket closed");
      } catch (IOException e) {
        logger.debug("Unexpected exception", e);
      }
    }
  }

  public JilterStatus abort() {
    logger.debug("abort");
    return JilterStatus.SMFIS_CONTINUE;
  }

  public JilterStatus body(ByteBuffer bodyp) {
    logger.debug("jilter body()");
    long maxMessageSizeMB = Config.getConfig().getArchiver().getMaxMessageSize();
    long maxMessageSizeBytes = maxMessageSizeMB * 1024 * 1024;
    if (bodyp.array().length > maxMessageSizeBytes) {
      logger.warn(
          "milter maximum message size exceeded { size='" + bodyp.array().length + " bytes'}");
      return JilterStatus.SMFIS_REJECT;
    }
    try {
      bos.write("\n".getBytes());
      bos.write(bodyp.array());
    } catch (IOException io) {
      logger.error("jilter failed to write milter body data to byte buffer", io);
    }
    logger.debug("jilter body written");
    return JilterStatus.SMFIS_CONTINUE;
  }

  public JilterStatus close() {
    logger.debug("jilter close()");
    return JilterStatus.SMFIS_CONTINUE;
  }

  public JilterStatus connect(String hostname, InetAddress hostaddr, Properties properties) {
    rcpts = new ArrayList<String>();
    if (hostaddr != null) {
      host = hostaddr.toString();
    } else if (host != null) {
      host = hostname;
    } else {
      host = "localhost";
    }
    logger.debug("jilter connect() {from='" + hostname + "',host='" + host + "'}");
    return JilterStatus.SMFIS_CONTINUE;
  }

  public JilterStatus envfrom(String[] argv, Properties properties) {
    for (int i = 0; i < argv.length; i++) {
      logger.debug("jilter envfrom() {from='" + argv[i] + "'}");
    }
    return JilterStatus.SMFIS_CONTINUE;
  }

  public JilterStatus envrcpt(String[] argv, Properties properties) {
    for (int i = 0; i < argv.length; i++) {
      String strRecipient = argv[i];
      boolean orcptFlag = strRecipient.toLowerCase(Locale.ENGLISH).trim().contains("orcpt=");
      if (!orcptFlag) {
        logger.debug("jilter envrcpt() {to='" + strRecipient + "'}");
        String recipient =
            strRecipient.toLowerCase(Locale.ENGLISH).trim().replaceAll("<", "").replaceAll(">", "");
        rcpts.add(recipient);

        logger.debug("jilter add recipient {recipient='" + recipient + "'}");
      }
    }
    return JilterStatus.SMFIS_CONTINUE;
  }

  protected boolean shouldIgnoreBCCAddress(String address) {
    MilterServerService milterService = Config.getConfig().getMilterServerService();
    List<String> ignoreAddresses = milterService.getIgnoreBCCAddress();
    Matcher m = headerPattern2.matcher(address.toLowerCase(Locale.ENGLISH).trim());
    if (m.matches()) {
      String mailAddress = m.group(1);
      for (String ignoreAddress : ignoreAddresses) {
        if (ignoreAddress.equalsIgnoreCase(mailAddress)) return true;
      }
    } else {
      m = headerPattern3.matcher(address.toLowerCase(Locale.ENGLISH).trim());
      if (m.matches()) {
        String mailAddress = m.group(1);
        for (String ignoreAddress : ignoreAddresses) {
          if (ignoreAddress.equalsIgnoreCase(mailAddress)) return true;
        }
      }
    }
    return false;
  }

  public JilterStatus eoh() {
    logger.debug("jilter eoh()");
    // includeBCC is false if RCPT TO does not contain at least one field in TO, FROM and CC
    // this is a safety check as sometimes, RCPT TO is something differently entirely
    // and does not contain the actual recipients in the email

    MilterServerService milterService = Config.getConfig().getMilterServerService();

    if (milterService.getIncludeBCC() && includeBCC) {
      logger.debug("including BCC addresses");
      // check to see if address is flagged to ignore
      if (rcpts.size() > 0) {
        Iterator<String> i = rcpts.iterator();
        while (i.hasNext()) {
          String rcpt = i.next();
          if (shouldIgnoreBCCAddress(rcpt)) {
            logger.debug("ignore include bcc address {address='" + rcpt + "'}");
            i.remove();
          }
        }
      }

      if (rcpts.size() > 0) {
        try {
          for (int j = 0; j < rcpts.size(); j++) {
            if (j == 0) {
              bos.write("bcc: ".getBytes());
            } else {
              bos.write(",".getBytes());
            }
            bos.write(rcpts.get(j).getBytes());
          }
          bos.write("\n".getBytes());
        } catch (IOException io) {
          logger.error("jilter failed to write end of header data", io);
        }
      }
    }
    return JilterStatus.SMFIS_CONTINUE;
  }

  public JilterStatus eom(JilterEOMActions eomActions, Properties properties) {
    logger.debug("jilter eom()");
    try {
      bos.close(); // close stream
    } catch (IOException io) {
      logger.error("jilter failed to close io stream during eom", io);
    }
    byte[] messageBytes = bos.toByteArray();
    bos = new ByteArrayOutputStream();
    ByteArrayInputStream bis = new ByteArrayInputStream(messageBytes);
    try {
      logger.debug("jilter store callback execute");
      Config.getStopBlockFactory()
          .detectBlock("milter server", Thread.currentThread(), this, IDLE_TIMEOUT);
      callback.store(bis, host);
      logger.debug("jilter store callback finished");
    } catch (ArchiveException e) {
      logger.error("failed to store the message via milter", e);
      if (e.getRecoveryDirective() == ArchiveException.RecoveryDirective.REJECT) {
        logger.debug("jilter reject");
        return JilterStatus.SMFIS_REJECT;
      } else if (e.getRecoveryDirective() == ArchiveException.RecoveryDirective.RETRYLATER) {
        logger.debug("jilter temp fail");
        return JilterStatus.SMFIS_TEMPFAIL;
      }
    } catch (Throwable oome) {
      logger.error("failed to store message:" + oome.getMessage(), oome);
      return JilterStatus.SMFIS_REJECT;
    } finally {
      Config.getStopBlockFactory().endDetectBlock(Thread.currentThread());
    }
    return JilterStatus.SMFIS_CONTINUE;
  }

  public int getRequiredModifications() {
    logger.debug("jilter requiredmodifications()");
    return SMFIF_NONE;
  }

  public int getSupportedProcesses() {
    logger.debug("jilter getsupportedprocesses()");
    return PROCESS_CONNECT | PROCESS_ENVRCPT | PROCESS_HEADER | PROCESS_BODY;
  }

  public JilterStatus header(String headerf, String headerv) {

    logger.debug("jilter header {name='" + headerf + "',value='" + headerv + "'}");
    StringBuffer header = new StringBuffer();
    header.append(headerf);
    header.append(": ");
    header.append(headerv);
    header.append("\n");
    try {
      bos.write(header.toString().getBytes());
    } catch (IOException io) {
      logger.error("jilter failed to write header field", io);
    }
    Matcher m = headerPattern1.matcher(headerf.toLowerCase(Locale.ENGLISH).trim());
    if (m.matches()) {
      logger.debug("jilter found to/bcc/cc header");
      String[] addresses = headerv.split(",");
      for (int i = 0; i < addresses.length; i++) {
        includeBCC = includeBCC | rcpts.remove(addresses[i].toLowerCase(Locale.ENGLISH).trim());
        logger.debug("jilter del recipient {recipient='" + addresses[i] + "'}");
        m = headerPattern2.matcher(addresses[i].toLowerCase(Locale.ENGLISH).trim());
        if (m.matches()) {
          String mailAddress = m.group(1);
          includeBCC = includeBCC | rcpts.remove(mailAddress);
          logger.debug("jilter del recipient {recipient='" + mailAddress + "'}");
        } else {
          m = headerPattern3.matcher(addresses[i].toLowerCase(Locale.ENGLISH).trim());
          if (m.matches()) {
            String mailAddress = m.group(1);
            includeBCC = includeBCC | rcpts.remove(mailAddress);
            logger.debug("jilter del recipient {recipient='" + mailAddress + "'}");
          }
        }
      }
    }
    return JilterStatus.SMFIS_CONTINUE;
  }

  public JilterStatus helo(String helohost, Properties properties) {
    logger.debug("jilter helo() " + helohost);
    return JilterStatus.SMFIS_CONTINUE;
  }

  public void handleBlock(Thread thread) {

    try {
      if (socket != null) {
        logger.debug("close socket()");
        socket.close();
      }
    } catch (Exception e) {
      // ignored
    }
    synchronized (this) {
      if (thread != null) {
        logger.debug("interrupt thread()");
        thread.interrupt();
      }
    }
  }
}