@Service public abstract class FantasyGame { private static final Log LOG = LogFactory.getLog(FantasyGame.class); protected FantasyTeam fantasyTeam; public FantasyGame() {} public void tradeForTomorrow() throws Exception { tradeForDate(DateUtil.getGameTomorrow()); } public void tradeForDate(Date date) throws Exception { BbcLeague bbcLeague = getLeague(date); Starters expectedStarters = fantasyTeam.getStrategy().pickStarters(date, bbcLeague); Starters actualStarters = tradeForStarters(expectedStarters); boolean startersSet = actualStarters.equals(expectedStarters); System.out.println("starters look good - " + startersSet); } public abstract BbcLeague getLeague(Date date) throws IOException; public abstract Starters tradeForStarters(Starters starters) throws IOException; }
/** * MDC server * * @author yjiang */ public abstract class MDCServer extends IoHandlerAdapter { static final Log log = LogFactory.getLog(MDCServer.class); static final AtomicInteger counter = new AtomicInteger(0); /** the the max size of a packet, 32KB */ static int MAX_SIZE = 10240 * 1024; // test protected InetSocketAddress address; protected Selector selector; protected ServerSocketChannel server; protected final int PROCESS_NUMBER = 4; protected static Configuration _conf; protected IoAcceptor acceptor; protected boolean isRunning = false; protected boolean testKey() { String data = UID.random(24); byte[] bb = RSA.encode(data.getBytes(), TConn.pub_key); if (bb != null) { bb = RSA.decode(bb, TConn.pri_key); if (bb != null && data.equals(new String(bb))) { return true; } } return false; } /** Close. */ public void close() { if (selector != null) { selector.wakeup(); try { selector.close(); } catch (IOException e1) { log.warn("close selector fails", e1); } finally { selector = null; } } if (server != null) { try { server.socket().close(); server.close(); } catch (IOException e) { log.warn("close socket server fails", e); } finally { server = null; } } } /** * Instantiates a new MDC server. * * @param host the host * @param port the port */ protected MDCServer(String host, int port) { _conf = Config.getConfig(); address = (host == null) ? new InetSocketAddress(port) : new InetSocketAddress(host, port); /** initialize app command */ Command.init(); /** initialize the connection center */ TConnCenter.init(_conf, port); synchronized (_conf) { /** load public key from database */ TConn.pub_key = SystemConfig.s("pub_key", null); TConn.pri_key = SystemConfig.s("pri_key", null); /** initialize the RSA key, hardcode 2048 bits */ if (TConn.pub_key == null || TConn.pri_key == null || "".equals(TConn.pub_key) || "".equals(TConn.pri_key)) { /** print out the old state */ log.warn( "the pub_key or pri_key missed, the old state are pub_key:[" + TConn.pub_key + "], pri_key:[" + TConn.pri_key + "]"); Key k = RSA.generate(2048); TConn.pri_key = k.pri_key; TConn.pub_key = k.pub_key; /** print out the new public key */ log.warn("create new RSA key pair, pub_key:[" + TConn.pub_key + ']'); /** set back in database */ SystemConfig.setConfig("pri_key", TConn.pri_key); SystemConfig.setConfig("pub_key", TConn.pub_key); } MAX_SIZE = SystemConfig.i("mdc.max_size", MAX_SIZE); } } /** * Start. * * @return the MDC server */ public abstract MDCServer start(); /** Stop. */ public void stop() { acceptor.unbind(); } /** * Service. * * @param o the o * @param session the session */ void service(IoBuffer o, IoSession session) { try { // System.out.println(o.remaining() + "/" + o.capacity()); session.setAttribute("last", System.currentTimeMillis()); SimpleIoBuffer in = (SimpleIoBuffer) session.getAttribute("buf"); if (in == null) { in = SimpleIoBuffer.create(4096); session.setAttribute("buf", in); } byte[] data = new byte[o.remaining()]; o.get(data); in.append(data); // log.debug("recv: " + data.length + ", " + // session.getRemoteAddress()); while (in.length() > 5) { in.mark(); /** * Byte 1: head of the package<br> * bit 7-6: "01", indicator of MDC<br> * bit 5: encrypt indicator, "0": no; "1": encrypted<br> * bit 4: zip indicator, "0": no, "1": ziped<br> * bit 0-3: reserved<br> * Byte 2-5: length of data<br> * Byte[…]: data array<br> */ byte head = in.read(); /** test the head indicator, if not correct close it */ if ((head & 0xC0) != 0x40) { log.info("flag is not correct! flag:" + head + ",from: " + session.getRemoteAddress()); session.write("error.head"); session.close(true); return; } int len = in.getInt(); if (len <= 0 || len > MAX_SIZE) { log.error( "mdcserver.Wrong lendth: " + len + "/" + MAX_SIZE + " - " + session.getRemoteAddress()); session.write("error.packet.size"); session.close(true); break; } // log.info("packet.len:" + len + ", len in buffer:" + // in.length()); if (in.length() < len) { in.reset(); break; } else { // do it byte[] b = new byte[len]; in.read(b); // log.info("stub.package.size: " + len + ", head:" + head + // ", cmd:" + Bean.toString(b)); // log.info("stub.package.size: " + len + ", head:" + head); /** test the zip flag */ if ((head & 0x10) != 0) { b = Zip.unzip(b); } final TConn d = (TConn) session.getAttribute("conn"); if (d != null) { /** test the encrypted flag */ if ((head & 0x20) != 0) { b = DES.decode(b, d.deskey); } final byte[] bb = b; /** test if the packet is for mdc or app */ new WorkerTask() { @Override public void onExecute() { d.process(bb); } }.schedule(0); session.setAttribute("last", System.currentTimeMillis()); } else { session.write("error.getconnection"); log.error("error to get connection: " + session.getRemoteAddress()); session.close(true); } } } } catch (Throwable e) { log.error("closing stub: " + session.getRemoteAddress(), e); session.write("exception." + e.getMessage()); session.close(true); } } /* * (non-Javadoc) * * @see * org.apache.mina.core.service.IoHandlerAdapter#sessionCreated(org.apache * .mina.core.session.IoSession) */ public void sessionCreated(IoSession session) throws Exception { log.info("stub created:" + session.getRemoteAddress()); Counter.add("mdc", "connection", 1); TConn d = new TConn(session); d.set("x-forwarded-for", session.getRemoteAddress().toString()); session.setAttribute("conn", d); } /* * (non-Javadoc) * * @see * org.apache.mina.core.service.IoHandlerAdapter#sessionClosed(org.apache * .mina.core.session.IoSession) */ public void sessionClosed(IoSession session) throws Exception { log.info("closed stub: " + session.getRemoteAddress()); TConn d = (TConn) session.getAttribute("conn"); if (d != null) { d.close(); session.removeAttribute("conn"); } } /* * (non-Javadoc) * * @see * org.apache.mina.core.service.IoHandlerAdapter#sessionIdle(org.apache. * mina.core.session.IoSession, org.apache.mina.core.session.IdleStatus) */ public void sessionIdle(IoSession session, IdleStatus status) throws Exception { if (IdleStatus.BOTH_IDLE.equals(status)) { Long l = (Long) session.getAttribute("last"); if (l != null && System.currentTimeMillis() - l > 60 * 1000) { session.close(true); } } } /* * (non-Javadoc) * * @see * org.apache.mina.core.service.IoHandlerAdapter#messageReceived(org.apache * .mina.core.session.IoSession, java.lang.Object) */ public void messageReceived(IoSession session, Object message) throws Exception { // System.out.println(message); if (message instanceof IoBuffer) { service((IoBuffer) message, session); } } /** * Creates the tcp server. * * @param host the host * @param port the port * @return the MDC server */ public static synchronized MDCServer createTcpServer(String host, int port) { return new TDCServer(host, port); } /** * Creates the udp server. * * @param host the host * @param port the port * @return the MDC server */ public static synchronized MDCServer createUdpServer(String host, int port) { return new UDCServer(host, port); } /* * (non-Javadoc) * * @see * org.apache.mina.core.service.IoHandlerAdapter#exceptionCaught(org.apache * .mina.core.session.IoSession, java.lang.Throwable) */ @Override public void exceptionCaught(IoSession session, Throwable cause) throws Exception { TConn d = (TConn) session.getAttribute("conn"); if (d != null && d.valid()) { App.bye(d); } } }
/** * The <code>DefaultTcpTransportMapping</code> implements a TCP transport mapping with the Java 1.4 * new IO API. * * <p>It uses a single thread for processing incoming and outgoing messages. The thread is started * when the <code>listen</code> method is called, or when an outgoing request is sent using the * <code>sendMessage</code> method. * * @author Frank Fock * @version 1.7.4a */ public class DefaultTcpTransportMapping extends TcpTransportMapping { private static final LogAdapter logger = LogFactory.getLogger(DefaultTcpTransportMapping.class); private Hashtable sockets = new Hashtable(); private ServerThread server; private Timer socketCleaner; // 1 minute default timeout private long connectionTimeout = 60000; private boolean serverEnabled = false; private static final int MIN_SNMP_HEADER_LENGTH = 6; private MessageLengthDecoder messageLengthDecoder = new SnmpMesssageLengthDecoder(); /** * Creates a default TCP transport mapping with the server for incoming messages disabled. * * @throws UnknownHostException * @throws IOException on failure of binding a local port. */ public DefaultTcpTransportMapping() throws UnknownHostException, IOException { super(new TcpAddress(InetAddress.getLocalHost(), 0)); } /** * Creates a default TCP transport mapping that binds to the given address (interface) on the * local host. * * @param serverAddress the TcpAddress instance that describes the server address to listen on * incoming connection requests. * @throws UnknownHostException if the specified interface does not exist. * @throws IOException if the given address cannot be bound. */ public DefaultTcpTransportMapping(TcpAddress serverAddress) throws UnknownHostException, IOException { super(serverAddress); this.serverEnabled = true; } /** * Listen for incoming and outgoing requests. If the <code>serverEnabled</code> member is <code> * false</code> the server for incoming requests is not started. This starts the internal server * thread that processes messages. * * @throws SocketException when the transport is already listening for incoming/outgoing messages. * @throws IOException */ public synchronized void listen() throws java.io.IOException { if (server != null) { throw new SocketException("Port already listening"); } server = new ServerThread(); if (connectionTimeout > 0) { socketCleaner = new Timer(true); // run as daemon } server.setDaemon(true); server.start(); } /** * Changes the priority of the server thread for this TCP transport mapping. This method has no * effect, if called before {@link #listen()} has been called for this transport mapping. * * @param newPriority the new priority. * @see Thread#setPriority * @since 1.2.2 */ public void setPriority(int newPriority) { ServerThread st = server; if (st != null) { st.setPriority(newPriority); } } /** * Returns the priority of the internal listen thread. * * @return a value between {@link Thread#MIN_PRIORITY} and {@link Thread#MAX_PRIORITY}. * @since 1.2.2 */ public int getPriority() { ServerThread st = server; if (st != null) { return st.getPriority(); } else { return Thread.NORM_PRIORITY; } } /** * Sets the name of the listen thread for this UDP transport mapping. This method has no effect, * if called before {@link #listen()} has been called for this transport mapping. * * @param name the new thread name. * @since 1.6 */ public void setThreadName(String name) { ServerThread st = server; if (st != null) { st.setName(name); } } /** * Returns the name of the listen thread. * * @return the thread name if in listening mode, otherwise <code>null</code>. * @since 1.6 */ public String getThreadName() { ServerThread st = server; if (st != null) { return st.getName(); } else { return null; } } /** Closes all open sockets and stops the internal server thread that processes messages. */ public void close() { ServerThread st = server; if (st != null) { st.close(); try { st.join(); } catch (InterruptedException ex) { logger.warn(ex); } server = null; for (Iterator it = sockets.values().iterator(); it.hasNext(); ) { SocketEntry entry = (SocketEntry) it.next(); try { synchronized (entry) { entry.getSocket().close(); } logger.debug("Socket to " + entry.getPeerAddress() + " closed"); } catch (IOException iox) { // ingore logger.debug(iox); } } if (socketCleaner != null) { socketCleaner.cancel(); } socketCleaner = null; } } /** * Closes a connection to the supplied remote address, if it is open. This method is particularly * useful when not using a timeout for remote connections. * * @param remoteAddress the address of the peer socket. * @return <code>true</code> if the connection has been closed and <code>false</code> if there was * nothing to close. * @since 1.7.1 */ public synchronized boolean close(Address remoteAddress) throws IOException { if (logger.isDebugEnabled()) { logger.debug("Closing socket for peer address " + remoteAddress); } SocketEntry entry = (SocketEntry) sockets.remove(remoteAddress); if (entry != null) { synchronized (entry) { entry.getSocket().close(); } logger.info("Socket to " + entry.getPeerAddress() + " closed"); return true; } return false; } /** * Sends a SNMP message to the supplied address. * * @param address an <code>TcpAddress</code>. A <code>ClassCastException</code> is thrown if * <code>address</code> is not a <code>TcpAddress</code> instance. * @param message byte[] the message to sent. * @throws IOException */ public void sendMessage(Address address, byte[] message) throws java.io.IOException { if (server == null) { listen(); } server.sendMessage(address, message); } /** * Gets the connection timeout. This timeout specifies the time a connection may be idle before it * is closed. * * @return long the idle timeout in milliseconds. */ public long getConnectionTimeout() { return connectionTimeout; } /** * Sets the connection timeout. This timeout specifies the time a connection may be idle before it * is closed. * * @param connectionTimeout the idle timeout in milliseconds. A zero or negative value will * disable any timeout and connections opened by this transport mapping will stay opened until * they are explicitly closed. */ public void setConnectionTimeout(long connectionTimeout) { this.connectionTimeout = connectionTimeout; } /** * Checks whether a server for incoming requests is enabled. * * @return boolean */ public boolean isServerEnabled() { return serverEnabled; } public MessageLengthDecoder getMessageLengthDecoder() { return messageLengthDecoder; } /** * Sets whether a server for incoming requests should be created when the transport is set into * listen state. Setting this value has no effect until the {@link #listen()} method is called (if * the transport is already listening, {@link #close()} has to be called before). * * @param serverEnabled if <code>true</code> if the transport will listens for incoming requests * after {@link #listen()} has been called. */ public void setServerEnabled(boolean serverEnabled) { this.serverEnabled = serverEnabled; } /** * Sets the message length decoder. Default message length decoder is the {@link * SnmpMesssageLengthDecoder}. The message length decoder must be able to decode the total length * of a message for this transport mapping protocol(s). * * @param messageLengthDecoder a <code>MessageLengthDecoder</code> instance. */ public void setMessageLengthDecoder(MessageLengthDecoder messageLengthDecoder) { if (messageLengthDecoder == null) { throw new NullPointerException(); } this.messageLengthDecoder = messageLengthDecoder; } /** * Gets the inbound buffer size for incoming requests. When SNMP packets are received that are * longer than this maximum size, the messages will be silently dropped and the connection will be * closed. * * @return the maximum inbound buffer size in bytes. */ public int getMaxInboundMessageSize() { return super.getMaxInboundMessageSize(); } /** * Sets the maximum buffer size for incoming requests. When SNMP packets are received that are * longer than this maximum size, the messages will be silently dropped and the connection will be * closed. * * @param maxInboundMessageSize the length of the inbound buffer in bytes. */ public void setMaxInboundMessageSize(int maxInboundMessageSize) { this.maxInboundMessageSize = maxInboundMessageSize; } private synchronized void timeoutSocket(SocketEntry entry) { if (connectionTimeout > 0) { socketCleaner.schedule(new SocketTimeout(entry), connectionTimeout); } } public boolean isListening() { return (server != null); } class SocketEntry { private Socket socket; private TcpAddress peerAddress; private long lastUse; private LinkedList message = new LinkedList(); private ByteBuffer readBuffer = null; public SocketEntry(TcpAddress address, Socket socket) { this.peerAddress = address; this.socket = socket; this.lastUse = System.currentTimeMillis(); } public long getLastUse() { return lastUse; } public void used() { lastUse = System.currentTimeMillis(); } public Socket getSocket() { return socket; } public TcpAddress getPeerAddress() { return peerAddress; } public synchronized void addMessage(byte[] message) { this.message.add(message); } public byte[] nextMessage() { if (this.message.size() > 0) { return (byte[]) this.message.removeFirst(); } return null; } public void setReadBuffer(ByteBuffer byteBuffer) { this.readBuffer = byteBuffer; } public ByteBuffer getReadBuffer() { return readBuffer; } public String toString() { return "SocketEntry[peerAddress=" + peerAddress + ",socket=" + socket + ",lastUse=" + new Date(lastUse) + "]"; } } public static class SnmpMesssageLengthDecoder implements MessageLengthDecoder { public int getMinHeaderLength() { return MIN_SNMP_HEADER_LENGTH; } public MessageLength getMessageLength(ByteBuffer buf) throws IOException { MutableByte type = new MutableByte(); BERInputStream is = new BERInputStream(buf); int ml = BER.decodeHeader(is, type); int hl = (int) is.getPosition(); MessageLength messageLength = new MessageLength(hl, ml); return messageLength; } } class SocketTimeout extends TimerTask { private SocketEntry entry; public SocketTimeout(SocketEntry entry) { this.entry = entry; } /** run */ public void run() { long now = System.currentTimeMillis(); if ((socketCleaner == null) || (now - entry.getLastUse() >= connectionTimeout)) { if (logger.isDebugEnabled()) { logger.debug( "Socket has not been used for " + (now - entry.getLastUse()) + " micro seconds, closing it"); } sockets.remove(entry.getPeerAddress()); try { synchronized (entry) { entry.getSocket().close(); } logger.info("Socket to " + entry.getPeerAddress() + " closed due to timeout"); } catch (IOException ex) { logger.error(ex); } } else { if (logger.isDebugEnabled()) { logger.debug("Scheduling " + ((entry.getLastUse() + connectionTimeout) - now)); } socketCleaner.schedule( new SocketTimeout(entry), (entry.getLastUse() + connectionTimeout) - now); } } } class ServerThread extends Thread { private byte[] buf; private volatile boolean stop = false; private Throwable lastError = null; private ServerSocketChannel ssc; private Selector selector; private LinkedList pending = new LinkedList(); public ServerThread() throws IOException { setName("DefaultTCPTransportMapping_" + getAddress()); buf = new byte[getMaxInboundMessageSize()]; // Selector for incoming requests selector = Selector.open(); if (serverEnabled) { // Create a new server socket and set to non blocking mode ssc = ServerSocketChannel.open(); ssc.configureBlocking(false); // Bind the server socket InetSocketAddress isa = new InetSocketAddress(tcpAddress.getInetAddress(), tcpAddress.getPort()); ssc.socket().bind(isa); // Register accepts on the server socket with the selector. This // step tells the selector that the socket wants to be put on the // ready list when accept operations occur, so allowing multiplexed // non-blocking I/O to take place. ssc.register(selector, SelectionKey.OP_ACCEPT); } } private void processPending() { synchronized (pending) { while (pending.size() > 0) { SocketEntry entry = (SocketEntry) pending.removeFirst(); try { // Register the channel with the selector, indicating // interest in connection completion and attaching the // target object so that we can get the target back // after the key is added to the selector's // selected-key set if (entry.getSocket().isConnected()) { entry.getSocket().getChannel().register(selector, SelectionKey.OP_WRITE, entry); } else { entry.getSocket().getChannel().register(selector, SelectionKey.OP_CONNECT, entry); } } catch (IOException iox) { logger.error(iox); // Something went wrong, so close the channel and // record the failure try { entry.getSocket().getChannel().close(); TransportStateEvent e = new TransportStateEvent( DefaultTcpTransportMapping.this, entry.getPeerAddress(), TransportStateEvent.STATE_CLOSED, iox); fireConnectionStateChanged(e); } catch (IOException ex) { logger.error(ex); } lastError = iox; } } } } public Throwable getLastError() { return lastError; } public void sendMessage(Address address, byte[] message) throws java.io.IOException { Socket s = null; SocketEntry entry = (SocketEntry) sockets.get(address); if (logger.isDebugEnabled()) { logger.debug("Looking up connection for destination '" + address + "' returned: " + entry); logger.debug(sockets.toString()); } if (entry != null) { s = entry.getSocket(); } if ((s == null) || (s.isClosed())) { if (logger.isDebugEnabled()) { logger.debug("Socket for address '" + address + "' is closed, opening it..."); } SocketChannel sc = null; try { // Open the channel, set it to non-blocking, initiate connect sc = SocketChannel.open(); sc.configureBlocking(false); sc.connect( new InetSocketAddress( ((TcpAddress) address).getInetAddress(), ((TcpAddress) address).getPort())); s = sc.socket(); entry = new SocketEntry((TcpAddress) address, s); entry.addMessage(message); sockets.put(address, entry); synchronized (pending) { pending.add(entry); } selector.wakeup(); logger.debug("Trying to connect to " + address); } catch (IOException iox) { logger.error(iox); throw iox; } } else { entry.addMessage(message); synchronized (pending) { pending.add(entry); } selector.wakeup(); } } public void run() { // Here's where everything happens. The select method will // return when any operations registered above have occurred, the // thread has been interrupted, etc. try { while (!stop) { try { if (selector.select() > 0) { if (stop) { break; } // Someone is ready for I/O, get the ready keys Set readyKeys = selector.selectedKeys(); Iterator it = readyKeys.iterator(); // Walk through the ready keys collection and process date requests. while (it.hasNext()) { SelectionKey sk = (SelectionKey) it.next(); it.remove(); SocketChannel readChannel = null; TcpAddress incomingAddress = null; if (sk.isAcceptable()) { // The key indexes into the selector so you // can retrieve the socket that's ready for I/O ServerSocketChannel nextReady = (ServerSocketChannel) sk.channel(); // Accept the date request and send back the date string Socket s = nextReady.accept().socket(); readChannel = s.getChannel(); readChannel.configureBlocking(false); readChannel.register(selector, SelectionKey.OP_READ); incomingAddress = new TcpAddress(s.getInetAddress(), s.getPort()); SocketEntry entry = new SocketEntry(incomingAddress, s); sockets.put(incomingAddress, entry); timeoutSocket(entry); TransportStateEvent e = new TransportStateEvent( DefaultTcpTransportMapping.this, incomingAddress, TransportStateEvent.STATE_CONNECTED, null); fireConnectionStateChanged(e); } else if (sk.isReadable()) { readChannel = (SocketChannel) sk.channel(); incomingAddress = new TcpAddress( readChannel.socket().getInetAddress(), readChannel.socket().getPort()); } else if (sk.isWritable()) { try { SocketEntry entry = (SocketEntry) sk.attachment(); SocketChannel sc = (SocketChannel) sk.channel(); if (entry != null) { writeMessage(entry, sc); } } catch (IOException iox) { if (logger.isDebugEnabled()) { iox.printStackTrace(); } logger.warn(iox); TransportStateEvent e = new TransportStateEvent( DefaultTcpTransportMapping.this, incomingAddress, TransportStateEvent.STATE_DISCONNECTED_REMOTELY, iox); fireConnectionStateChanged(e); sk.cancel(); } } else if (sk.isConnectable()) { try { SocketEntry entry = (SocketEntry) sk.attachment(); SocketChannel sc = (SocketChannel) sk.channel(); if ((!sc.isConnected()) && (sc.finishConnect())) { sc.configureBlocking(false); logger.debug("Connected to " + entry.getPeerAddress()); // make sure conncetion is closed if not used for timeout // micro seconds timeoutSocket(entry); sc.register(selector, SelectionKey.OP_WRITE, entry); } TransportStateEvent e = new TransportStateEvent( DefaultTcpTransportMapping.this, incomingAddress, TransportStateEvent.STATE_CONNECTED, null); fireConnectionStateChanged(e); } catch (IOException iox) { if (logger.isDebugEnabled()) { iox.printStackTrace(); } logger.warn(iox); sk.cancel(); } } if (readChannel != null) { try { readMessage(sk, readChannel, incomingAddress); } catch (IOException iox) { // IO exception -> channel closed remotely if (logger.isDebugEnabled()) { iox.printStackTrace(); } logger.warn(iox); sk.cancel(); readChannel.close(); TransportStateEvent e = new TransportStateEvent( DefaultTcpTransportMapping.this, incomingAddress, TransportStateEvent.STATE_DISCONNECTED_REMOTELY, iox); fireConnectionStateChanged(e); } } } } } catch (NullPointerException npex) { // There seems to happen a NullPointerException within the select() npex.printStackTrace(); logger.warn("NullPointerException within select()?"); } processPending(); } if (ssc != null) { ssc.close(); } } catch (IOException iox) { logger.error(iox); lastError = iox; } if (!stop) { stop = true; synchronized (DefaultTcpTransportMapping.this) { server = null; } } } private void readMessage(SelectionKey sk, SocketChannel readChannel, TcpAddress incomingAddress) throws IOException { // note that socket has been used SocketEntry entry = (SocketEntry) sockets.get(incomingAddress); if (entry != null) { entry.used(); ByteBuffer readBuffer = entry.getReadBuffer(); if (readBuffer != null) { readChannel.read(readBuffer); if (readBuffer.hasRemaining()) { readChannel.register(selector, SelectionKey.OP_READ, entry); } else { dispatchMessage(incomingAddress, readBuffer, readBuffer.capacity()); } return; } } ByteBuffer byteBuffer = ByteBuffer.wrap(buf); byteBuffer.limit(messageLengthDecoder.getMinHeaderLength()); long bytesRead = readChannel.read(byteBuffer); if (logger.isDebugEnabled()) { logger.debug("Reading header " + bytesRead + " bytes from " + incomingAddress); } MessageLength messageLength = new MessageLength(0, Integer.MIN_VALUE); if (bytesRead == messageLengthDecoder.getMinHeaderLength()) { messageLength = messageLengthDecoder.getMessageLength(ByteBuffer.wrap(buf)); if (logger.isDebugEnabled()) { logger.debug("Message length is " + messageLength); } if ((messageLength.getMessageLength() > getMaxInboundMessageSize()) || (messageLength.getMessageLength() <= 0)) { logger.error( "Received message length " + messageLength + " is greater than inboundBufferSize " + getMaxInboundMessageSize()); synchronized (entry) { entry.getSocket().close(); logger.info("Socket to " + entry.getPeerAddress() + " closed due to an error"); } } else { byteBuffer.limit(messageLength.getMessageLength()); bytesRead += readChannel.read(byteBuffer); if (bytesRead == messageLength.getMessageLength()) { dispatchMessage(incomingAddress, byteBuffer, bytesRead); } else { byte[] message = new byte[byteBuffer.limit()]; byteBuffer.flip(); byteBuffer.get(message, 0, byteBuffer.limit() - byteBuffer.remaining()); entry.setReadBuffer(ByteBuffer.wrap(message)); } readChannel.register(selector, SelectionKey.OP_READ, entry); } } else if (bytesRead < 0) { logger.debug("Socket closed remotely"); sk.cancel(); readChannel.close(); TransportStateEvent e = new TransportStateEvent( DefaultTcpTransportMapping.this, incomingAddress, TransportStateEvent.STATE_DISCONNECTED_REMOTELY, null); fireConnectionStateChanged(e); } } private void dispatchMessage( TcpAddress incomingAddress, ByteBuffer byteBuffer, long bytesRead) { byteBuffer.flip(); if (logger.isDebugEnabled()) { logger.debug( "Received message from " + incomingAddress + " with length " + bytesRead + ": " + new OctetString(byteBuffer.array(), 0, (int) bytesRead).toHexString()); } ByteBuffer bis; if (isAsyncMsgProcessingSupported()) { byte[] bytes = new byte[(int) bytesRead]; System.arraycopy(byteBuffer.array(), 0, bytes, 0, (int) bytesRead); bis = ByteBuffer.wrap(bytes); } else { bis = ByteBuffer.wrap(byteBuffer.array(), 0, (int) bytesRead); } fireProcessMessage(incomingAddress, bis); } private void writeMessage(SocketEntry entry, SocketChannel sc) throws IOException { byte[] message = entry.nextMessage(); if (message != null) { ByteBuffer buffer = ByteBuffer.wrap(message); sc.write(buffer); if (logger.isDebugEnabled()) { logger.debug( "Send message with length " + message.length + " to " + entry.getPeerAddress() + ": " + new OctetString(message).toHexString()); } sc.register(selector, SelectionKey.OP_READ); } } public void close() { stop = true; ServerThread st = server; if (st != null) { st.interrupt(); } } } }
/** @author Javier Paniza */ public class MetaFinder implements Serializable { private static Log log = LogFactory.getLog(MetaFinder.class); private static Map argumentsJBoss11ToEJBQL; private static Map argumentsToHQL; private static Map tokensToChangeDollarsAndNL; private String name; private String arguments; private boolean collection; private String condition; private String order; private MetaModel metaModel; public String getArguments() { arguments = Strings.change(arguments, "String", "java.lang.String"); arguments = Strings.change(arguments, "java.lang.java.lang.String", "java.lang.String"); return arguments; } public Collection getMetaPropertiesArguments() throws XavaException { StringTokenizer st = new StringTokenizer(getArguments(), ","); Collection result = new ArrayList(); while (st.hasMoreTokens()) { String argument = st.nextToken(); StringTokenizer argumentSt = new StringTokenizer(argument); String type = argumentSt.nextToken().trim(); String name = argumentSt.nextToken().trim(); MetaProperty p = new MetaProperty(); p.setName(name); p.setTypeName(type); result.add(p); } return result; } public boolean isCollection() { return collection; } public String getCondition() { return condition; } public String getName() { return name; } public void setArguments(String arguments) { this.arguments = arguments; } public void setCollection(boolean collection) { this.collection = collection; } public void setCondition(String condition) { this.condition = condition; } public void setName(String name) { this.name = name; } public boolean isSupportedForEJB2() throws XavaException { return !hasSome3LevelProperty(getCondition()) && !hasSome3LevelProperty(getOrder()); } private boolean hasSome3LevelProperty(String sentence) throws XavaException { if (sentence == null) return false; int i = sentence.indexOf("${"); int f = 0; while (i >= 0) { f = sentence.indexOf("}", i + 2); if (f < 0) break; String property = sentence.substring(i + 2, f); StringTokenizer st = new StringTokenizer(property, "."); if (st.countTokens() > 3) { log.warn(XavaResources.getString("property_3_level_in_ejb2_finder", property, getName())); return true; } if (st.countTokens() == 3) { if (!getMetaModel().getMetaProperty(property).isKey()) { log.warn(XavaResources.getString("property_3_level_in_ejb2_finder", property, getName())); return true; } } i = sentence.indexOf("${", i + 1); } return false; } public String getEJBQLCondition() throws XavaException { StringBuffer sb = new StringBuffer("SELECT OBJECT(o) FROM "); sb.append(getMetaModel().getName()); sb.append(" o"); if (!Is.emptyString(this.condition)) { sb.append(" WHERE "); String attributesCondition = getMetaModel().getMapping().changePropertiesByCMPAttributes(this.condition); sb.append(Strings.change(attributesCondition, getArgumentsJBoss11ToEJBQL())); } if (!Is.emptyString(this.order)) { sb.append(" ORDER BY "); sb.append(getMetaModel().getMapping().changePropertiesByCMPAttributes(this.order)); } return sb.toString(); } public String getHQLCondition() throws XavaException { return getHQLCondition(true); } private String getHQLCondition(boolean order) throws XavaException { StringBuffer sb = new StringBuffer("from "); sb.append(getMetaModel().getName()); sb.append(" as o"); if (!Is.emptyString(this.condition)) { sb.append(" where "); String condition = transformAggregateProperties(getCondition()); condition = Strings.change(condition, getArgumentsToHQL()); sb.append(Strings.change(condition, getTokensToChangeDollarsAndNL())); } if (order && !Is.emptyString(this.order)) { sb.append(" order by "); sb.append( Strings.change( transformAggregateProperties(this.order), getTokensToChangeDollarsAndNL())); } return sb.toString(); } /** * Transforms ${address.street} in ${address_street} if address if an aggregate of container * model. * * @param condition * @return */ private String transformAggregateProperties(String condition) { int i = condition.indexOf("${"); if (i < 0) return condition; StringBuffer result = new StringBuffer(condition.substring(0, i + 2)); while (i >= 0) { int f = condition.indexOf("}", i); String property = condition.substring(i + 2, f); String transformedProperty = transformAgregateProperty(property); result.append(transformedProperty); i = condition.indexOf("${", f); if (i >= 0) result.append(condition.substring(f, i)); else result.append(condition.substring(f)); } return result.toString(); } private String transformAgregateProperty(String property) { StringBuffer result = new StringBuffer(); StringTokenizer st = new StringTokenizer(property, "."); String member = ""; while (st.hasMoreTokens()) { String token = st.nextToken(); result.append(token); if (!st.hasMoreTokens()) break; member = member + token; try { MetaReference ref = getMetaModel().getMetaReference(member); if (ref.isAggregate()) result.append('_'); else result.append('.'); } catch (XavaException ex) { result.append('.'); } member = member + "."; } return result.toString(); } public String getHQLCountSentence() throws XavaException { StringBuffer sb = new StringBuffer("select count(*) "); sb.append(getHQLCondition(false)); return sb.toString(); } public MetaModel getMetaModel() { return metaModel; } public void setMetaModel(MetaModel metaModel) { this.metaModel = metaModel; } public String getOrder() { return order; } public void setOrder(String order) { this.order = order; } private static Map getArgumentsJBoss11ToEJBQL() { if (argumentsJBoss11ToEJBQL == null) { argumentsJBoss11ToEJBQL = new HashMap(); for (int i = 0; i < 30; i++) { argumentsJBoss11ToEJBQL.put("{" + i + "}", "?" + (i + 1)); } } return argumentsJBoss11ToEJBQL; } private static Map getArgumentsToHQL() { if (argumentsToHQL == null) { argumentsToHQL = new HashMap(); for (int i = 0; i < 30; i++) { argumentsToHQL.put("{" + i + "}", ":arg" + i); } } return argumentsToHQL; } static Map getTokensToChangeDollarsAndNL() { if (tokensToChangeDollarsAndNL == null) { tokensToChangeDollarsAndNL = new HashMap(); tokensToChangeDollarsAndNL.put("${", "o."); tokensToChangeDollarsAndNL.put("}", ""); tokensToChangeDollarsAndNL.put("\n", ""); } return tokensToChangeDollarsAndNL; } public boolean equals(Object other) { if (!(other instanceof MetaFinder)) return false; return toString().equals(other.toString()); } public int hashCode() { return toString().hashCode(); } public String toString() { return "Finder: " + getMetaModel().getName() + "." + getName(); } }
/** * Distribute application-specific large, read-only files efficiently. * * <p><code>DistributedCache</code> is a facility provided by the Map-Reduce framework to cache * files (text, archives, jars etc.) needed by applications. * * <p>Applications specify the files, via urls (hdfs:// or http://) to be cached via the {@link * org.apache.hadoop.mapred.JobConf}. The <code>DistributedCache</code> assumes that the files * specified via hdfs:// urls are already present on the {@link FileSystem} at the path specified by * the url. * * <p>The framework will copy the necessary files on to the slave node before any tasks for the job * are executed on that node. Its efficiency stems from the fact that the files are only copied once * per job and the ability to cache archives which are un-archived on the slaves. * * <p><code>DistributedCache</code> can be used to distribute simple, read-only data/text files * and/or more complex types such as archives, jars etc. Archives (zip, tar and tgz/tar.gz files) * are un-archived at the slave nodes. Jars may be optionally added to the classpath of the tasks, a * rudimentary software distribution mechanism. Files have execution permissions. Optionally users * can also direct it to symlink the distributed cache file(s) into the working directory of the * task. * * <p><code>DistributedCache</code> tracks modification timestamps of the cache files. Clearly the * cache files should not be modified by the application or externally while the job is executing. * * <p>Here is an illustrative example on how to use the <code>DistributedCache</code>: * * <p> * * <blockquote> * * <pre> * // Setting up the cache for the application * * 1. Copy the requisite files to the <code>FileSystem</code>: * * $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat * $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip * $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar * $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar * $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz * $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz * * 2. Setup the application's <code>JobConf</code>: * * JobConf job = new JobConf(); * DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"), * job); * DistributedCache.addCacheArchive(new URI("/myapp/map.zip", job); * DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job); * DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar", job); * DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz", job); * DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz", job); * * 3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper} * or {@link org.apache.hadoop.mapred.Reducer}: * * public static class MapClass extends MapReduceBase * implements Mapper<K, V, K, V> { * * private Path[] localArchives; * private Path[] localFiles; * * public void configure(JobConf job) { * // Get the cached archives/files * localArchives = DistributedCache.getLocalCacheArchives(job); * localFiles = DistributedCache.getLocalCacheFiles(job); * } * * public void map(K key, V value, * OutputCollector<K, V> output, Reporter reporter) * throws IOException { * // Use data from the cached archives/files here * // ... * // ... * output.collect(k, v); * } * } * * </pre> * * </blockquote> * * @see org.apache.hadoop.mapred.JobConf * @see org.apache.hadoop.mapred.JobClient */ public class DistributedCache { // cacheID to cacheStatus mapping private static TreeMap<String, CacheStatus> cachedArchives = new TreeMap<String, CacheStatus>(); private static TreeMap<Path, Long> baseDirSize = new TreeMap<Path, Long>(); private static TreeMap<Path, Integer> baseDirNumberSubDir = new TreeMap<Path, Integer>(); // default total cache size private static final long DEFAULT_CACHE_SIZE = 10737418240L; private static final long DEFAULT_CACHE_SUBDIR_LIMIT = 10000; private static final Log LOG = LogFactory.getLog(DistributedCache.class); private static Random random = new Random(); /** * Get the locally cached file or archive; it could either be previously cached (and valid) or * copy it from the {@link FileSystem} now. * * @param cache the cache to be localized, this should be specified as new * URI(hdfs://hostname:port/absolute_path_to_file#LINKNAME). If no schema or hostname:port is * provided the file is assumed to be in the filesystem being used in the Configuration * @param conf The Confguration file which contains the filesystem * @param baseDir The base cache Dir where you wnat to localize the files/archives * @param fileStatus The file status on the dfs. * @param isArchive if the cache is an archive or a file. In case it is an archive with a .zip or * .jar or .tar or .tgz or .tar.gz extension it will be unzipped/unjarred/untarred * automatically and the directory where the archive is unzipped/unjarred/untarred is returned * as the Path. In case of a file, the path to the file is returned * @param confFileStamp this is the hdfs file modification timestamp to verify that the file to be * cached hasn't changed since the job started * @param currentWorkDir this is the directory where you would want to create symlinks for the * locally cached files/archives * @return the path to directory where the archives are unjarred in case of archives, the path to * the file where the file is copied locally * @throws IOException */ public static Path getLocalCache( URI cache, Configuration conf, Path baseDir, FileStatus fileStatus, boolean isArchive, long confFileStamp, Path currentWorkDir, MRAsyncDiskService asyncDiskService) throws IOException { return getLocalCache( cache, conf, baseDir, fileStatus, isArchive, confFileStamp, fileStatus.getLen(), currentWorkDir, true, asyncDiskService, new LocalDirAllocator("mapred.local.dir")); } public static Path getLocalCacheFromTimestamps( URI cache, Configuration conf, Path subDir, FileStatus fileStatus, boolean isArchive, long confFileStamp, long fileLength, Path currentWorkDir, boolean honorSymLinkConf, MRAsyncDiskService asyncDiskService, LocalDirAllocator lDirAllocator) throws IOException { return getLocalCache( cache, conf, subDir, fileStatus, isArchive, confFileStamp, fileLength, currentWorkDir, honorSymLinkConf, asyncDiskService, lDirAllocator); } public static Path getLocalCacheFromURI( URI cache, Configuration conf, Path subDir, boolean isArchive, long fileLength, Path currentWorkDir, boolean honorSymLinkConf, MRAsyncDiskService asyncDiskService, LocalDirAllocator lDirAllocator) throws IOException { return getLocalCache( cache, conf, subDir, null, isArchive, 0, fileLength, currentWorkDir, honorSymLinkConf, asyncDiskService, lDirAllocator); } /** Added for back compatibility. */ public static Path getLocalCache( URI cache, Configuration conf, Path subdir, FileStatus fileStatus, boolean isArchive, long confFileStamp, Path currentWorkDir, boolean honorSymLinkConf, MRAsyncDiskService asyncDiskService, LocalDirAllocator lDirAllocator) throws IOException { return getLocalCache( cache, conf, subdir, fileStatus, isArchive, confFileStamp, fileStatus.getLen(), currentWorkDir, honorSymLinkConf, asyncDiskService, lDirAllocator); } /** * Get the locally cached file or archive; it could either be previously cached (and valid) or * copy it from the {@link FileSystem} now. * * @param cache the cache to be localized, this should be specified as new * URI(hdfs://hostname:port/absolute_path_to_file#LINKNAME). If no schema or hostname:port is * provided the file is assumed to be in the filesystem being used in the Configuration * @param conf The Confguration file which contains the filesystem * @param subDir The sub cache Dir where you want to localize the files/archives * @param fileStatus The file status on the dfs. * @param isArchive if the cache is an archive or a file. In case it is an archive with a .zip or * .jar or .tar or .tgz or .tar.gz extension it will be unzipped/unjarred/untarred * automatically and the directory where the archive is unzipped/unjarred/untarred is returned * as the Path. In case of a file, the path to the file is returned * @param confFileStamp this is the hdfs file modification timestamp to verify that the file to be * cached hasn't changed since the job started * @param fileLength this is the length of the cache file * @param currentWorkDir this is the directory where you would want to create symlinks for the * locally cached files/archives * @param honorSymLinkConf if this is false, then the symlinks are not created even if conf says * so (this is required for an optimization in task launches * @param lDirAllocator LocalDirAllocator of the tracker * @return the path to directory where the archives are unjarred in case of archives, the path to * the file where the file is copied locally * @throws IOException */ private static Path getLocalCache( URI cache, Configuration conf, Path subDir, FileStatus fileStatus, boolean isArchive, long confFileStamp, long fileLength, Path currentWorkDir, boolean honorSymLinkConf, MRAsyncDiskService asyncDiskService, LocalDirAllocator lDirAllocator) throws IOException { String key = getKey(cache, conf, confFileStamp); CacheStatus lcacheStatus; Path localizedPath; synchronized (cachedArchives) { lcacheStatus = cachedArchives.get(key); if (lcacheStatus == null) { // was never localized Path uniqueParentDir = new Path(subDir, String.valueOf(random.nextLong())); String cachePath = new Path(uniqueParentDir, makeRelative(cache, conf)).toString(); Path localPath = lDirAllocator.getLocalPathForWrite(cachePath, fileLength, conf); lcacheStatus = new CacheStatus( new Path(localPath.toString().replace(cachePath, "")), localPath, uniqueParentDir); cachedArchives.put(key, lcacheStatus); } lcacheStatus.refcount++; } boolean initSuccessful = false; try { synchronized (lcacheStatus) { if (!lcacheStatus.isInited()) { localizedPath = localizeCache(conf, cache, confFileStamp, lcacheStatus, isArchive); lcacheStatus.initComplete(); } else { if (fileStatus != null) { localizedPath = checkCacheStatusValidity( conf, cache, confFileStamp, lcacheStatus, fileStatus, isArchive); } else { // if fileStatus is null, then the md5 must be correct // so there is no need to check for cache validity localizedPath = lcacheStatus.localizedLoadPath; } } createSymlink(conf, cache, lcacheStatus, isArchive, currentWorkDir, honorSymLinkConf); } // try deleting stuff if you can long size = 0; int numberSubDir = 0; synchronized (lcacheStatus) { synchronized (baseDirSize) { Long get = baseDirSize.get(lcacheStatus.getBaseDir()); if (get != null) { size = get.longValue(); } else { LOG.warn("Cannot find size of baseDir: " + lcacheStatus.getBaseDir()); } } synchronized (baseDirNumberSubDir) { Integer get = baseDirNumberSubDir.get(lcacheStatus.getBaseDir()); if (get != null) { numberSubDir = get.intValue(); } else { LOG.warn("Cannot find subdirectories limit of baseDir: " + lcacheStatus.getBaseDir()); } } } // setting the cache size to a default of 10GB long allowedSize = conf.getLong("local.cache.size", DEFAULT_CACHE_SIZE); long allowedNumberSubDir = conf.getLong("local.cache.numbersubdir", DEFAULT_CACHE_SUBDIR_LIMIT); if (allowedSize < size || allowedNumberSubDir < numberSubDir) { // try some cache deletions LOG.debug( "Start deleting released cache because" + " [size, allowedSize, numberSubDir, allowedNumberSubDir] =" + " [" + size + ", " + allowedSize + ", " + numberSubDir + ", " + allowedNumberSubDir + "]"); deleteCache(conf, asyncDiskService); } initSuccessful = true; return localizedPath; } finally { if (!initSuccessful) { synchronized (cachedArchives) { lcacheStatus.refcount--; } } } } /** * Get the locally cached file or archive; it could either be previously cached (and valid) or * copy it from the {@link FileSystem} now. * * @param cache the cache to be localized, this should be specified as new * URI(hdfs://hostname:port/absolute_path_to_file#LINKNAME). If no schema or hostname:port is * provided the file is assumed to be in the filesystem being used in the Configuration * @param conf The Confguration file which contains the filesystem * @param baseDir The base cache Dir where you wnat to localize the files/archives * @param isArchive if the cache is an archive or a file. In case it is an archive with a .zip or * .jar or .tar or .tgz or .tar.gz extension it will be unzipped/unjarred/untarred * automatically and the directory where the archive is unzipped/unjarred/untarred is returned * as the Path. In case of a file, the path to the file is returned * @param confFileStamp this is the hdfs file modification timestamp to verify that the file to be * cached hasn't changed since the job started * @param currentWorkDir this is the directory where you would want to create symlinks for the * locally cached files/archives * @return the path to directory where the archives are unjarred in case of archives, the path to * the file where the file is copied locally * @throws IOException */ public static Path getLocalCache( URI cache, Configuration conf, Path baseDir, boolean isArchive, long confFileStamp, Path currentWorkDir, MRAsyncDiskService asyncDiskService) throws IOException { return getLocalCache( cache, conf, baseDir, null, isArchive, confFileStamp, currentWorkDir, asyncDiskService); } /** * This is the opposite of getlocalcache. When you are done with using the cache, you need to * release the cache * * @param cache The cache URI to be released * @param conf configuration which contains the filesystem the cache is contained in. * @throws IOException */ public static void releaseCache(URI cache, Configuration conf, long timeStamp) throws IOException { String cacheId = getKey(cache, conf, timeStamp); synchronized (cachedArchives) { CacheStatus lcacheStatus = cachedArchives.get(cacheId); if (lcacheStatus == null) { LOG.warn( "Cannot find localized cache: " + cache + " (key: " + cacheId + ") in releaseCache!"); return; } lcacheStatus.refcount--; } } /** Runnable which removes the cache directories from the disk */ private static class CacheFileCleanTask implements Runnable { private MRAsyncDiskService asyncDiskService; private LocalFileSystem fs; private List<CacheStatus> toBeDeletedCache; public CacheFileCleanTask( MRAsyncDiskService asyncDiskService, LocalFileSystem fs, List<CacheStatus> toBeDeletedCache) { this.asyncDiskService = asyncDiskService; this.fs = fs; this.toBeDeletedCache = toBeDeletedCache; } @Override public void run() { for (CacheStatus lcacheStatus : toBeDeletedCache) { synchronized (lcacheStatus) { Path fullUniqueParentDir = new Path(lcacheStatus.localizedBaseDir, lcacheStatus.uniqueParentDir); try { LOG.info("Deleting local cached path: " + fullUniqueParentDir.toString()); deleteLocalPath(asyncDiskService, fs, fullUniqueParentDir); // decrement the size of the cache from baseDirSize deleteCacheInfoUpdate(lcacheStatus); LOG.info("Removed cache " + lcacheStatus.localizedLoadPath); } catch (IOException e) { LOG.warn("Error when deleting " + fullUniqueParentDir, e); } } } } } // To delete the caches which have a refcount of zero private static void deleteCache(Configuration conf, MRAsyncDiskService asyncDiskService) throws IOException { List<CacheStatus> deleteSet = new LinkedList<CacheStatus>(); // try deleting cache Status with refcount of zero synchronized (cachedArchives) { for (Iterator<String> it = cachedArchives.keySet().iterator(); it.hasNext(); ) { String cacheId = (String) it.next(); CacheStatus lcacheStatus = cachedArchives.get(cacheId); if (lcacheStatus.refcount == 0) { // delete this cache entry from the global list // and mark the localized file for deletion deleteSet.add(lcacheStatus); it.remove(); } } } // do the deletion asynchronously, after releasing the global lock Thread cacheFileCleaner = new Thread(new CacheFileCleanTask(asyncDiskService, FileSystem.getLocal(conf), deleteSet)); cacheFileCleaner.start(); } /** * Delete a local path with asyncDiskService if available, or otherwise synchronously with local * file system. */ private static void deleteLocalPath( MRAsyncDiskService asyncDiskService, LocalFileSystem fs, Path path) throws IOException { boolean deleted = false; if (asyncDiskService != null) { // Try to delete using asyncDiskService String localPathToDelete = path.toUri().getPath(); deleted = asyncDiskService.moveAndDeleteAbsolutePath(localPathToDelete); if (!deleted) { LOG.warn( "Cannot find DistributedCache path " + localPathToDelete + " on any of the asyncDiskService volumes!"); } } if (!deleted) { // If no asyncDiskService, we will delete the files synchronously fs.delete(path, true); } LOG.info("Deleted path " + path); } /* * Returns the relative path of the dir this cache will be localized in * relative path that this cache will be localized in. For * hdfs://hostname:port/absolute_path -- the relative path is * hostname/absolute path -- if it is just /absolute_path -- then the * relative path is hostname of DFS this mapred cluster is running * on/absolute_path */ public static String makeRelative(URI cache, Configuration conf) throws IOException { String host = cache.getHost(); if (host == null) { host = cache.getScheme(); } if (host == null) { URI defaultUri = FileSystem.get(conf).getUri(); host = defaultUri.getHost(); if (host == null) { host = defaultUri.getScheme(); } } String path = host + cache.getPath(); path = path.replace(":/", "/"); // remove windows device colon return path; } static String getKey(URI cache, Configuration conf, long timeStamp) throws IOException { return makeRelative(cache, conf) + String.valueOf(timeStamp); } private static Path checkCacheStatusValidity( Configuration conf, URI cache, long confFileStamp, CacheStatus cacheStatus, FileStatus fileStatus, boolean isArchive) throws IOException { FileSystem fs = FileSystem.get(cache, conf); // Has to be if (!ifExistsAndFresh(conf, fs, cache, confFileStamp, cacheStatus, fileStatus)) { throw new IOException( "Stale cache file: " + cacheStatus.localizedLoadPath + " for cache-file: " + cache); } LOG.info( String.format( "Using existing cache of %s->%s", cache.toString(), cacheStatus.localizedLoadPath)); return cacheStatus.localizedLoadPath; } private static void createSymlink( Configuration conf, URI cache, CacheStatus cacheStatus, boolean isArchive, Path currentWorkDir, boolean honorSymLinkConf) throws IOException { boolean doSymlink = honorSymLinkConf && DistributedCache.getSymlink(conf); if (cache.getFragment() == null) { doSymlink = false; } String link = currentWorkDir.toString() + Path.SEPARATOR + cache.getFragment(); File flink = new File(link); if (doSymlink) { if (!flink.exists()) { FileUtil.symLink(cacheStatus.localizedLoadPath.toString(), link); } } } // the method which actually copies the caches locally and unjars/unzips them // and does chmod for the files private static Path localizeCache( Configuration conf, URI cache, long confFileStamp, CacheStatus cacheStatus, boolean isArchive) throws IOException { FileSystem fs = getFileSystem(cache, conf); FileSystem localFs = FileSystem.getLocal(conf); Path parchive = null; if (isArchive) { parchive = new Path( cacheStatus.localizedLoadPath, new Path(cacheStatus.localizedLoadPath.getName())); } else { parchive = cacheStatus.localizedLoadPath; } if (!localFs.mkdirs(parchive.getParent())) { throw new IOException( "Mkdirs failed to create directory " + cacheStatus.localizedLoadPath.toString()); } String cacheId = cache.getPath(); fs.copyToLocalFile(new Path(cacheId), parchive); if (isArchive) { String tmpArchive = parchive.toString().toLowerCase(); File srcFile = new File(parchive.toString()); File destDir = new File(parchive.getParent().toString()); if (tmpArchive.endsWith(".jar")) { RunJar.unJar(srcFile, destDir); } else if (tmpArchive.endsWith(".zip")) { FileUtil.unZip(srcFile, destDir); } else if (isTarFile(tmpArchive)) { FileUtil.unTar(srcFile, destDir); } // else will not do anyhting // and copy the file into the dir as it is } long cacheSize = FileUtil.getDU(new File(parchive.getParent().toString())); cacheStatus.size = cacheSize; addCacheInfoUpdate(cacheStatus); // do chmod here try { // Setting recursive permission to grant everyone read and execute Path localDir = new Path(cacheStatus.localizedBaseDir, cacheStatus.uniqueParentDir); LOG.info("Doing chmod on localdir :" + localDir); FileUtil.chmod(localDir.toString(), "ugo+rx", true); } catch (InterruptedException e) { LOG.warn("Exception in chmod" + e.toString()); } // update cacheStatus to reflect the newly cached file cacheStatus.mtime = getTimestamp(conf, cache); return cacheStatus.localizedLoadPath; } private static boolean isTarFile(String filename) { return (filename.endsWith(".tgz") || filename.endsWith(".tar.gz") || filename.endsWith(".tar")); } // Checks if the cache has already been localized and is fresh private static boolean ifExistsAndFresh( Configuration conf, FileSystem fs, URI cache, long confFileStamp, CacheStatus lcacheStatus, FileStatus fileStatus) throws IOException { // check for existence of the cache long dfsFileStamp; if (fileStatus != null) { dfsFileStamp = fileStatus.getModificationTime(); } else { dfsFileStamp = getTimestamp(conf, cache); } // ensure that the file on hdfs hasn't been modified since the job started if (dfsFileStamp != confFileStamp) { LOG.fatal("File: " + cache + " has changed on HDFS since job started"); throw new IOException("File: " + cache + " has changed on HDFS since job started"); } if (dfsFileStamp != lcacheStatus.mtime) { // needs refreshing return false; } return true; } /** * Returns mtime of a given cache file on hdfs. * * @param conf configuration * @param cache cache file * @return mtime of a given cache file on hdfs * @throws IOException */ public static long getTimestamp(Configuration conf, URI cache) throws IOException { FileSystem fileSystem = FileSystem.get(cache, conf); Path filePath = new Path(cache.getPath()); return fileSystem.getFileStatus(filePath).getModificationTime(); } /** * Returns the status of a given cache file on hdfs. * * @param conf configuration * @param cache cache file * @return FileStatus object of the file * @throws IOException */ public static FileStatus getFileStatus(Configuration conf, URI cache) throws IOException { FileSystem fileSystem = FileSystem.get(cache, conf); Path filePath = new Path(cache.getPath()); return fileSystem.getFileStatus(filePath); } /** * This method create symlinks for all files in a given dir in another directory * * @param conf the configuration * @param jobCacheDir the target directory for creating symlinks * @param workDir the directory in which the symlinks are created * @throws IOException */ public static void createAllSymlink(Configuration conf, File jobCacheDir, File workDir) throws IOException { if ((jobCacheDir == null || !jobCacheDir.isDirectory()) || workDir == null || (!workDir.isDirectory())) { return; } boolean createSymlink = getSymlink(conf); if (createSymlink) { File[] list = jobCacheDir.listFiles(); for (int i = 0; i < list.length; i++) { FileUtil.symLink( list[i].getAbsolutePath(), new File(workDir, list[i].getName()).toString()); } } } private static String getFileSysName(URI url) { String fsname = url.getScheme(); if ("hdfs".equals(fsname)) { String host = url.getHost(); int port = url.getPort(); return (port == (-1)) ? host : (host + ":" + port); } else { return null; } } private static FileSystem getFileSystem(URI cache, Configuration conf) throws IOException { String fileSysName = getFileSysName(cache); if (fileSysName != null) return FileSystem.getNamed(fileSysName, conf); else return FileSystem.get(conf); } /** * Set the configuration with the given set of archives * * @param archives The list of archives that need to be localized * @param conf Configuration which will be changed */ public static void setCacheArchives(URI[] archives, Configuration conf) { String sarchives = StringUtils.uriToString(archives); conf.set("mapred.cache.archives", sarchives); } /** * Set the configuration with the given set of files * * @param files The list of files that need to be localized * @param conf Configuration which will be changed */ public static void setCacheFiles(URI[] files, Configuration conf) { String sfiles = StringUtils.uriToString(files); conf.set("mapred.cache.files", sfiles); } /** * Get cache archives set in the Configuration * * @param conf The configuration which contains the archives * @return A URI array of the caches set in the Configuration * @throws IOException */ public static URI[] getCacheArchives(Configuration conf) throws IOException { return StringUtils.stringToURI(conf.getStrings("mapred.cache.archives")); } /** * Get cache archives set in the Configuration * * @param conf The configuration which contains the archives * @return A URI array of the caches set in the Configuration * @throws IOException */ public static URI[] getSharedCacheArchives(Configuration conf) throws IOException { return StringUtils.stringToURI(conf.getStrings("mapred.cache.shared.archives")); } /** * Get cache files set in the Configuration * * @param conf The configuration which contains the files * @return A URI array of the files set in the Configuration * @throws IOException */ public static URI[] getCacheFiles(Configuration conf) throws IOException { return StringUtils.stringToURI(conf.getStrings("mapred.cache.files")); } /** * Get cache files set in the Configuration * * @param conf The configuration which contains the files * @return A URI array of the files set in the Configuration * @throws IOException */ public static URI[] getSharedCacheFiles(Configuration conf) throws IOException { return StringUtils.stringToURI(conf.getStrings("mapred.cache.shared.files")); } /** * Return the path array of the localized caches * * @param conf Configuration that contains the localized archives * @return A path array of localized caches * @throws IOException */ public static Path[] getLocalCacheArchives(Configuration conf) throws IOException { return StringUtils.stringToPath(conf.getStrings("mapred.cache.localArchives")); } /** * Return the path array of the localized caches * * @param conf Configuration that contains the localized archives * @return A path array of localized caches * @throws IOException */ public static Path[] getLocalSharedCacheArchives(Configuration conf) throws IOException { return StringUtils.stringToPath(conf.getStrings("mapred.cache.shared.localArchives")); } /** * Return the path array of the localized files * * @param conf Configuration that contains the localized files * @return A path array of localized files * @throws IOException */ public static Path[] getLocalCacheFiles(Configuration conf) throws IOException { return StringUtils.stringToPath(conf.getStrings("mapred.cache.localFiles")); } /** * Return the path array of the localized files * * @param conf Configuration that contains the localized files * @return A path array of localized files * @throws IOException */ public static Path[] getLocalSharedCacheFiles(Configuration conf) throws IOException { return StringUtils.stringToPath(conf.getStrings("mapred.cache.shared.localFiles")); } /** * Get the timestamps of the archives * * @param conf The configuration which stored the timestamps * @return a string array of timestamps * @throws IOException */ public static String[] getArchiveTimestamps(Configuration conf) { return conf.getStrings("mapred.cache.archives.timestamps"); } /** * Get the timestamps of the files * * @param conf The configuration which stored the timestamps * @return a string array of timestamps * @throws IOException */ public static String[] getFileTimestamps(Configuration conf) { return conf.getStrings("mapred.cache.files.timestamps"); } public static String[] getSharedArchiveLength(Configuration conf) { return conf.getStrings("mapred.cache.shared.archives.length"); } public static String[] getSharedFileLength(Configuration conf) { return conf.getStrings("mapred.cache.shared.files.length"); } /** * This is to check the timestamp of the archives to be localized * * @param conf Configuration which stores the timestamp's * @param timestamps comma separated list of timestamps of archives. The order should be the same * as the order in which the archives are added. */ public static void setArchiveTimestamps(Configuration conf, String timestamps) { conf.set("mapred.cache.archives.timestamps", timestamps); } public static void setSharedArchiveLength(Configuration conf, String length) { conf.set("mapred.cache.shared.archives.length", length); } /** * This is to check the timestamp of the files to be localized * * @param conf Configuration which stores the timestamp's * @param timestamps comma separated list of timestamps of files. The order should be the same as * the order in which the files are added. */ public static void setFileTimestamps(Configuration conf, String timestamps) { conf.set("mapred.cache.files.timestamps", timestamps); } public static void setSharedFileLength(Configuration conf, String length) { conf.set("mapred.cache.shared.files.length", length); } /** * Set the conf to contain the location for localized archives * * @param conf The conf to modify to contain the localized caches * @param str a comma separated list of local archives */ public static void setLocalArchives(Configuration conf, String str) { conf.set("mapred.cache.localArchives", str); } /** * Set the conf to contain the location for localized archives * * @param conf The conf to modify to contain the localized caches * @param str a comma separated list of local archives */ public static void setLocalSharedArchives(Configuration conf, String str) { conf.set("mapred.cache.shared.localArchives", str); } /** * Set the conf to contain the location for localized files * * @param conf The conf to modify to contain the localized caches * @param str a comma separated list of local files */ public static void setLocalFiles(Configuration conf, String str) { conf.set("mapred.cache.localFiles", str); } /** * Set the conf to contain the location for localized files * * @param conf The conf to modify to contain the localized caches * @param str a comma separated list of local files */ public static void setLocalSharedFiles(Configuration conf, String str) { conf.set("mapred.cache.shared.localFiles", str); } /** * Add a archives to be localized to the conf * * @param uri The uri of the cache to be localized * @param conf Configuration to add the cache to */ public static void addCacheArchive(URI uri, Configuration conf) { String archives = conf.get("mapred.cache.archives"); conf.set( "mapred.cache.archives", archives == null ? uri.toString() : archives + "," + uri.toString()); } /** * Add a archives to be localized to the conf * * @param uri The uri of the cache to be localized * @param conf Configuration to add the cache to */ public static void addSharedCacheArchive(URI uri, Configuration conf) { String archives = conf.get("mapred.cache.shared.archives"); conf.set( "mapred.cache.shared.archives", archives == null ? uri.toString() : archives + "," + uri.toString()); } /** * Add a file to be localized to the conf * * @param uri The uri of the cache to be localized * @param conf Configuration to add the cache to */ public static void addCacheFile(URI uri, Configuration conf) { String files = conf.get("mapred.cache.files"); conf.set("mapred.cache.files", files == null ? uri.toString() : files + "," + uri.toString()); } /** * Add a file to be localized to the conf * * @param uri The uri of the cache to be localized * @param conf Configuration to add the cache to */ public static void addSharedCacheFile(URI uri, Configuration conf) { String files = conf.get("mapred.cache.shared.files"); conf.set( "mapred.cache.shared.files", files == null ? uri.toString() : files + "," + uri.toString()); } /** * Add an file path to the current set of classpath entries It adds the file to cache as well. * * @param file Path of the file to be added * @param conf Configuration that contains the classpath setting */ public static void addFileToClassPath(Path file, Configuration conf) throws IOException { String classpath = conf.get("mapred.job.classpath.files"); conf.set( "mapred.job.classpath.files", classpath == null ? file.toString() : classpath + System.getProperty("path.separator") + file.toString()); URI uri = file.makeQualified(file.getFileSystem(conf)).toUri(); addCacheFile(uri, conf); } /** * Get the file entries in classpath as an array of Path * * @param conf Configuration that contains the classpath setting */ public static Path[] getFileClassPaths(Configuration conf) { String classpath = conf.get("mapred.job.classpath.files"); if (classpath == null) return null; ArrayList list = Collections.list(new StringTokenizer(classpath, System.getProperty("path.separator"))); Path[] paths = new Path[list.size()]; for (int i = 0; i < list.size(); i++) { paths[i] = new Path((String) list.get(i)); } return paths; } private static URI addArchiveToClassPathHelper(Path archive, Configuration conf) throws IOException { String classpath = conf.get("mapred.job.classpath.archives"); // the scheme/authority use ':' as separator. put the unqualified path in classpath String archivePath = archive.toUri().getPath(); conf.set( "mapred.job.classpath.archives", classpath == null ? archivePath : classpath + System.getProperty("path.separator") + archivePath); return archive.makeQualified(archive.getFileSystem(conf)).toUri(); } /** * Add an archive path to the current set of classpath entries. It adds the archive to cache as * well. * * @param archive Path of the archive to be added * @param conf Configuration that contains the classpath setting */ public static void addArchiveToClassPath(Path archive, Configuration conf) throws IOException { URI uri = addArchiveToClassPathHelper(archive, conf); addCacheArchive(uri, conf); } /** * Add an archive path to the current set of classpath entries. It adds the archive to cache as * well. * * @param archive Path of the archive to be added * @param conf Configuration that contains the classpath setting */ public static void addSharedArchiveToClassPath(Path archive, Configuration conf) throws IOException { URI uri = addArchiveToClassPathHelper(archive, conf); addSharedCacheArchive(uri, conf); } /** * Get the archive entries in classpath as an array of Path * * @param conf Configuration that contains the classpath setting */ public static Path[] getArchiveClassPaths(Configuration conf) { String classpath = conf.get("mapred.job.classpath.archives"); if (classpath == null) return null; ArrayList list = Collections.list(new StringTokenizer(classpath, System.getProperty("path.separator"))); Path[] paths = new Path[list.size()]; for (int i = 0; i < list.size(); i++) { paths[i] = new Path((String) list.get(i)); } return paths; } /** * This method allows you to create symlinks in the current working directory of the task to all * the cache files/archives * * @param conf the jobconf */ public static void createSymlink(Configuration conf) { conf.set("mapred.create.symlink", "yes"); } /** * This method checks to see if symlinks are to be create for the localized cache files in the * current working directory * * @param conf the jobconf * @return true if symlinks are to be created- else return false */ public static boolean getSymlink(Configuration conf) { String result = conf.get("mapred.create.symlink"); if ("yes".equals(result)) { return true; } return false; } /** * This method checks if there is a conflict in the fragment names of the uris. Also makes sure * that each uri has a fragment. It is only to be called if you want to create symlinks for the * various archives and files. * * @param uriFiles The uri array of urifiles * @param uriArchives the uri array of uri archives */ public static boolean checkURIs(URI[] uriFiles, URI[] uriArchives) { if ((uriFiles == null) && (uriArchives == null)) { return true; } if (uriFiles != null) { for (int i = 0; i < uriFiles.length; i++) { String frag1 = uriFiles[i].getFragment(); if (frag1 == null) return false; for (int j = i + 1; j < uriFiles.length; j++) { String frag2 = uriFiles[j].getFragment(); if (frag2 == null) return false; if (frag1.equalsIgnoreCase(frag2)) return false; } if (uriArchives != null) { for (int j = 0; j < uriArchives.length; j++) { String frag2 = uriArchives[j].getFragment(); if (frag2 == null) { return false; } if (frag1.equalsIgnoreCase(frag2)) return false; for (int k = j + 1; k < uriArchives.length; k++) { String frag3 = uriArchives[k].getFragment(); if (frag3 == null) return false; if (frag2.equalsIgnoreCase(frag3)) return false; } } } } } return true; } private static class CacheStatus { // the local load path of this cache Path localizedLoadPath; // the base dir where the cache lies Path localizedBaseDir; // the unique directory in localizedBaseDir, where the cache lies Path uniqueParentDir; // the size of this cache long size; // number of instances using this cache int refcount; // the cache-file modification time long mtime; // is it initialized boolean inited = false; public CacheStatus(Path baseDir, Path localLoadPath, Path uniqueParentDir) { super(); this.localizedLoadPath = localLoadPath; this.refcount = 0; this.mtime = -1; this.localizedBaseDir = baseDir; this.size = 0; this.uniqueParentDir = uniqueParentDir; } // get the base dir for the cache Path getBaseDir() { return localizedBaseDir; } // Is it initialized? boolean isInited() { return inited; } // mark it as initalized void initComplete() { inited = true; } } /** * Clear the entire contents of the cache and delete the backing files. This should only be used * when the server is reinitializing, because the users are going to lose their files. */ public static void purgeCache(Configuration conf, MRAsyncDiskService service) throws IOException { synchronized (cachedArchives) { LocalFileSystem localFs = FileSystem.getLocal(conf); for (Map.Entry<String, CacheStatus> f : cachedArchives.entrySet()) { try { deleteLocalPath(service, localFs, f.getValue().localizedLoadPath); } catch (IOException ie) { LOG.debug("Error cleaning up cache", ie); } } cachedArchives.clear(); } } /** * Update the maps baseDirSize and baseDirNumberSubDir when deleting cache. * * @param cacheStatus cache status of the cache is deleted */ private static void deleteCacheInfoUpdate(CacheStatus cacheStatus) { if (!cacheStatus.isInited()) { // if it is not created yet, do nothing. return; } synchronized (baseDirSize) { Long dirSize = baseDirSize.get(cacheStatus.getBaseDir()); if (dirSize != null) { dirSize -= cacheStatus.size; baseDirSize.put(cacheStatus.getBaseDir(), dirSize); } } synchronized (baseDirNumberSubDir) { Integer dirSubDir = baseDirNumberSubDir.get(cacheStatus.getBaseDir()); if (dirSubDir != null) { dirSubDir--; baseDirNumberSubDir.put(cacheStatus.getBaseDir(), dirSubDir); } } } /** * Update the maps baseDirSize and baseDirNumberSubDir when adding cache. * * @param cacheStatus cache status of the cache is added */ private static void addCacheInfoUpdate(CacheStatus cacheStatus) { long cacheSize = cacheStatus.size; synchronized (baseDirSize) { Long dirSize = baseDirSize.get(cacheStatus.getBaseDir()); if (dirSize == null) { dirSize = Long.valueOf(cacheSize); } else { dirSize += cacheSize; } baseDirSize.put(cacheStatus.getBaseDir(), dirSize); } synchronized (baseDirNumberSubDir) { Integer dirSubDir = baseDirNumberSubDir.get(cacheStatus.getBaseDir()); if (dirSubDir == null) { dirSubDir = 1; } else { dirSubDir += 1; } baseDirNumberSubDir.put(cacheStatus.getBaseDir(), dirSubDir); } } }
/** * Shared functionality for hadoopStreaming formats. A custom reader can be defined to be a * RecordReader with the constructor below and is selected with the option bin/hadoopStreaming * -inputreader ... * * @see StreamXmlRecordReader */ public abstract class StreamBaseRecordReader implements RecordReader<Text, Text> { protected static final Log LOG = LogFactory.getLog(StreamBaseRecordReader.class.getName()); // custom JobConf properties for this class are prefixed with this namespace static final String CONF_NS = "stream.recordreader."; public StreamBaseRecordReader( FSDataInputStream in, FileSplit split, Reporter reporter, JobConf job, FileSystem fs) throws IOException { in_ = in; split_ = split; start_ = split_.getStart(); length_ = split_.getLength(); end_ = start_ + length_; splitName_ = split_.getPath().getName(); reporter_ = reporter; job_ = job; fs_ = fs; statusMaxRecordChars_ = job_.getInt(CONF_NS + "statuschars", 200); } /// RecordReader API /** Read a record. Implementation should call numRecStats at the end */ public abstract boolean next(Text key, Text value) throws IOException; /** This implementation always returns true. */ public void validateInput(JobConf job) throws IOException {} /** Returns the current position in the input. */ public synchronized long getPos() throws IOException { return in_.getPos(); } /** Close this to future operations. */ public synchronized void close() throws IOException { in_.close(); } public float getProgress() throws IOException { if (end_ == start_) { return 1.0f; } else { return ((float) (in_.getPos() - start_)) / ((float) (end_ - start_)); } } public Text createKey() { return new Text(); } public Text createValue() { return new Text(); } /// StreamBaseRecordReader API /** * Implementation should seek forward in_ to the first byte of the next record. The initial byte * offset in the stream is arbitrary. */ public abstract void seekNextRecordBoundary() throws IOException; void numRecStats(byte[] record, int start, int len) throws IOException { numRec_++; if (numRec_ == nextStatusRec_) { String recordStr = new String(record, start, Math.min(len, statusMaxRecordChars_), "UTF-8"); nextStatusRec_ += 100; // *= 10; String status = getStatus(recordStr); LOG.info(status); reporter_.setStatus(status); } } long lastMem = 0; String getStatus(CharSequence record) { long pos = -1; try { pos = getPos(); } catch (IOException io) { } String recStr; if (record.length() > statusMaxRecordChars_) { recStr = record.subSequence(0, statusMaxRecordChars_) + "..."; } else { recStr = record.toString(); } String unqualSplit = split_.getPath().getName() + ":" + split_.getStart() + "+" + split_.getLength(); String status = "HSTR " + StreamUtil.HOST + " " + numRec_ + ". pos=" + pos + " " + unqualSplit + " Processing record=" + recStr; status += " " + splitName_; return status; } FSDataInputStream in_; FileSplit split_; long start_; long end_; long length_; String splitName_; Reporter reporter_; JobConf job_; FileSystem fs_; int numRec_ = 0; int nextStatusRec_ = 1; int statusMaxRecordChars_; }
private static class AllocatorPerContext { private final Log LOG = LogFactory.getLog(AllocatorPerContext.class); private int dirNumLastAccessed; private Random dirIndexRandomizer = new Random(); private FileSystem localFS; private DF[] dirDF; private String contextCfgItemName; private Path[] localDirsPath; private String savedLocalDirs = ""; public AllocatorPerContext(String contextCfgItemName) { this.contextCfgItemName = contextCfgItemName; } /** * This method gets called everytime before any read/write to make sure that any change to * localDirs is reflected immediately. */ private synchronized void confChanged(Configuration conf) throws IOException { String newLocalDirs = conf.get(contextCfgItemName); if (!newLocalDirs.equals(savedLocalDirs)) { String[] localDirs = conf.getStrings(contextCfgItemName); localFS = FileSystem.getLocal(conf); int numDirs = localDirs.length; ArrayList<String> dirs = new ArrayList<String>(numDirs); ArrayList<DF> dfList = new ArrayList<DF>(numDirs); for (int i = 0; i < numDirs; i++) { try { // filter problematic directories Path tmpDir = new Path(localDirs[i]); if (localFS.mkdirs(tmpDir) || localFS.exists(tmpDir)) { try { DiskChecker.checkDir(new File(localDirs[i])); dirs.add(localDirs[i]); dfList.add(new DF(new File(localDirs[i]), 30000)); } catch (DiskErrorException de) { LOG.warn(localDirs[i] + "is not writable\n" + StringUtils.stringifyException(de)); } } else { LOG.warn("Failed to create " + localDirs[i]); } } catch (IOException ie) { LOG.warn( "Failed to create " + localDirs[i] + ": " + ie.getMessage() + "\n" + StringUtils.stringifyException(ie)); } // ignore } localDirsPath = new Path[dirs.size()]; for (int i = 0; i < localDirsPath.length; i++) { localDirsPath[i] = new Path(dirs.get(i)); } dirDF = dfList.toArray(new DF[dirs.size()]); savedLocalDirs = newLocalDirs; // randomize the first disk picked in the round-robin selection dirNumLastAccessed = dirIndexRandomizer.nextInt(dirs.size()); } } private Path createPath(Path path, boolean checkWrite) throws IOException { Path file = new Path(localDirsPath[dirNumLastAccessed], path); if (checkWrite) { // check whether we are able to create a directory here. If the disk // happens to be RDONLY we will fail try { DiskChecker.checkDir(new File(file.getParent().toUri().getPath())); } catch (DiskErrorException d) { LOG.warn(StringUtils.stringifyException(d)); return null; } } return file; } /** * Get the current directory index. * * @return the current directory index. */ int getCurrentDirectoryIndex() { return dirNumLastAccessed; } /** * Get a path from the local FS. If size is known, we go round-robin over the set of disks (via * the configured dirs) and return the first complete path which has enough space. * * <p>If size is not known, use roulette selection -- pick directories with probability * proportional to their available space. */ public synchronized Path getLocalPathForWrite( String pathStr, long size, Configuration conf, boolean checkWrite) throws IOException { confChanged(conf); int numDirs = localDirsPath.length; int numDirsSearched = 0; // remove the leading slash from the path (to make sure that the uri // resolution results in a valid path on the dir being checked) if (pathStr.startsWith("/")) { pathStr = pathStr.substring(1); } Path returnPath = null; Path path = new Path(pathStr); if (size == SIZE_UNKNOWN) { // do roulette selection: pick dir with probability // proportional to available size long[] availableOnDisk = new long[dirDF.length]; long totalAvailable = 0; // build the "roulette wheel" for (int i = 0; i < dirDF.length; ++i) { availableOnDisk[i] = dirDF[i].getAvailable(); totalAvailable += availableOnDisk[i]; } // Keep rolling the wheel till we get a valid path Random r = new java.util.Random(); while (numDirsSearched < numDirs && returnPath == null) { long randomPosition = Math.abs(r.nextLong()) % totalAvailable; int dir = 0; while (randomPosition > availableOnDisk[dir]) { randomPosition -= availableOnDisk[dir]; dir++; } dirNumLastAccessed = dir; returnPath = createPath(path, checkWrite); if (returnPath == null) { totalAvailable -= availableOnDisk[dir]; availableOnDisk[dir] = 0; // skip this disk numDirsSearched++; } } } else { while (numDirsSearched < numDirs && returnPath == null) { long capacity = dirDF[dirNumLastAccessed].getAvailable(); if (capacity > size) { returnPath = createPath(path, checkWrite); } dirNumLastAccessed++; dirNumLastAccessed = dirNumLastAccessed % numDirs; numDirsSearched++; } } if (returnPath != null) { return returnPath; } // no path found throw new DiskErrorException("Could not find any valid local " + "directory for " + pathStr); } /** * Creates a file on the local FS. Pass size as {@link LocalDirAllocator.SIZE_UNKNOWN} if not * known apriori. We round-robin over the set of disks (via the configured dirs) and return a * file on the first path which has enough space. The file is guaranteed to go away when the JVM * exits. */ public File createTmpFileForWrite(String pathStr, long size, Configuration conf) throws IOException { // find an appropriate directory Path path = getLocalPathForWrite(pathStr, size, conf, true); File dir = new File(path.getParent().toUri().getPath()); String prefix = path.getName(); // create a temp file on this directory File result = File.createTempFile(prefix, null, dir); result.deleteOnExit(); return result; } /** * Get a path from the local FS for reading. We search through all the configured dirs for the * file's existence and return the complete path to the file when we find one */ public synchronized Path getLocalPathToRead(String pathStr, Configuration conf) throws IOException { confChanged(conf); int numDirs = localDirsPath.length; int numDirsSearched = 0; // remove the leading slash from the path (to make sure that the uri // resolution results in a valid path on the dir being checked) if (pathStr.startsWith("/")) { pathStr = pathStr.substring(1); } Path childPath = new Path(pathStr); while (numDirsSearched < numDirs) { Path file = new Path(localDirsPath[numDirsSearched], childPath); if (localFS.exists(file)) { return file; } numDirsSearched++; } // no path found throw new DiskErrorException( "Could not find " + pathStr + " in any of" + " the configured local directories"); } private static class PathIterator implements Iterator<Path>, Iterable<Path> { private final FileSystem fs; private final String pathStr; private int i = 0; private final Path[] rootDirs; private Path next = null; private PathIterator(FileSystem fs, String pathStr, Path[] rootDirs) throws IOException { this.fs = fs; this.pathStr = pathStr; this.rootDirs = rootDirs; advance(); } @Override public boolean hasNext() { return next != null; } private void advance() throws IOException { while (i < rootDirs.length) { next = new Path(rootDirs[i++], pathStr); if (fs.exists(next)) { return; } } next = null; } @Override public Path next() { Path result = next; try { advance(); } catch (IOException ie) { throw new RuntimeException("Can't check existance of " + next, ie); } return result; } @Override public void remove() { throw new UnsupportedOperationException("read only iterator"); } @Override public Iterator<Path> iterator() { return this; } } /** * Get all of the paths that currently exist in the working directories. * * @param pathStr the path underneath the roots * @param conf the configuration to look up the roots in * @return all of the paths that exist under any of the roots * @throws IOException */ synchronized Iterable<Path> getAllLocalPathsToRead(String pathStr, Configuration conf) throws IOException { confChanged(conf); if (pathStr.startsWith("/")) { pathStr = pathStr.substring(1); } return new PathIterator(localFS, pathStr, localDirsPath); } /** * We search through all the configured dirs for the file's existence and return true when we * find one */ public synchronized boolean ifExists(String pathStr, Configuration conf) { try { int numDirs = localDirsPath.length; int numDirsSearched = 0; // remove the leading slash from the path (to make sure that the uri // resolution results in a valid path on the dir being checked) if (pathStr.startsWith("/")) { pathStr = pathStr.substring(1); } Path childPath = new Path(pathStr); while (numDirsSearched < numDirs) { Path file = new Path(localDirsPath[numDirsSearched], childPath); if (localFS.exists(file)) { return true; } numDirsSearched++; } } catch (IOException e) { // IGNORE and try again } return false; } }
/** * To generate automatically reports from list mode. * * <p>Uses JasperReports. * * @author Javier Paniza */ public class GenerateReportServlet extends HttpServlet { private static Log log = LogFactory.getLog(GenerateReportServlet.class); public static class TableModelDecorator implements TableModel { private TableModel original; private List metaProperties; private boolean withValidValues = false; private Locale locale; private boolean labelAsHeader = false; private HttpServletRequest request; private boolean format = false; // format or no the values. If format = true, all values to the report are String private Integer columnCountLimit; public TableModelDecorator( HttpServletRequest request, TableModel original, List metaProperties, Locale locale, boolean labelAsHeader, boolean format, Integer columnCountLimit) throws Exception { this.request = request; this.original = original; this.metaProperties = metaProperties; this.locale = locale; this.withValidValues = calculateWithValidValues(); this.labelAsHeader = labelAsHeader; this.format = format; this.columnCountLimit = columnCountLimit; } private boolean calculateWithValidValues() { Iterator it = metaProperties.iterator(); while (it.hasNext()) { MetaProperty m = (MetaProperty) it.next(); if (m.hasValidValues()) return true; } return false; } private MetaProperty getMetaProperty(int i) { return (MetaProperty) metaProperties.get(i); } public int getRowCount() { return original.getRowCount(); } public int getColumnCount() { return columnCountLimit == null ? original.getColumnCount() : columnCountLimit; } public String getColumnName(int c) { return labelAsHeader ? getMetaProperty(c).getLabel(locale) : Strings.change(getMetaProperty(c).getQualifiedName(), ".", "_"); } public Class getColumnClass(int c) { return original.getColumnClass(c); } public boolean isCellEditable(int row, int column) { return original.isCellEditable(row, column); } public Object getValueAt(int row, int column) { if (isFormat()) return getValueWithWebEditorsFormat(row, column); else return getValueWithoutWebEditorsFormat(row, column); } private Object getValueWithoutWebEditorsFormat(int row, int column) { Object r = original.getValueAt(row, column); if (r instanceof Boolean) { if (((Boolean) r).booleanValue()) return XavaResources.getString(locale, "yes"); return XavaResources.getString(locale, "no"); } if (withValidValues) { MetaProperty p = getMetaProperty(column); if (p.hasValidValues()) { return p.getValidValueLabel(locale, original.getValueAt(row, column)); } } if (r instanceof java.util.Date) { MetaProperty p = getMetaProperty(column); // In order to use the type declared by the developer // and not the one returned by JDBC or the JPA engine if (java.sql.Time.class.isAssignableFrom(p.getType())) { return DateFormat.getTimeInstance(DateFormat.SHORT, locale).format(r); } if (java.sql.Timestamp.class.isAssignableFrom(p.getType())) { DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"); return dateFormat.format(r); } return DateFormat.getDateInstance(DateFormat.SHORT, locale).format(r); } if (r instanceof BigDecimal) { return formatBigDecimal(r, locale); } return r; } private Object getValueWithWebEditorsFormat(int row, int column) { Object r = original.getValueAt(row, column); MetaProperty metaProperty = getMetaProperty(column); String result = WebEditors.format(this.request, metaProperty, r, null, "", true); if (isHtml(result)) { // this avoids that the report shows html content result = WebEditors.format(this.request, metaProperty, r, null, "", false); } return result; } public void setValueAt(Object value, int row, int column) { original.setValueAt(value, row, column); } public void addTableModelListener(TableModelListener l) { original.addTableModelListener(l); } public void removeTableModelListener(TableModelListener l) { original.removeTableModelListener(l); } private boolean isHtml(String value) { return value.matches("<.*>"); } public boolean isFormat() { return format; } public void setFormat(boolean format) { this.format = format; } } protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { try { Locales.setCurrent(request); if (Users.getCurrent() == null) { // for a bug in websphere portal 5.1 with Domino LDAP Users.setCurrent((String) request.getSession().getAttribute("xava.user")); } request.getParameter("application"); // for a bug in websphere 5.1 request.getParameter("module"); // for a bug in websphere 5.1 Tab tab = (Tab) request.getSession().getAttribute("xava_reportTab"); int[] selectedRowsNumber = (int[]) request.getSession().getAttribute("xava_selectedRowsReportTab"); Map[] selectedKeys = (Map[]) request.getSession().getAttribute("xava_selectedKeysReportTab"); int[] selectedRows = getSelectedRows(selectedRowsNumber, selectedKeys, tab); request.getSession().removeAttribute("xava_selectedRowsReportTab"); Integer columnCountLimit = (Integer) request.getSession().getAttribute("xava_columnCountLimitReportTab"); request.getSession().removeAttribute("xava_columnCountLimitReportTab"); setDefaultSchema(request); String user = (String) request.getSession().getAttribute("xava_user"); request.getSession().removeAttribute("xava_user"); Users.setCurrent(user); String uri = request.getRequestURI(); if (uri.endsWith(".pdf")) { InputStream is; JRDataSource ds; Map parameters = new HashMap(); synchronized (tab) { tab.setRequest(request); parameters.put("Title", tab.getTitle()); parameters.put("Organization", getOrganization()); parameters.put("Date", getCurrentDate()); for (String totalProperty : tab.getTotalPropertiesNames()) { parameters.put(totalProperty + "__TOTAL__", getTotal(request, tab, totalProperty)); } TableModel tableModel = getTableModel(request, tab, selectedRows, false, true, null); tableModel.getValueAt(0, 0); if (tableModel.getRowCount() == 0) { generateNoRowsPage(response); return; } is = getReport(request, response, tab, tableModel, columnCountLimit); ds = new JRTableModelDataSource(tableModel); } JasperPrint jprint = JasperFillManager.fillReport(is, parameters, ds); response.setContentType("application/pdf"); response.setHeader( "Content-Disposition", "inline; filename=\"" + getFileName(tab) + ".pdf\""); JasperExportManager.exportReportToPdfStream(jprint, response.getOutputStream()); } else if (uri.endsWith(".csv")) { String csvEncoding = XavaPreferences.getInstance().getCSVEncoding(); if (!Is.emptyString(csvEncoding)) { response.setCharacterEncoding(csvEncoding); } response.setContentType("text/x-csv"); response.setHeader( "Content-Disposition", "inline; filename=\"" + getFileName(tab) + ".csv\""); synchronized (tab) { tab.setRequest(request); response .getWriter() .print( TableModels.toCSV( getTableModel(request, tab, selectedRows, true, false, columnCountLimit))); } } else { throw new ServletException( XavaResources.getString("report_type_not_supported", "", ".pdf .csv")); } } catch (Exception ex) { log.error(ex.getMessage(), ex); throw new ServletException(XavaResources.getString("report_error")); } finally { request.getSession().removeAttribute("xava_reportTab"); } } private void generateNoRowsPage(HttpServletResponse response) throws Exception { response.setContentType("text/html"); response.getWriter().println("<html><head><title>"); response.getWriter().println(XavaResources.getString("no_rows_report_message_title")); response .getWriter() .println( "</title></head><body style='font-family:Tahoma,Arial,sans-serif;color:black;background-color:white;'>"); response.getWriter().println("<h1 style='font-size:22px;'>"); response.getWriter().println(XavaResources.getString("no_rows_report_message_title")); response.getWriter().println("</h1>"); response.getWriter().println("<p style='font-size:16px;'>"); response.getWriter().println(XavaResources.getString("no_rows_report_message_detail")); response.getWriter().println("</p></body></html>"); } private String getCurrentDate() { return java.text.DateFormat.getDateInstance(DateFormat.MEDIUM, Locales.getCurrent()) .format(new java.util.Date()); } private String getFileName(Tab tab) { String now = new SimpleDateFormat("yyyyMMdd_HHmm").format(new Date()); return tab.getTitle() + " " + now; } private Object getTotal(HttpServletRequest request, Tab tab, String totalProperty) { Object total = tab.getTotal(totalProperty); return WebEditors.format( request, tab.getMetaProperty(totalProperty), total, new Messages(), null, true); } private void setDefaultSchema(HttpServletRequest request) { String hibernateDefaultSchemaTab = (String) request.getSession().getAttribute("xava_hibernateDefaultSchemaTab"); if (hibernateDefaultSchemaTab != null) { request.getSession().removeAttribute("xava_hibernateDefaultSchemaTab"); XHibernate.setDefaultSchema(hibernateDefaultSchemaTab); } String jpaDefaultSchemaTab = (String) request.getSession().getAttribute("xava_jpaDefaultSchemaTab"); if (jpaDefaultSchemaTab != null) { request.getSession().removeAttribute("xava_jpaDefaultSchemaTab"); XPersistence.setDefaultSchema(jpaDefaultSchemaTab); } } protected String getOrganization() throws MissingResourceException, XavaException { return ReportParametersProviderFactory.getInstance().getOrganization(); } private InputStream getReport( HttpServletRequest request, HttpServletResponse response, Tab tab, TableModel tableModel, Integer columnCountLimit) throws ServletException, IOException { StringBuffer suri = new StringBuffer(); suri.append("/xava/jasperReport"); suri.append("?language="); suri.append(Locales.getCurrent().getLanguage()); suri.append("&widths="); suri.append(Arrays.toString(getWidths(tableModel))); if (columnCountLimit != null) { suri.append("&columnCountLimit="); suri.append(columnCountLimit); } response.setCharacterEncoding(XSystem.getEncoding()); return Servlets.getURIAsStream(request, response, suri.toString()); } private int[] getWidths(TableModel tableModel) { int[] widths = new int[tableModel.getColumnCount()]; for (int r = 0; r < Math.min(tableModel.getRowCount(), 500); r++) { // 500 is not for performance, but for using only a sample of data with huge table for (int c = 0; c < tableModel.getColumnCount(); c++) { Object o = tableModel.getValueAt(r, c); if (o instanceof String) { String s = ((String) o).trim(); if (s.length() > widths[c]) widths[c] = s.length(); } } } return widths; } private TableModel getTableModel( HttpServletRequest request, Tab tab, int[] selectedRows, boolean labelAsHeader, boolean format, Integer columnCountLimit) throws Exception { TableModel data = null; if (selectedRows != null && selectedRows.length > 0) { data = new SelectedRowsXTableModel(tab.getTableModel(), selectedRows); } else { data = tab.getAllDataTableModel(); } return new TableModelDecorator( request, data, tab.getMetaProperties(), Locales.getCurrent(), labelAsHeader, format, columnCountLimit); } private static Object formatBigDecimal(Object number, Locale locale) { NumberFormat nf = NumberFormat.getNumberInstance(locale); nf.setMinimumFractionDigits(2); return nf.format(number); } private int[] getSelectedRows(int[] selectedRowsNumber, Map[] selectedRowsKeys, Tab tab) { if (selectedRowsKeys == null || selectedRowsKeys.length == 0) return new int[0]; // selectedRowsNumber is the most performant so we use it when possible else if (selectedRowsNumber.length == selectedRowsKeys.length) return selectedRowsNumber; else { // find the rows from the selectedKeys // This has a poor performance, but it covers the case when the selected // rows are not loaded for the tab, something that can occurs if the user // select rows and afterwards reorder the list. try { int[] s = new int[selectedRowsKeys.length]; List selectedKeys = Arrays.asList(selectedRowsKeys); int end = tab.getTableModel().getTotalSize(); int x = 0; for (int i = 0; i < end; i++) { Map key = (Map) tab.getTableModel().getObjectAt(i); if (selectedKeys.contains(key)) { s[x] = i; x++; } } return s; } catch (Exception ex) { log.warn(XavaResources.getString("fails_selected"), ex); throw new XavaException("fails_selected"); } } } }
/** * A simple RPC mechanism. * * <p>A <i>protocol</i> is a Java interface. All parameters and return types must be one of: * * <ul> * <li>a primitive type, <code>boolean</code>, <code>byte</code>, <code>char</code>, <code>short * </code>, <code>int</code>, <code>long</code>, <code>float</code>, <code>double</code>, or * <code>void</code>; or * <li>a {@link String}; or * <li>a {@link Writable}; or * <li>an array of the above types * </ul> * * All methods in the protocol should throw only IOException. No field data of the protocol instance * is transmitted. */ public class RPC { private static final Log LOG = LogFactory.getLog(RPC.class); private RPC() {} // no public ctor /** A method invocation, including the method name and its parameters. */ private static class Invocation implements Writable, Configurable { private String methodName; private Class[] parameterClasses; private Object[] parameters; private Configuration conf; public Invocation() {} public Invocation(Method method, Object[] parameters) { this.methodName = method.getName(); this.parameterClasses = method.getParameterTypes(); this.parameters = parameters; } /** The name of the method invoked. */ public String getMethodName() { return methodName; } /** The parameter classes. */ public Class[] getParameterClasses() { return parameterClasses; } /** The parameter instances. */ public Object[] getParameters() { return parameters; } public void readFields(DataInput in) throws IOException { methodName = UTF8.readString(in); parameters = new Object[in.readInt()]; parameterClasses = new Class[parameters.length]; ObjectWritable objectWritable = new ObjectWritable(); for (int i = 0; i < parameters.length; i++) { parameters[i] = ObjectWritable.readObject(in, objectWritable, this.conf); parameterClasses[i] = objectWritable.getDeclaredClass(); } } public void write(DataOutput out) throws IOException { ObjectWritable.writeStringCached(out, methodName); out.writeInt(parameterClasses.length); for (int i = 0; i < parameterClasses.length; i++) { ObjectWritable.writeObject(out, parameters[i], parameterClasses[i], conf); } } public String toString() { StringBuffer buffer = new StringBuffer(); buffer.append(methodName); buffer.append("("); for (int i = 0; i < parameters.length; i++) { if (i != 0) buffer.append(", "); buffer.append(parameters[i]); } buffer.append(")"); return buffer.toString(); } public void setConf(Configuration conf) { this.conf = conf; } public Configuration getConf() { return this.conf; } } /* Cache a client using its socket factory as the hash key */ private static class ClientCache { private Map<SocketFactory, Client> clients = new HashMap<SocketFactory, Client>(); /** * Construct & cache an IPC client with the user-provided SocketFactory if no cached client * exists. * * @param conf Configuration * @return an IPC client */ private synchronized Client getClient(Configuration conf, SocketFactory factory) { // Construct & cache client. The configuration is only used for timeout, // and Clients have connection pools. So we can either (a) lose some // connection pooling and leak sockets, or (b) use the same timeout for all // configurations. Since the IPC is usually intended globally, not // per-job, we choose (a). Client client = clients.get(factory); if (client == null) { client = new Client(ObjectWritable.class, conf, factory); clients.put(factory, client); } else { client.incCount(); } return client; } /** * Construct & cache an IPC client with the default SocketFactory if no cached client exists. * * @param conf Configuration * @return an IPC client */ private synchronized Client getClient(Configuration conf) { return getClient(conf, SocketFactory.getDefault()); } /** * Stop a RPC client connection A RPC client is closed only when its reference count becomes * zero. */ private void stopClient(Client client) { synchronized (this) { client.decCount(); if (client.isZeroReference()) { clients.remove(client.getSocketFactory()); } } if (client.isZeroReference()) { client.stop(); } } } private static ClientCache CLIENTS = new ClientCache(); private static class Invoker implements InvocationHandler { private InetSocketAddress address; private UserGroupInformation ticket; private Client client; private boolean isClosed = false; private boolean needCheckDnsUpdate = false; private long timeLastDnsCheck = 0; private final long MIN_DNS_CHECK_INTERVAL_MSEC = 120 * 1000; private final int rpcTimeout; private final Class<?> protocol; public Invoker( InetSocketAddress address, UserGroupInformation ticket, Configuration conf, SocketFactory factory, int rpcTimeout, Class<?> protocol) { this.address = address; this.ticket = ticket; this.client = CLIENTS.getClient(conf, factory); this.rpcTimeout = rpcTimeout; this.protocol = protocol; } private synchronized InetSocketAddress getAddress() { if (needCheckDnsUpdate && address != null && address.getHostName() != null && System.currentTimeMillis() - this.timeLastDnsCheck > MIN_DNS_CHECK_INTERVAL_MSEC) { try { InetSocketAddress newAddr = NetUtils.resolveAddress(address); if (newAddr != null) { LOG.info("DNS change: " + newAddr); address = newAddr; } } finally { this.timeLastDnsCheck = System.currentTimeMillis(); } } needCheckDnsUpdate = false; return address; } public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { final boolean logDebug = LOG.isDebugEnabled(); long startTime = 0; if (logDebug) { startTime = System.currentTimeMillis(); } ObjectWritable value = null; try { value = (ObjectWritable) client.call( new Invocation(method, args), getAddress(), protocol, ticket, rpcTimeout); } catch (RemoteException re) { throw re; } catch (ConnectException ce) { needCheckDnsUpdate = true; throw ce; } catch (NoRouteToHostException nrhe) { needCheckDnsUpdate = true; throw nrhe; } catch (PortUnreachableException pue) { needCheckDnsUpdate = true; throw pue; } catch (UnknownHostException uhe) { needCheckDnsUpdate = true; throw uhe; } if (logDebug) { long callTime = System.currentTimeMillis() - startTime; LOG.debug("Call: " + method.getName() + " " + callTime); } return value.get(); } /* close the IPC client that's responsible for this invoker's RPCs */ private synchronized void close() { if (!isClosed) { isClosed = true; CLIENTS.stopClient(client); } } } /** * An exception indicating that the client and server have incompatible versions. They are not * able to communicate with each other. */ public static class VersionIncompatible extends IOException { private String interfaceName; private long clientVersion; private long serverVersion; /** * Create a version incompatible exception * * @param interfaceName the name of the protocol mismatch * @param clientVersion the client's version of the protocol * @param serverVersion the server's version of the protocol */ public VersionIncompatible(String interfaceName, long clientVersion, long serverVersion) { super( "Protocol " + interfaceName + " version mismatch. (client = " + clientVersion + ", server = " + serverVersion + ")"); this.interfaceName = interfaceName; this.clientVersion = clientVersion; this.serverVersion = serverVersion; } /** * Get the interface name * * @return the java class name (eg. org.apache.hadoop.mapred.InterTrackerProtocol) */ public String getInterfaceName() { return interfaceName; } /** Get the client's preferred version */ public long getClientVersion() { return clientVersion; } /** Get the server's agreed to version. */ public long getServerVersion() { return serverVersion; } } /** * A version mismatch for the RPC protocol. * * <p>The client & server have different versions. But the server is not able to determine if they * are compatible mostly because the client is newer than the server. So the proxy is created and * the application client is left to decide if the client & server are compatible or not. */ public static class VersionMismatch extends VersionIncompatible { private static final long serialVersionUID = 1L; private final VersionedProtocol proxy; /** * Create a version mismatch exception * * @param interfaceName the name of the protocol mismatch * @param clientVersion the client's version of the protocol * @param serverVersion the server's version of the protocol */ public VersionMismatch(String interfaceName, long clientVersion, long serverVersion) { super(interfaceName, clientVersion, serverVersion); proxy = null; } /** * Create a version mismatch exception * * @param interfaceName the name of the protocol mismatch * @param clientVersion the client's version of the protocol * @param serverVersion the server's version of the protocol * @param proxy the proxy */ public VersionMismatch( String interfaceName, long clientVersion, long serverVersion, VersionedProtocol proxy) { super(interfaceName, clientVersion, serverVersion); this.proxy = proxy; } /** Return the proxy */ public VersionedProtocol getProxy() { return proxy; } } public static <T extends VersionedProtocol> T waitForProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { return waitForProtocolProxy(protocol, clientVersion, addr, conf).getProxy(); } public static <T extends VersionedProtocol> ProtocolProxy<T> waitForProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { return waitForProtocolProxy(protocol, clientVersion, addr, conf, Long.MAX_VALUE); } public static <T extends VersionedProtocol> T waitForProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, long timeout) throws IOException { return waitForProtocolProxy(protocol, clientVersion, addr, conf, timeout).getProxy(); } public static <T extends VersionedProtocol> ProtocolProxy<T> waitForProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, long timeout) throws IOException { return waitForProtocolProxy(protocol, clientVersion, addr, conf, timeout, 0); } /** * Get a proxy connection to a remote server * * @param protocol protocol class * @param clientVersion client version * @param addr remote address * @param conf configuration to use * @param connTimeout time in milliseconds before giving up * @param rpcTimeout timeout for each RPC * @return the proxy * @throws IOException if the far end through a RemoteException */ public static <T extends VersionedProtocol> T waitForProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, long connTimeout, int rpcTimeout) throws IOException { return waitForProtocolProxy(protocol, clientVersion, addr, conf, connTimeout, rpcTimeout) .getProxy(); } /** * Get a proxy connection to a remote server * * @param protocol protocol class * @param clientVersion client version * @param addr remote address * @param conf configuration to use * @param rpcTimeout timeout for each RPC * @param timeout time in milliseconds before giving up * @return the proxy * @throws IOException if the far end through a RemoteException */ static <T extends VersionedProtocol> ProtocolProxy<T> waitForProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, long timeout, int rpcTimeout) throws IOException { long startTime = System.currentTimeMillis(); UserGroupInformation ugi = null; try { ugi = UserGroupInformation.login(conf); } catch (LoginException le) { throw new RuntimeException("Couldn't login!"); } IOException ioe; while (true) { try { return getProtocolProxy( protocol, clientVersion, addr, ugi, conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout); } catch (ConnectException se) { // namenode has not been started LOG.info("Server at " + addr + " not available yet, Zzzzz..."); ioe = se; } catch (SocketTimeoutException te) { // namenode is busy LOG.info("Problem connecting to server: " + addr); ioe = te; } // check if timed out if (System.currentTimeMillis() - timeout >= startTime) { throw ioe; } // wait for retry try { Thread.sleep(1000); } catch (InterruptedException ie) { // IGNORE } } } /** * Construct a client-side proxy object that implements the named protocol, talking to a server at * the named address. */ public static <T extends VersionedProtocol> T getProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, SocketFactory factory) throws IOException { return getProtocolProxy(protocol, clientVersion, addr, conf, factory).getProxy(); } /** * Construct a client-side protocol proxy that contains a set of server methods and a proxy object * implementing the named protocol, talking to a server at the named address. */ public static <T extends VersionedProtocol> ProtocolProxy<T> getProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, SocketFactory factory) throws IOException { UserGroupInformation ugi = null; try { ugi = UserGroupInformation.login(conf); } catch (LoginException le) { throw new RuntimeException("Couldn't login!"); } return getProtocolProxy(protocol, clientVersion, addr, ugi, conf, factory); } /** * Construct a client-side proxy object that implements the named protocol, talking to a server at * the named address. */ public static <T extends VersionedProtocol> T getProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory) throws IOException { return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory).getProxy(); } /** * Construct a client-side protocol proxy that contains a set of server methods and a proxy object * implementing the named protocol, talking to a server at the named address. */ public static <T extends VersionedProtocol> ProtocolProxy<T> getProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory) throws IOException { return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory, 0); } /** * Construct a client-side proxy that implements the named protocol, talking to a server at the * named address. * * @param protocol protocol * @param clientVersion client's version * @param addr server address * @param ticket security ticket * @param conf configuration * @param factory socket factory * @param rpcTimeout max time for each rpc; 0 means no timeout * @return the proxy * @throws IOException if any error occurs */ public static <T extends VersionedProtocol> T getProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory, int rpcTimeout) throws IOException { return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout) .getProxy(); } /** * Construct a client-side proxy that implements the named protocol, talking to a server at the * named address. * * @param protocol protocol * @param clientVersion client's version * @param addr server address * @param ticket security ticket * @param conf configuration * @param factory socket factory * @param rpcTimeout max time for each rpc; 0 means no timeout * @return the proxy * @throws IOException if any error occurs */ @SuppressWarnings("unchecked") public static <T extends VersionedProtocol> ProtocolProxy<T> getProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory, int rpcTimeout) throws IOException { T proxy = (T) Proxy.newProxyInstance( protocol.getClassLoader(), new Class[] {protocol}, new Invoker(addr, ticket, conf, factory, rpcTimeout, protocol)); String protocolName = protocol.getName(); try { ProtocolSignature serverInfo = proxy.getProtocolSignature( protocolName, clientVersion, ProtocolSignature.getFingerprint(protocol.getMethods())); return new ProtocolProxy<T>(protocol, proxy, serverInfo.getMethods()); } catch (RemoteException re) { IOException ioe = re.unwrapRemoteException(IOException.class); if (ioe.getMessage() .startsWith(IOException.class.getName() + ": " + NoSuchMethodException.class.getName())) { // Method getProtocolSignature not supported long serverVersion = proxy.getProtocolVersion(protocol.getName(), clientVersion); if (serverVersion == clientVersion) { return new ProtocolProxy<T>(protocol, proxy, null); } throw new VersionMismatch(protocolName, clientVersion, serverVersion, proxy); } throw re; } } /** * Construct a client-side proxy object with the default SocketFactory * * @param protocol * @param clientVersion * @param addr * @param conf * @return a proxy instance * @throws IOException */ public static <T extends VersionedProtocol> T getProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { return getProtocolProxy(protocol, clientVersion, addr, conf).getProxy(); } /** * Construct a client-side proxy object with the default SocketFactory * * @param protocol * @param clientVersion * @param addr * @param conf * @return a proxy instance * @throws IOException */ public static <T extends VersionedProtocol> ProtocolProxy<T> getProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { return getProtocolProxy( protocol, clientVersion, addr, conf, NetUtils.getDefaultSocketFactory(conf)); } /** * Stop this proxy and release its invoker's resource * * @param <T> * @param proxy the proxy to be stopped */ public static <T extends VersionedProtocol> void stopProxy(T proxy) { if (proxy != null) { ((Invoker) Proxy.getInvocationHandler(proxy)).close(); } } /** * Expert: Make multiple, parallel calls to a set of servers. * * @deprecated Use {@link #call(Method, Object[][], InetSocketAddress[], UserGroupInformation, * Configuration)} instead */ public static Object[] call( Method method, Object[][] params, InetSocketAddress[] addrs, Configuration conf) throws IOException { return call(method, params, addrs, null, conf); } /** Expert: Make multiple, parallel calls to a set of servers. */ public static Object[] call( Method method, Object[][] params, InetSocketAddress[] addrs, UserGroupInformation ticket, Configuration conf) throws IOException { Invocation[] invocations = new Invocation[params.length]; for (int i = 0; i < params.length; i++) invocations[i] = new Invocation(method, params[i]); Client client = CLIENTS.getClient(conf); try { Writable[] wrappedValues = client.call(invocations, addrs, method.getDeclaringClass(), ticket); if (method.getReturnType() == Void.TYPE) { return null; } Object[] values = (Object[]) Array.newInstance(method.getReturnType(), wrappedValues.length); for (int i = 0; i < values.length; i++) if (wrappedValues[i] != null) values[i] = ((ObjectWritable) wrappedValues[i]).get(); return values; } finally { CLIENTS.stopClient(client); } } static Client getClient(Configuration conf, SocketFactory socketFactory) { return CLIENTS.getClient(conf, socketFactory); } /** Construct a server for a protocol implementation instance listening on a port and address. */ public static Server getServer( final Object instance, final String bindAddress, final int port, Configuration conf) throws IOException { return getServer(instance, bindAddress, port, 1, false, conf); } /** Construct a server for a protocol implementation instance listening on a port and address. */ public static Server getServer( final Object instance, final String bindAddress, final int port, final int numHandlers, final boolean verbose, Configuration conf) throws IOException { return getServer(instance, bindAddress, port, numHandlers, verbose, conf, true); } /** Construct a server for a protocol implementation instance listening on a port and address. */ public static Server getServer( final Object instance, final String bindAddress, final int port, final int numHandlers, final boolean verbose, Configuration conf, boolean supportOldJobConf) throws IOException { return new Server(instance, conf, bindAddress, port, numHandlers, verbose, supportOldJobConf); } /** An RPC Server. */ public static class Server extends org.apache.hadoop.ipc.Server { private Object instance; private boolean verbose; private boolean authorize = false; /** * Construct an RPC server. * * @param instance the instance whose methods will be called * @param conf the configuration to use * @param bindAddress the address to bind on to listen for connection * @param port the port to listen for connections on */ public Server(Object instance, Configuration conf, String bindAddress, int port) throws IOException { this(instance, conf, bindAddress, port, 1, false); } private static String classNameBase(String className) { String[] names = className.split("\\.", -1); if (names == null || names.length == 0) { return className; } return names[names.length - 1]; } /** * Construct an RPC server. * * @param instance the instance whose methods will be called * @param conf the configuration to use * @param bindAddress the address to bind on to listen for connection * @param port the port to listen for connections on * @param numHandlers the number of method handler threads to run * @param verbose whether each call should be logged */ public Server( Object instance, Configuration conf, String bindAddress, int port, int numHandlers, boolean verbose) throws IOException { this(instance, conf, bindAddress, port, numHandlers, verbose, true); } /** * Construct an RPC server. * * @param instance the instance whose methods will be called * @param conf the configuration to use * @param bindAddress the address to bind on to listen for connection * @param port the port to listen for connections on * @param numHandlers the number of method handler threads to run * @param verbose whether each call should be logged * @param supportOldJobConf supports server to deserialize old job conf */ public Server( Object instance, Configuration conf, String bindAddress, int port, int numHandlers, boolean verbose, boolean supportOldJobConf) throws IOException { super( bindAddress, port, Invocation.class, numHandlers, conf, classNameBase(instance.getClass().getName()), supportOldJobConf); this.instance = instance; this.verbose = verbose; this.authorize = conf.getBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, false); } public Writable call(Class<?> protocol, Writable param, long receivedTime) throws IOException { try { Invocation call = (Invocation) param; if (verbose) log("Call: " + call); Method method = protocol.getMethod(call.getMethodName(), call.getParameterClasses()); method.setAccessible(true); int qTime = (int) (System.currentTimeMillis() - receivedTime); long startNanoTime = System.nanoTime(); Object value = method.invoke(instance, call.getParameters()); long processingMicroTime = (System.nanoTime() - startNanoTime) / 1000; if (LOG.isDebugEnabled()) { LOG.debug( "Served: " + call.getMethodName() + " queueTime (millisec)= " + qTime + " procesingTime (microsec)= " + processingMicroTime); } rpcMetrics.rpcQueueTime.inc(qTime); rpcMetrics.rpcProcessingTime.inc(processingMicroTime); MetricsTimeVaryingRate m = (MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName()); if (m == null) { try { m = new MetricsTimeVaryingRate(call.getMethodName(), rpcMetrics.registry); } catch (IllegalArgumentException iae) { // the metrics has been registered; re-fetch the handle LOG.debug("Error register " + call.getMethodName(), iae); m = (MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName()); } } // record call time in microseconds m.inc(processingMicroTime); if (verbose) log("Return: " + value); return new ObjectWritable(method.getReturnType(), value); } catch (InvocationTargetException e) { Throwable target = e.getTargetException(); if (target instanceof IOException) { throw (IOException) target; } else { IOException ioe = new IOException(target.toString()); ioe.setStackTrace(target.getStackTrace()); throw ioe; } } catch (Throwable e) { if (!(e instanceof IOException)) { LOG.error("Unexpected throwable object ", e); } IOException ioe = new IOException(e.toString()); ioe.setStackTrace(e.getStackTrace()); throw ioe; } } @Override public void authorize(Subject user, ConnectionHeader connection) throws AuthorizationException { if (authorize) { Class<?> protocol = null; try { protocol = getProtocolClass(connection.getProtocol(), getConf()); } catch (ClassNotFoundException cfne) { throw new AuthorizationException("Unknown protocol: " + connection.getProtocol()); } ServiceAuthorizationManager.authorize(user, protocol); } } } private static void log(String value) { if (value != null && value.length() > 55) value = value.substring(0, 55) + "..."; LOG.info(value); } }
/** An RpcEngine implementation for Writable data. */ @InterfaceStability.Evolving public class WritableRpcEngine implements RpcEngine { private static final Log LOG = LogFactory.getLog(RPC.class); // writableRpcVersion should be updated if there is a change // in format of the rpc messages. // 2L - added declared class to Invocation public static final long writableRpcVersion = 2L; /** Whether or not this class has been initialized. */ private static boolean isInitialized = false; static { ensureInitialized(); } /** Initialize this class if it isn't already. */ public static synchronized void ensureInitialized() { if (!isInitialized) { initialize(); } } /** Register the rpcRequest deserializer for WritableRpcEngine */ private static synchronized void initialize() { org.apache.hadoop.ipc.Server.registerProtocolEngine( RPC.RpcKind.RPC_WRITABLE, Invocation.class, new Server.WritableRpcInvoker()); isInitialized = true; } /** A method invocation, including the method name and its parameters. */ private static class Invocation implements Writable, Configurable { private String methodName; private Class<?>[] parameterClasses; private Object[] parameters; private Configuration conf; private long clientVersion; private int clientMethodsHash; private String declaringClassProtocolName; // This could be different from static writableRpcVersion when received // at server, if client is using a different version. private long rpcVersion; @SuppressWarnings("unused") // called when deserializing an invocation public Invocation() {} public Invocation(Method method, Object[] parameters) { this.methodName = method.getName(); this.parameterClasses = method.getParameterTypes(); this.parameters = parameters; rpcVersion = writableRpcVersion; if (method.getDeclaringClass().equals(VersionedProtocol.class)) { // VersionedProtocol is exempted from version check. clientVersion = 0; clientMethodsHash = 0; } else { this.clientVersion = RPC.getProtocolVersion(method.getDeclaringClass()); this.clientMethodsHash = ProtocolSignature.getFingerprint(method.getDeclaringClass().getMethods()); } this.declaringClassProtocolName = RPC.getProtocolName(method.getDeclaringClass()); } /** The name of the method invoked. */ public String getMethodName() { return methodName; } /** The parameter classes. */ public Class<?>[] getParameterClasses() { return parameterClasses; } /** The parameter instances. */ public Object[] getParameters() { return parameters; } private long getProtocolVersion() { return clientVersion; } @SuppressWarnings("unused") private int getClientMethodsHash() { return clientMethodsHash; } /** * Returns the rpc version used by the client. * * @return rpcVersion */ public long getRpcVersion() { return rpcVersion; } @SuppressWarnings("deprecation") public void readFields(DataInput in) throws IOException { rpcVersion = in.readLong(); declaringClassProtocolName = UTF8.readString(in); methodName = UTF8.readString(in); clientVersion = in.readLong(); clientMethodsHash = in.readInt(); parameters = new Object[in.readInt()]; parameterClasses = new Class[parameters.length]; ObjectWritable objectWritable = new ObjectWritable(); for (int i = 0; i < parameters.length; i++) { parameters[i] = ObjectWritable.readObject(in, objectWritable, this.conf); parameterClasses[i] = objectWritable.getDeclaredClass(); } } @SuppressWarnings("deprecation") public void write(DataOutput out) throws IOException { out.writeLong(rpcVersion); UTF8.writeString(out, declaringClassProtocolName); UTF8.writeString(out, methodName); out.writeLong(clientVersion); out.writeInt(clientMethodsHash); out.writeInt(parameterClasses.length); for (int i = 0; i < parameterClasses.length; i++) { ObjectWritable.writeObject(out, parameters[i], parameterClasses[i], conf, true); } } public String toString() { StringBuilder buffer = new StringBuilder(); buffer.append(methodName); buffer.append("("); for (int i = 0; i < parameters.length; i++) { if (i != 0) buffer.append(", "); buffer.append(parameters[i]); } buffer.append(")"); buffer.append(", rpc version=" + rpcVersion); buffer.append(", client version=" + clientVersion); buffer.append(", methodsFingerPrint=" + clientMethodsHash); return buffer.toString(); } public void setConf(Configuration conf) { this.conf = conf; } public Configuration getConf() { return this.conf; } } private static ClientCache CLIENTS = new ClientCache(); private static class Invoker implements RpcInvocationHandler { private Client.ConnectionId remoteId; private Client client; private boolean isClosed = false; public Invoker( Class<?> protocol, InetSocketAddress address, UserGroupInformation ticket, Configuration conf, SocketFactory factory, int rpcTimeout) throws IOException { this.remoteId = Client.ConnectionId.getConnectionId(address, protocol, ticket, rpcTimeout, conf); this.client = CLIENTS.getClient(conf, factory); } public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { long startTime = 0; if (LOG.isDebugEnabled()) { startTime = Time.now(); } ObjectWritable value = (ObjectWritable) client.call(RPC.RpcKind.RPC_WRITABLE, new Invocation(method, args), remoteId); if (LOG.isDebugEnabled()) { long callTime = Time.now() - startTime; LOG.debug("Call: " + method.getName() + " " + callTime); } return value.get(); } /* close the IPC client that's responsible for this invoker's RPCs */ public synchronized void close() { if (!isClosed) { isClosed = true; CLIENTS.stopClient(client); } } @Override public ConnectionId getConnectionId() { return remoteId; } } // for unit testing only @InterfaceAudience.Private @InterfaceStability.Unstable static Client getClient(Configuration conf) { return CLIENTS.getClient(conf); } /** * Construct a client-side proxy object that implements the named protocol, talking to a server at * the named address. * * @param <T> */ @Override @SuppressWarnings("unchecked") public <T> ProtocolProxy<T> getProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy) throws IOException { if (connectionRetryPolicy != null) { throw new UnsupportedOperationException( "Not supported: connectionRetryPolicy=" + connectionRetryPolicy); } T proxy = (T) Proxy.newProxyInstance( protocol.getClassLoader(), new Class[] {protocol}, new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout)); return new ProtocolProxy<T>(protocol, proxy, true); } /* Construct a server for a protocol implementation instance listening on a * port and address. */ @Override public RPC.Server getServer( Class<?> protocolClass, Object protocolImpl, String bindAddress, int port, int numHandlers, int numReaders, int queueSizePerHandler, boolean verbose, Configuration conf, SecretManager<? extends TokenIdentifier> secretManager, String portRangeConfig) throws IOException { return new Server( protocolClass, protocolImpl, conf, bindAddress, port, numHandlers, numReaders, queueSizePerHandler, verbose, secretManager, portRangeConfig); } /** An RPC Server. */ public static class Server extends RPC.Server { /** * Construct an RPC server. * * @param instance the instance whose methods will be called * @param conf the configuration to use * @param bindAddress the address to bind on to listen for connection * @param port the port to listen for connections on * @deprecated Use #Server(Class, Object, Configuration, String, int) */ @Deprecated public Server(Object instance, Configuration conf, String bindAddress, int port) throws IOException { this(null, instance, conf, bindAddress, port); } /** * Construct an RPC server. * * @param protocolClass class * @param protocolImpl the instance whose methods will be called * @param conf the configuration to use * @param bindAddress the address to bind on to listen for connection * @param port the port to listen for connections on */ public Server( Class<?> protocolClass, Object protocolImpl, Configuration conf, String bindAddress, int port) throws IOException { this(protocolClass, protocolImpl, conf, bindAddress, port, 1, -1, -1, false, null, null); } /** * Construct an RPC server. * * @param protocolImpl the instance whose methods will be called * @param conf the configuration to use * @param bindAddress the address to bind on to listen for connection * @param port the port to listen for connections on * @param numHandlers the number of method handler threads to run * @param verbose whether each call should be logged * @deprecated use Server#Server(Class, Object, Configuration, String, int, int, int, int, * boolean, SecretManager) */ @Deprecated public Server( Object protocolImpl, Configuration conf, String bindAddress, int port, int numHandlers, int numReaders, int queueSizePerHandler, boolean verbose, SecretManager<? extends TokenIdentifier> secretManager) throws IOException { this( null, protocolImpl, conf, bindAddress, port, numHandlers, numReaders, queueSizePerHandler, verbose, secretManager, null); } /** * Construct an RPC server. * * @param protocolClass - the protocol being registered can be null for compatibility with old * usage (see below for details) * @param protocolImpl the protocol impl that will be called * @param conf the configuration to use * @param bindAddress the address to bind on to listen for connection * @param port the port to listen for connections on * @param numHandlers the number of method handler threads to run * @param verbose whether each call should be logged */ public Server( Class<?> protocolClass, Object protocolImpl, Configuration conf, String bindAddress, int port, int numHandlers, int numReaders, int queueSizePerHandler, boolean verbose, SecretManager<? extends TokenIdentifier> secretManager, String portRangeConfig) throws IOException { super( bindAddress, port, null, numHandlers, numReaders, queueSizePerHandler, conf, classNameBase(protocolImpl.getClass().getName()), secretManager, portRangeConfig); this.verbose = verbose; Class<?>[] protocols; if (protocolClass == null) { // derive protocol from impl /* * In order to remain compatible with the old usage where a single * target protocolImpl is suppled for all protocol interfaces, and * the protocolImpl is derived from the protocolClass(es) * we register all interfaces extended by the protocolImpl */ protocols = RPC.getProtocolInterfaces(protocolImpl.getClass()); } else { if (!protocolClass.isAssignableFrom(protocolImpl.getClass())) { throw new IOException( "protocolClass " + protocolClass + " is not implemented by protocolImpl which is of class " + protocolImpl.getClass()); } // register protocol class and its super interfaces registerProtocolAndImpl(RPC.RpcKind.RPC_WRITABLE, protocolClass, protocolImpl); protocols = RPC.getProtocolInterfaces(protocolClass); } for (Class<?> p : protocols) { if (!p.equals(VersionedProtocol.class)) { registerProtocolAndImpl(RPC.RpcKind.RPC_WRITABLE, p, protocolImpl); } } } private static void log(String value) { if (value != null && value.length() > 55) value = value.substring(0, 55) + "..."; LOG.info(value); } static class WritableRpcInvoker implements RpcInvoker { @Override public Writable call( org.apache.hadoop.ipc.RPC.Server server, String protocolName, Writable rpcRequest, long receivedTime) throws IOException { try { Invocation call = (Invocation) rpcRequest; if (server.verbose) log("Call: " + call); // Verify rpc version if (call.getRpcVersion() != writableRpcVersion) { // Client is using a different version of WritableRpc throw new IOException( "WritableRpc version mismatch, client side version=" + call.getRpcVersion() + ", server side version=" + writableRpcVersion); } long clientVersion = call.getProtocolVersion(); final String protoName; ProtoClassProtoImpl protocolImpl; if (call.declaringClassProtocolName.equals(VersionedProtocol.class.getName())) { // VersionProtocol methods are often used by client to figure out // which version of protocol to use. // // Versioned protocol methods should go the protocolName protocol // rather than the declaring class of the method since the // the declaring class is VersionedProtocol which is not // registered directly. // Send the call to the highest protocol version VerProtocolImpl highest = server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protocolName); if (highest == null) { throw new IOException("Unknown protocol: " + protocolName); } protocolImpl = highest.protocolTarget; } else { protoName = call.declaringClassProtocolName; // Find the right impl for the protocol based on client version. ProtoNameVer pv = new ProtoNameVer(call.declaringClassProtocolName, clientVersion); protocolImpl = server.getProtocolImplMap(RPC.RpcKind.RPC_WRITABLE).get(pv); if (protocolImpl == null) { // no match for Protocol AND Version VerProtocolImpl highest = server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protoName); if (highest == null) { throw new IOException("Unknown protocol: " + protoName); } else { // protocol supported but not the version that client wants throw new RPC.VersionMismatch(protoName, clientVersion, highest.version); } } } // Invoke the protocol method long startTime = Time.now(); Method method = protocolImpl.protocolClass.getMethod( call.getMethodName(), call.getParameterClasses()); method.setAccessible(true); server.rpcDetailedMetrics.init(protocolImpl.protocolClass); Object value = method.invoke(protocolImpl.protocolImpl, call.getParameters()); int processingTime = (int) (Time.now() - startTime); int qTime = (int) (startTime - receivedTime); if (LOG.isDebugEnabled()) { LOG.debug( "Served: " + call.getMethodName() + " queueTime= " + qTime + " procesingTime= " + processingTime); } server.rpcMetrics.addRpcQueueTime(qTime); server.rpcMetrics.addRpcProcessingTime(processingTime); server.rpcDetailedMetrics.addProcessingTime(call.getMethodName(), processingTime); if (server.verbose) log("Return: " + value); return new ObjectWritable(method.getReturnType(), value); } catch (InvocationTargetException e) { Throwable target = e.getTargetException(); if (target instanceof IOException) { throw (IOException) target; } else { IOException ioe = new IOException(target.toString()); ioe.setStackTrace(target.getStackTrace()); throw ioe; } } catch (Throwable e) { if (!(e instanceof IOException)) { LOG.error("Unexpected throwable object ", e); } IOException ioe = new IOException(e.toString()); ioe.setStackTrace(e.getStackTrace()); throw ioe; } } } } @Override public ProtocolProxy<ProtocolMetaInfoPB> getProtocolMetaInfoProxy( ConnectionId connId, Configuration conf, SocketFactory factory) throws IOException { throw new UnsupportedOperationException("This proxy is not supported"); } }
/** @author stone */ public class LSATApplication extends javax.swing.JFrame { // private user variables declaration private Project theProject; private JDesktopPane theDesktop; private DocumentViewerFrame documentViewerTable; private LSAResults currentResults; private DebugFrame debugFrame; private Log log = LogFactory.getLog(); private LSATPreferences thePrefs = LSATPreferencesFactory.getPrefs(false); private DocumentFrameTableModel docFrameTableModel; private File lastPath; // end private user variables declaration /** Creates new form LSATApplication */ public LSATApplication() { initComponents(); initUserComponents(); setWindowSizeAndTitle(); } /** * This method is called from within the constructor to initialize the form. WARNING: Do NOT * modify this code. The content of this method is always regenerated by the Form Editor. */ // <editor-fold defaultstate="collapsed" desc=" Generated Code ">//GEN-BEGIN:initComponents private void initComponents() { jMenuBar = new javax.swing.JMenuBar(); jMenuFile = new javax.swing.JMenu(); jMenuItemNewProject = new javax.swing.JMenuItem(); jMenuItemLoadProject = new javax.swing.JMenuItem(); jMenuItemSaveProject = new javax.swing.JMenuItem(); jMenuItemCloseProject = new javax.swing.JMenuItem(); jSeparator = new javax.swing.JSeparator(); jMenuItemImport = new javax.swing.JMenuItem(); jSeparator1 = new javax.swing.JSeparator(); jMenuItemQuit = new javax.swing.JMenuItem(); jMenuEdit = new javax.swing.JMenu(); jMenuItemCut = new javax.swing.JMenuItem(); jMenuItemCopy = new javax.swing.JMenuItem(); jMenuItemPaste = new javax.swing.JMenuItem(); jMenuView = new javax.swing.JMenu(); jMenuItemViewDocuments = new javax.swing.JMenuItem(); jMenuItemViewDebug = new javax.swing.JMenuItem(); jMenuLSA = new javax.swing.JMenu(); jMenuItemPerformLSA = new javax.swing.JMenuItem(); jSeparator2 = new javax.swing.JSeparator(); jMenuItemViewLSAResults = new javax.swing.JMenuItem(); jMenuItemViewVisualResults = new javax.swing.JMenuItem(); jSeparator3 = new javax.swing.JSeparator(); jMenuItemLoadLSAResults = new javax.swing.JMenuItem(); jMenuItemSaveLSAResults = new javax.swing.JMenuItem(); jMenuItemDumpChunks = new javax.swing.JMenuItem(); jMenuItemLogGraphingStats = new javax.swing.JMenuItem(); jMenuItemLogRSGraphingStats = new javax.swing.JMenuItem(); jMenuPreferences = new javax.swing.JMenu(); jMenuItemShowPreferences = new javax.swing.JMenuItem(); getContentPane().setLayout(null); setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE); jMenuFile.setText("File"); jMenuItemNewProject.setText("New Project"); jMenuItemNewProject.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemNewProjectActionPerformed(evt); } }); jMenuFile.add(jMenuItemNewProject); jMenuItemLoadProject.setText("Load Project"); jMenuItemLoadProject.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemLoadProjectActionPerformed(evt); } }); jMenuFile.add(jMenuItemLoadProject); jMenuItemSaveProject.setText("Save Project"); jMenuItemSaveProject.setEnabled(false); jMenuItemSaveProject.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemSaveProjectActionPerformed(evt); } }); jMenuFile.add(jMenuItemSaveProject); jMenuItemCloseProject.setText("Close Project"); jMenuItemCloseProject.setEnabled(false); jMenuFile.add(jMenuItemCloseProject); jMenuFile.add(jSeparator); jMenuItemImport.setText("Import"); jMenuItemImport.setEnabled(false); jMenuItemImport.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemImportActionPerformed(evt); } }); jMenuFile.add(jMenuItemImport); jMenuFile.add(jSeparator1); jMenuItemQuit.setText("Quit"); jMenuItemQuit.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemQuitActionPerformed(evt); } }); jMenuFile.add(jMenuItemQuit); jMenuBar.add(jMenuFile); jMenuEdit.setText("Edit"); jMenuItemCut.setText("Cut"); jMenuItemCut.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemCutActionPerformed(evt); } }); jMenuEdit.add(jMenuItemCut); jMenuItemCopy.setText("Copy"); jMenuEdit.add(jMenuItemCopy); jMenuItemPaste.setText("Paste"); jMenuEdit.add(jMenuItemPaste); jMenuBar.add(jMenuEdit); jMenuView.setText("View"); jMenuView.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuViewActionPerformed(evt); } }); jMenuItemViewDocuments.setText("Documents"); jMenuItemViewDocuments.setEnabled(false); jMenuItemViewDocuments.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemViewDocumentsActionPerformed(evt); } }); jMenuView.add(jMenuItemViewDocuments); jMenuItemViewDebug.setText("View Debug"); jMenuItemViewDebug.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemViewDebugActionPerformed(evt); } }); jMenuView.add(jMenuItemViewDebug); jMenuBar.add(jMenuView); jMenuLSA.setText("LSA"); jMenuItemPerformLSA.setText("Perform LSA"); jMenuItemPerformLSA.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemPerformLSAActionPerformed(evt); } }); jMenuLSA.add(jMenuItemPerformLSA); jMenuLSA.add(jSeparator2); jMenuItemViewLSAResults.setText("View LSA Results"); jMenuItemViewLSAResults.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemViewLSAResultsActionPerformed(evt); } }); jMenuLSA.add(jMenuItemViewLSAResults); jMenuItemViewVisualResults.setText("View results graph"); jMenuItemViewVisualResults.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemViewVisualResultsActionPerformed(evt); } }); jMenuLSA.add(jMenuItemViewVisualResults); jMenuLSA.add(jSeparator3); jMenuItemLoadLSAResults.setText("Load LSA Results"); jMenuItemLoadLSAResults.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemLoadLSAResultsActionPerformed(evt); } }); jMenuLSA.add(jMenuItemLoadLSAResults); jMenuItemSaveLSAResults.setText("Save LSA Results"); jMenuItemSaveLSAResults.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemSaveLSAResultsActionPerformed(evt); } }); jMenuLSA.add(jMenuItemSaveLSAResults); jMenuItemDumpChunks.setText("Dump Chunks Belonging to Class"); jMenuItemDumpChunks.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemDumpChunksActionPerformed(evt); } }); jMenuLSA.add(jMenuItemDumpChunks); jMenuItemLogGraphingStats.setText("Log Graphing Stats"); jMenuItemLogGraphingStats.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemLogGraphingStatsActionPerformed(evt); } }); jMenuItemLogGraphingStats.addMouseListener( new java.awt.event.MouseAdapter() { public void mouseClicked(java.awt.event.MouseEvent evt) { jMenuItemLogGraphingStatsMouseClicked(evt); } }); jMenuLSA.add(jMenuItemLogGraphingStats); jMenuItemLogRSGraphingStats.setText("Log RS Graphing Stats"); jMenuItemLogRSGraphingStats.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemLogRSGraphingStatsActionPerformed(evt); } }); jMenuItemLogRSGraphingStats.addMouseListener( new java.awt.event.MouseAdapter() { public void mouseClicked(java.awt.event.MouseEvent evt) { jMenuItemLogRSGraphingStatsMouseClicked(evt); } }); jMenuLSA.add(jMenuItemLogRSGraphingStats); jMenuBar.add(jMenuLSA); jMenuPreferences.setText("Preferences"); jMenuItemShowPreferences.setText("Show preferences dialog"); jMenuItemShowPreferences.addActionListener( new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { jMenuItemShowPreferencesActionPerformed(evt); } }); jMenuPreferences.add(jMenuItemShowPreferences); jMenuBar.add(jMenuPreferences); setJMenuBar(jMenuBar); pack(); } // </editor-fold>//GEN-END:initComponents private void jMenuItemLogRSGraphingStatsMouseClicked( java.awt.event.MouseEvent evt) // GEN-FIRST:event_jMenuItemLogRSGraphingStatsMouseClicked { // GEN-HEADEREND:event_jMenuItemLogRSGraphingStatsMouseClicked // TODO add your handling code here: } // GEN-LAST:event_jMenuItemLogRSGraphingStatsMouseClicked private void jMenuItemLogRSGraphingStatsActionPerformed( java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemLogRSGraphingStatsActionPerformed { // GEN-HEADEREND:event_jMenuItemLogRSGraphingStatsActionPerformed if (currentResults != null) { this.currentResults.printGraphingStatsFromReqSimilieData(); } } // GEN-LAST:event_jMenuItemLogRSGraphingStatsActionPerformed private void jMenuItemViewVisualResultsActionPerformed( java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemViewVisualResultsActionPerformed { // GEN-HEADEREND:event_jMenuItemViewVisualResultsActionPerformed if (this.currentResults != null) { VisualAnalysisFrame f = new VisualAnalysisFrame(this.currentResults, this.theProject); theDesktop.add(f); } } // GEN-LAST:event_jMenuItemViewVisualResultsActionPerformed private void jMenuItemLoadProjectActionPerformed( java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemLoadProjectActionPerformed { // GEN-HEADEREND:event_jMenuItemLoadProjectActionPerformed JFileChooser jfc = new JFileChooser(); if (lastPath != null) { jfc.setCurrentDirectory(lastPath); } int fileDialogReturnVal = jfc.showOpenDialog(this); if (fileDialogReturnVal == JFileChooser.APPROVE_OPTION) { try { File inputFile = jfc.getSelectedFile(); FileInputStream fis = new FileInputStream(inputFile); ObjectInputStream ois = new ObjectInputStream(fis); this.theProject = (Project) ois.readObject(); this.currentResults = (LSAResults) ois.readObject(); lastPath = new File(jfc.getSelectedFile().getPath()); } catch (IOException e) { if (this.theProject == null) { log.log(Log.ERROR, "Failed to load project"); } if (this.currentResults == null) { log.log(Log.WARNING, "Failed to load results"); } log.log(Log.WARNING, e.getMessage()); } catch (ClassNotFoundException e) { log.log(Log.ERROR, "Class not found error, version mismatch"); } } if (this.theProject != null) { jMenuItemViewDocuments.setEnabled(true); jMenuItemSaveProject.setEnabled(true); this.setTitle(theProject.getProjectName()); log.log(Log.INFO, "Project Loaded"); } if (this.currentResults != null) { log.log(Log.INFO, "Results loaded"); } } // GEN-LAST:event_jMenuItemLoadProjectActionPerformed private void jMenuItemSaveProjectActionPerformed( java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemSaveProjectActionPerformed { // GEN-HEADEREND:event_jMenuItemSaveProjectActionPerformed if (this.theProject != null) { JFileChooser jfc = new JFileChooser(); int fileDialogReturnVal = jfc.showSaveDialog(this); if (fileDialogReturnVal == JFileChooser.APPROVE_OPTION) { try { File outputFile = jfc.getSelectedFile(); FileOutputStream fos = new FileOutputStream(outputFile); ObjectOutputStream oos = new ObjectOutputStream(fos); oos.writeObject(this.theProject); if (this.currentResults != null) { oos.writeObject(this.currentResults); } } catch (IOException e) { log.log(Log.ERROR, "Failed to save file\n" + e.getMessage()); } } } } // GEN-LAST:event_jMenuItemSaveProjectActionPerformed private void jMenuItemCutActionPerformed( java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemCutActionPerformed { // GEN-HEADEREND:event_jMenuItemCutActionPerformed Toolkit t = java.awt.Toolkit.getDefaultToolkit(); Clipboard c = t.getSystemClipboard(); JInternalFrame currentFrame = theDesktop.getSelectedFrame(); /*StringSelection contents = new StringSelection(srcData); clipboard.setContents(contents, this);*/ } // GEN-LAST:event_jMenuItemCutActionPerformed private void jMenuItemShowPreferencesActionPerformed( java.awt.event.ActionEvent evt) // GEN-FIRST:event_jMenuItemShowPreferencesActionPerformed { // GEN-HEADEREND:event_jMenuItemShowPreferencesActionPerformed PreferencesDialog.showPreferencesDialog(this, this.thePrefs); } // GEN-LAST:event_jMenuItemShowPreferencesActionPerformed private void jMenuItemLogGraphingStatsActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemLogGraphingStatsActionPerformed if (currentResults != null) { // this.currentResults.printGraphingStats(); } } // GEN-LAST:event_jMenuItemLogGraphingStatsActionPerformed private void jMenuItemLogGraphingStatsMouseClicked( java.awt.event.MouseEvent evt) { // GEN-FIRST:event_jMenuItemLogGraphingStatsMouseClicked } // GEN-LAST:event_jMenuItemLogGraphingStatsMouseClicked private void jMenuItemDumpChunksActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemDumpChunksActionPerformed if (currentResults != null) { DocumentClassDialog d = DocumentClassDialog.showClassDialog(this, this.theProject); DocumentClass selectedClass = d.getSelectedDocumentClass(); currentResults.dumpChunksBelongingToDocumentsWithClass(selectedClass.getId()); } } // GEN-LAST:event_jMenuItemDumpChunksActionPerformed private void jMenuItemViewDebugActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemViewDebugActionPerformed this.debugFrame.setVisible(true); } // GEN-LAST:event_jMenuItemViewDebugActionPerformed private void jMenuItemViewLSAResultsActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemViewLSAResultsActionPerformed if (this.currentResults != null) { LSAResultsFrame f = new LSAResultsFrame(this.currentResults, this.theProject); theDesktop.add(f); } } // GEN-LAST:event_jMenuItemViewLSAResultsActionPerformed private void jMenuItemLoadLSAResultsActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemLoadLSAResultsActionPerformed JFileChooser jfc = new JFileChooser(); int fileDialogReturnVal = jfc.showOpenDialog(this); if (fileDialogReturnVal == JFileChooser.APPROVE_OPTION) { try { File inputFile = jfc.getSelectedFile(); FileInputStream fis = new FileInputStream(inputFile); ObjectInputStream ois = new ObjectInputStream(fis); this.currentResults = (LSAResults) ois.readObject(); } catch (IOException e) { log.log(Log.ERROR, "Failed to load LSA results\n" + e.getMessage()); } catch (ClassNotFoundException e) { log.log(Log.ERROR, "Class not found : Error loading LSA results due to version mismatch"); } System.out.println(currentResults == null); } } // GEN-LAST:event_jMenuItemLoadLSAResultsActionPerformed private void jMenuItemSaveLSAResultsActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemSaveLSAResultsActionPerformed if (this.currentResults != null) { JFileChooser jfc = new JFileChooser(); int fileDialogReturnVal = jfc.showSaveDialog(this); if (fileDialogReturnVal == JFileChooser.APPROVE_OPTION) { try { File outputFile = jfc.getSelectedFile(); FileOutputStream fos = new FileOutputStream(outputFile); ObjectOutputStream oos = new ObjectOutputStream(fos); oos.writeObject(this.currentResults); } catch (IOException e) { System.out.println("IOexception"); System.out.println(e.getMessage()); } } } } // GEN-LAST:event_jMenuItemSaveLSAResultsActionPerformed private void jMenuItemPerformLSAActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemPerformLSAActionPerformed Thread t = new LSAThread( theProject.getDocumentCollection(), theProject.getDocumentClassCollection(), thePrefs.get("lsa-regex"), this); t.start(); } // GEN-LAST:event_jMenuItemPerformLSAActionPerformed private void jMenuItemImportActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemImportActionPerformed JFileChooser jfc = new JFileChooser(); jfc.setMultiSelectionEnabled(true); if (lastPath != null) { jfc.setCurrentDirectory(lastPath); } int fileDialogReturnVal = jfc.showOpenDialog(this); // now select the file if (fileDialogReturnVal == JFileChooser.APPROVE_OPTION) { // add code here to allow selection of a document class DocumentClassDialog d = DocumentClassDialog.showClassDialog(this, this.theProject); DocumentClass selectedClass = d.getSelectedDocumentClass(); File[] selected = jfc.getSelectedFiles(); for (int i = 0; i < selected.length; i++) { theProject.addNewDocument(selected[i], 1.0f, selected[i].toString(), selectedClass); } docFrameTableModel.fireTableDataChanged(); lastPath = new File(jfc.getSelectedFile().getPath()); } } // GEN-LAST:event_jMenuItemImportActionPerformed private void jMenuItemViewDocumentsActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemViewDocumentsActionPerformed if (this.docFrameTableModel == null) { this.docFrameTableModel = new DocumentFrameTableModel(theProject.getDocumentCollection()); } DocumentFrame d = new DocumentFrame(this.docFrameTableModel, theDesktop, theProject.getDocumentCollection()); d.setVisible(true); theDesktop.add(d); } // GEN-LAST:event_jMenuItemViewDocumentsActionPerformed private void jMenuViewActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuViewActionPerformed } // GEN-LAST:event_jMenuViewActionPerformed private void jMenuItemQuitActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemQuitActionPerformed this.shutDown(); System.exit(0); } // GEN-LAST:event_jMenuItemQuitActionPerformed private void jMenuItemNewProjectActionPerformed( java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jMenuItemNewProjectActionPerformed String projectName = JOptionPane.showInputDialog(null, "Please enter a project name"); if (projectName != null) { theProject = new Project(projectName); // now we've got a project we need to enalbe the save and close buttons jMenuItemSaveProject.setEnabled(true); jMenuItemCloseProject.setEnabled(true); jMenuItemViewDocuments.setEnabled(true); jMenuItemImport.setEnabled(true); // set title setTitle("LSAT - " + projectName); jMenuItemViewDocumentsActionPerformed(null); } } // GEN-LAST:event_jMenuItemNewProjectActionPerformed private void setWindowSizeAndTitle() { Dimension screenSize = this.getToolkit().getScreenSize(); int newHeight = (int) ((float) screenSize.height * 0.7); int newWidth = (int) ((float) screenSize.width * 0.7); this.setSize(newWidth, newHeight); newHeight = (int) ((0.3 * (float) screenSize.height) / 2); newWidth = (int) ((0.3 * (float) screenSize.width) / 2); this.setLocation(newWidth, newHeight); this.setTitle("LSAT"); } private void initUserComponents() { // required to attach windows to in the MDI world theDesktop = new JDesktopPane(); setContentPane(theDesktop); // create and add the debug frame debugFrame = new DebugFrame(); debugFrame.setVisible(false); theDesktop.add(debugFrame); } private void shutDown() { // nothing in here yet } public void showLSAResults(LSAResults results) { this.currentResults = results; jMenuItemViewLSAResultsActionPerformed(null); } // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JMenuBar jMenuBar; private javax.swing.JMenu jMenuEdit; private javax.swing.JMenu jMenuFile; private javax.swing.JMenuItem jMenuItemCloseProject; private javax.swing.JMenuItem jMenuItemCopy; private javax.swing.JMenuItem jMenuItemCut; private javax.swing.JMenuItem jMenuItemDumpChunks; private javax.swing.JMenuItem jMenuItemImport; private javax.swing.JMenuItem jMenuItemLoadLSAResults; private javax.swing.JMenuItem jMenuItemLoadProject; private javax.swing.JMenuItem jMenuItemLogGraphingStats; private javax.swing.JMenuItem jMenuItemLogRSGraphingStats; private javax.swing.JMenuItem jMenuItemNewProject; private javax.swing.JMenuItem jMenuItemPaste; private javax.swing.JMenuItem jMenuItemPerformLSA; private javax.swing.JMenuItem jMenuItemQuit; private javax.swing.JMenuItem jMenuItemSaveLSAResults; private javax.swing.JMenuItem jMenuItemSaveProject; private javax.swing.JMenuItem jMenuItemShowPreferences; private javax.swing.JMenuItem jMenuItemViewDebug; private javax.swing.JMenuItem jMenuItemViewDocuments; private javax.swing.JMenuItem jMenuItemViewLSAResults; private javax.swing.JMenuItem jMenuItemViewVisualResults; private javax.swing.JMenu jMenuLSA; private javax.swing.JMenu jMenuPreferences; private javax.swing.JMenu jMenuView; private javax.swing.JSeparator jSeparator; private javax.swing.JSeparator jSeparator1; private javax.swing.JSeparator jSeparator2; private javax.swing.JSeparator jSeparator3; // End of variables declaration//GEN-END:variables /** @param args the command line arguments */ public static void main(String args[]) { java.awt.EventQueue.invokeLater( new Runnable() { public void run() { new LSATApplication().setVisible(true); } }); } }
/** * ******************************************************** The Secondary NameNode is a helper to * the primary NameNode. The Secondary is responsible for supporting periodic checkpoints of the * HDFS metadata. The current design allows only one Secondary NameNode per HDFs cluster. * * <p>The Secondary NameNode is a daemon that periodically wakes up (determined by the schedule * specified in the configuration), triggers a periodic checkpoint and then goes back to sleep. The * Secondary NameNode uses the ClientProtocol to talk to the primary NameNode. * * <p>******************************************************** */ public class SecondaryNameNode implements Runnable { public static final Log LOG = LogFactory.getLog(SecondaryNameNode.class.getName()); private String fsName; private CheckpointStorage checkpointImage; private FSNamesystem namesystem; private NamenodeProtocol namenode; private Configuration conf; private InetSocketAddress nameNodeAddr; private volatile boolean shouldRun; private HttpServer infoServer; private int infoPort; private String infoBindAddress; private Collection<File> checkpointDirs; private Collection<File> checkpointEditsDirs; private long checkpointPeriod; // in seconds private long checkpointSize; // size (in MB) of current Edit Log /** Utility class to facilitate junit test error simulation. */ static class ErrorSimulator { private static boolean[] simulation = null; // error simulation events static void initializeErrorSimulationEvent(int numberOfEvents) { simulation = new boolean[numberOfEvents]; for (int i = 0; i < numberOfEvents; i++) { simulation[i] = false; } } static boolean getErrorSimulation(int index) { if (simulation == null) return false; assert (index < simulation.length); return simulation[index]; } static void setErrorSimulation(int index) { assert (index < simulation.length); simulation[index] = true; } static void clearErrorSimulation(int index) { assert (index < simulation.length); simulation[index] = false; } } FSImage getFSImage() { return checkpointImage; } /** Create a connection to the primary namenode. */ public SecondaryNameNode(Configuration conf) throws IOException { try { initialize(conf); } catch (IOException e) { shutdown(); throw e; } } /** Initialize SecondaryNameNode. */ private void initialize(Configuration conf) throws IOException { // initiate Java VM metrics JvmMetrics.init("SecondaryNameNode", conf.get("session.id")); // Create connection to the namenode. shouldRun = true; nameNodeAddr = NameNode.getAddress(conf); this.conf = conf; this.namenode = (NamenodeProtocol) RPC.waitForProxy( NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, conf); // initialize checkpoint directories fsName = getInfoServer(); checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary"); checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary"); checkpointImage = new CheckpointStorage(conf); checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs); // Initialize other scheduling parameters from the configuration checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600); checkpointSize = conf.getLong("fs.checkpoint.size", 4194304); // initialize the webserver for uploading files. String infoAddr = NetUtils.getServerAddress( conf, "dfs.secondary.info.bindAddress", "dfs.secondary.info.port", "dfs.secondary.http.address"); InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr); infoBindAddress = infoSocAddr.getHostName(); int tmpInfoPort = infoSocAddr.getPort(); infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf); infoServer.setAttribute("name.system.image", checkpointImage); this.infoServer.setAttribute("name.conf", conf); infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class); infoServer.start(); // The web-server port can be ephemeral... ensure we have the correct info infoPort = infoServer.getPort(); conf.set("dfs.secondary.http.address", infoBindAddress + ":" + infoPort); LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" + infoPort); LOG.warn( "Checkpoint Period :" + checkpointPeriod + " secs " + "(" + checkpointPeriod / 60 + " min)"); LOG.warn( "Log Size Trigger :" + checkpointSize + " bytes " + "(" + checkpointSize / 1024 + " KB)"); } /** Shut down this instance of the datanode. Returns only after shutdown is complete. */ public void shutdown() { shouldRun = false; try { if (infoServer != null) infoServer.stop(); } catch (Exception e) { LOG.warn("Exception shutting down SecondaryNameNode", e); } try { if (checkpointImage != null) checkpointImage.close(); } catch (IOException e) { LOG.warn(StringUtils.stringifyException(e)); } } // // The main work loop // public void run() { // // Poll the Namenode (once every 5 minutes) to find the size of the // pending edit log. // long period = 5 * 60; // 5 minutes long lastCheckpointTime = 0; if (checkpointPeriod < period) { period = checkpointPeriod; } while (shouldRun) { try { Thread.sleep(1000 * period); } catch (InterruptedException ie) { // do nothing } if (!shouldRun) { break; } try { long now = System.currentTimeMillis(); long size = namenode.getEditLogSize(); if (size >= checkpointSize || now >= lastCheckpointTime + 1000 * checkpointPeriod) { doCheckpoint(); lastCheckpointTime = now; } } catch (IOException e) { LOG.error("Exception in doCheckpoint: "); LOG.error(StringUtils.stringifyException(e)); e.printStackTrace(); checkpointImage.imageDigest = null; } catch (Throwable e) { LOG.error("Throwable Exception in doCheckpoint: "); LOG.error(StringUtils.stringifyException(e)); e.printStackTrace(); Runtime.getRuntime().exit(-1); } } } /** * Download <code>fsimage</code> and <code>edits</code> files from the name-node. * * @return true if a new image has been downloaded and needs to be loaded * @throws IOException */ private boolean downloadCheckpointFiles(CheckpointSignature sig) throws IOException { checkpointImage.cTime = sig.cTime; checkpointImage.checkpointTime = sig.checkpointTime; boolean downloadImage = true; String fileid; File[] srcNames; if (sig.imageDigest.equals(checkpointImage.imageDigest)) { downloadImage = false; LOG.info("Image has not changed. Will not download image."); } else { // get fsimage srcNames = checkpointImage.getImageFiles(); assert srcNames.length > 0 : "No checkpoint targets."; fileid = "getimage=1"; TransferFsImage.getFileClient(fsName, fileid, srcNames, false); checkpointImage.imageDigest = sig.imageDigest; LOG.info( "Downloaded file " + srcNames[0].getName() + " size " + srcNames[0].length() + " bytes."); } // get edits file fileid = "getedit=1"; srcNames = checkpointImage.getEditsFiles(); assert srcNames.length > 0 : "No checkpoint targets."; TransferFsImage.getFileClient(fsName, fileid, srcNames, false); LOG.info( "Downloaded file " + srcNames[0].getName() + " size " + srcNames[0].length() + " bytes."); checkpointImage.checkpointUploadDone(null); return downloadImage; } /** Copy the new fsimage into the NameNode */ private void putFSImage(CheckpointSignature sig) throws IOException { String fileid = "putimage=1&port=" + infoPort + "&machine=" + InetAddress.getLocalHost().getHostAddress() + "&token=" + sig.toString(); LOG.info("Posted URL " + fsName + fileid); TransferFsImage.getFileClient(fsName, fileid, (File[]) null, false); } /** Returns the Jetty server that the Namenode is listening on. */ private String getInfoServer() throws IOException { URI fsName = FileSystem.getDefaultUri(conf); if (!"hdfs".equals(fsName.getScheme())) { throw new IOException("This is not a DFS"); } return NetUtils.getServerAddress( conf, "dfs.info.bindAddress", "dfs.info.port", "dfs.http.address"); } /** Create a new checkpoint */ void doCheckpoint() throws IOException { LOG.info("Checkpoint starting"); // Do the required initialization of the merge work area. startCheckpoint(); // Tell the namenode to start logging transactions in a new edit file // Returns a token that would be used to upload the merged image. CheckpointSignature sig = (CheckpointSignature) namenode.rollEditLog(); // error simulation code for junit test if (ErrorSimulator.getErrorSimulation(0)) { throw new IOException("Simulating error0 " + "after creating edits.new"); } boolean loadImage = downloadCheckpointFiles(sig); // Fetch fsimage and edits doMerge(sig, loadImage); // Do the merge // // Upload the new image into the NameNode. Then tell the Namenode // to make this new uploaded image as the most current image. // putFSImage(sig); // error simulation code for junit test if (ErrorSimulator.getErrorSimulation(1)) { throw new IOException("Simulating error1 " + "after uploading new image to NameNode"); } namenode.rollFsImage(new CheckpointSignature(checkpointImage)); checkpointImage.endCheckpoint(); LOG.info("Checkpoint done. New Image Size: " + checkpointImage.getFsImageName().length()); } private void startCheckpoint() throws IOException { checkpointImage.unlockAll(); checkpointImage.getEditLog().close(); checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs); checkpointImage.startCheckpoint(); } /** Merge downloaded image and edits and write the new image into current storage directory. */ private void doMerge(CheckpointSignature sig, boolean loadImage) throws IOException { if (loadImage) { // create an empty namespace if new image namesystem = new FSNamesystem(checkpointImage, conf); } assert namesystem.dir.fsImage == checkpointImage; checkpointImage.doMerge(sig, loadImage); } /** * @param argv The parameters passed to this program. * @exception Exception if the filesystem does not exist. * @return 0 on success, non zero on error. */ private int processArgs(String[] argv) throws Exception { if (argv.length < 1) { printUsage(""); return -1; } int exitCode = -1; int i = 0; String cmd = argv[i++]; // // verify that we have enough command line parameters // if ("-geteditsize".equals(cmd)) { if (argv.length != 1) { printUsage(cmd); return exitCode; } } else if ("-checkpoint".equals(cmd)) { if (argv.length != 1 && argv.length != 2) { printUsage(cmd); return exitCode; } if (argv.length == 2 && !"force".equals(argv[i])) { printUsage(cmd); return exitCode; } } exitCode = 0; try { if ("-checkpoint".equals(cmd)) { long size = namenode.getEditLogSize(); if (size >= checkpointSize || argv.length == 2 && "force".equals(argv[i])) { doCheckpoint(); } else { System.err.println( "EditLog size " + size + " bytes is " + "smaller than configured checkpoint " + "size " + checkpointSize + " bytes."); System.err.println("Skipping checkpoint."); } } else if ("-geteditsize".equals(cmd)) { long size = namenode.getEditLogSize(); System.out.println("EditLog size is " + size + " bytes"); } else { exitCode = -1; LOG.error(cmd.substring(1) + ": Unknown command"); printUsage(""); } } catch (RemoteException e) { // // This is a error returned by hadoop server. Print // out the first line of the error mesage, ignore the stack trace. exitCode = -1; try { String[] content; content = e.getLocalizedMessage().split("\n"); LOG.error(cmd.substring(1) + ": " + content[0]); } catch (Exception ex) { LOG.error(cmd.substring(1) + ": " + ex.getLocalizedMessage()); } } catch (IOException e) { // // IO exception encountered locally. // exitCode = -1; LOG.error(cmd.substring(1) + ": " + e.getLocalizedMessage()); } finally { // Does the RPC connection need to be closed? } return exitCode; } /** * Displays format of commands. * * @param cmd The command that is being executed. */ private void printUsage(String cmd) { if ("-geteditsize".equals(cmd)) { System.err.println("Usage: java SecondaryNameNode" + " [-geteditsize]"); } else if ("-checkpoint".equals(cmd)) { System.err.println("Usage: java SecondaryNameNode" + " [-checkpoint [force]]"); } else { System.err.println( "Usage: java SecondaryNameNode " + "[-checkpoint [force]] " + "[-geteditsize] "); } } /** * main() has some simple utility methods. * * @param argv Command line parameters. * @exception Exception if the filesystem does not exist. */ public static void main(String[] argv) throws Exception { StringUtils.startupShutdownMessage(SecondaryNameNode.class, argv, LOG); Configuration tconf = new Configuration(); if (argv.length >= 1) { SecondaryNameNode secondary = new SecondaryNameNode(tconf); int ret = secondary.processArgs(argv); System.exit(ret); } // Create a never ending deamon Daemon checkpointThread = new Daemon(new SecondaryNameNode(tconf)); checkpointThread.start(); } static class CheckpointStorage extends FSImage { /** */ CheckpointStorage(Configuration conf) throws IOException { super(conf); } @Override public boolean isConversionNeeded(StorageDirectory sd) { return false; } /** * Analyze checkpoint directories. Create directories if they do not exist. Recover from an * unsuccessful checkpoint is necessary. * * @param dataDirs * @param editsDirs * @throws IOException */ void recoverCreate(Collection<File> dataDirs, Collection<File> editsDirs) throws IOException { Collection<File> tempDataDirs = new ArrayList<File>(dataDirs); Collection<File> tempEditsDirs = new ArrayList<File>(editsDirs); this.storageDirs = new ArrayList<StorageDirectory>(); setStorageDirectories(tempDataDirs, tempEditsDirs); for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext(); ) { StorageDirectory sd = it.next(); boolean isAccessible = true; try { // create directories if don't exist yet if (!sd.getRoot().mkdirs()) { // do nothing, directory is already created } } catch (SecurityException se) { isAccessible = false; } if (!isAccessible) throw new InconsistentFSStateException( sd.getRoot(), "cannot access checkpoint directory."); StorageState curState; try { curState = sd.analyzeStorage(HdfsConstants.StartupOption.REGULAR); // sd is locked but not opened switch (curState) { case NON_EXISTENT: // fail if any of the configured checkpoint dirs are inaccessible throw new InconsistentFSStateException( sd.getRoot(), "checkpoint directory does not exist or is not accessible."); case NOT_FORMATTED: break; // it's ok since initially there is no current and VERSION case NORMAL: break; default: // recovery is possible sd.doRecover(curState); } } catch (IOException ioe) { sd.unlock(); throw ioe; } } } /** * Prepare directories for a new checkpoint. * * <p>Rename <code>current</code> to <code>lastcheckpoint.tmp</code> and recreate <code>current * </code>. * * @throws IOException */ void startCheckpoint() throws IOException { for (StorageDirectory sd : storageDirs) { moveCurrent(sd); } } void endCheckpoint() throws IOException { for (StorageDirectory sd : storageDirs) { moveLastCheckpoint(sd); } } /** Merge image and edits, and verify consistency with the signature. */ private void doMerge(CheckpointSignature sig, boolean loadImage) throws IOException { getEditLog().open(); StorageDirectory sdName = null; StorageDirectory sdEdits = null; Iterator<StorageDirectory> it = null; if (loadImage) { it = dirIterator(NameNodeDirType.IMAGE); if (it.hasNext()) sdName = it.next(); if (sdName == null) throw new IOException("Could not locate checkpoint fsimage"); } it = dirIterator(NameNodeDirType.EDITS); if (it.hasNext()) sdEdits = it.next(); if (sdEdits == null) throw new IOException("Could not locate checkpoint edits"); if (loadImage) { loadFSImage(FSImage.getImageFile(sdName, NameNodeFile.IMAGE)); } loadFSEdits(sdEdits); sig.validateStorageInfo(this); saveNamespace(false); } } }
/** This is <em>not</em> reusable. */ @SuppressWarnings("deprecation") public class BSONWritable implements BSONObject, WritableComparable { /** Constructs a new instance. */ public BSONWritable() { _doc = new BasicBSONObject(); } /** * Copy constructor, copies data from an existing BSONWritable * * @param other The BSONWritable to copy from */ public BSONWritable(BSONWritable other) { this(); copy(other); } /** Constructs a new instance around an existing BSONObject */ public BSONWritable(BSONObject doc) { this(); putAll(doc); } /** * {@inheritDoc} * * @see BSONObject#put(String,Object) */ public Object put(String key, Object value) { return _doc.put(key, value); } /** * {@inheritDoc} * * @see BSONObject#putAll(BSONObject) */ public void putAll(BSONObject otherDoc) { _doc.putAll(otherDoc); } /** * {@inheritDoc} * * @see BSONObject#putAll(Map) */ public void putAll(Map otherMap) { _doc.putAll(otherMap); } /** * {@inheritDoc} * * @see BSONObject#get(String) */ public Object get(String key) { return _doc.get(key); } /** * {@inheritDoc} * * @see BSONObject#toMap() */ public Map toMap() { return _doc.toMap(); } /** * {@inheritDoc} * * @see BSONObject#removeField(String) */ public Object removeField(String key) { return _doc.removeField(key); } /** * {@inheritDoc} * * @see BSONObject#containsKey(String) */ public boolean containsKey(String key) { return _doc.containsKey(key); } /** * {@inheritDoc} * * @see BSONObject#containsField(String) */ public boolean containsField(String fieldName) { return _doc.containsField(fieldName); } /** * {@inheritDoc} * * @see BSONObject#keySet() */ public Set<java.lang.String> keySet() { return _doc.keySet(); } /** * {@inheritDoc} * * @see Writable#write(DataOutput) */ public void write(DataOutput out) throws IOException { BSONEncoder enc = new BSONEncoder(); BasicOutputBuffer buf = new BasicOutputBuffer(); enc.set(buf); enc.putObject(_doc); enc.done(); out.writeInt(buf.size()); // For better performance we can copy BasicOutputBuffer.pipe(OutputStream) // to have a method signature that works with DataOutput out.write(buf.toByteArray()); } /** * {@inheritDoc} * * @see Writable#readFields(DataInput) */ public void readFields(DataInput in) throws IOException { BSONDecoder dec = new BSONDecoder(); BSONCallback cb = new BasicBSONCallback(); // Read the BSON length from the start of the record int dataLen = in.readInt(); byte[] buf = new byte[dataLen]; in.readFully(buf); dec.decode(buf, cb); _doc = (BSONObject) cb.get(); log.info("Decoded a BSON Object: " + _doc); } /** {@inheritDoc} */ @Override public String toString() { BSONEncoder enc = new BSONEncoder(); BasicOutputBuffer buf = new BasicOutputBuffer(); enc.set(buf); enc.putObject(_doc); enc.done(); String str = buf.asString(); log.debug("Output As String: '" + str + "'"); return str; } /** Used by child copy constructors. */ protected synchronized void copy(Writable other) { if (other != null) { try { DataOutputBuffer out = new DataOutputBuffer(); other.write(out); DataInputBuffer in = new DataInputBuffer(); in.reset(out.getData(), out.getLength()); readFields(in); } catch (IOException e) { throw new IllegalArgumentException("map cannot be copied: " + e.getMessage()); } } else { throw new IllegalArgumentException("source map cannot be null"); } } public static class Comparator extends WritableComparator { public Comparator() { super(BSONWritable.class); } public int compare(WritableComparable a, WritableComparable b) { if (a instanceof BSONWritable && b instanceof BSONWritable) { return ((BSONWritable) a)._doc.toString().compareTo(((BSONWritable) b)._doc.toString()); } else { return -1; } } } static { // register this comparator WritableComparator.define(BSONWritable.class, new Comparator()); } @Override public int compareTo(Object o) { return new Comparator().compare(this, o); } protected BSONObject _doc; private static final Log log = LogFactory.getLog(BSONWritable.class); }
/** * Base class that runs a task in a separate process. Tasks are run in a separate process in order * to isolate the map/reduce system code from bugs in user supplied map and reduce functions. */ abstract class TaskRunner extends Thread { public static final Log LOG = LogFactory.getLog(TaskRunner.class); volatile boolean killed = false; private TaskTracker.TaskInProgress tip; private Task t; private Object lock = new Object(); private volatile boolean done = false; private int exitCode = -1; private boolean exitCodeSet = false; private TaskTracker tracker; protected JobConf conf; JvmManager jvmManager; /** for cleaning up old map outputs */ protected MapOutputFile mapOutputFile; public TaskRunner(TaskTracker.TaskInProgress tip, TaskTracker tracker, JobConf conf) { this.tip = tip; this.t = tip.getTask(); this.tracker = tracker; this.conf = conf; this.mapOutputFile = new MapOutputFile(t.getJobID()); this.mapOutputFile.setConf(conf); this.jvmManager = tracker.getJvmManagerInstance(); } public Task getTask() { return t; } public TaskTracker.TaskInProgress getTaskInProgress() { return tip; } public TaskTracker getTracker() { return tracker; } /** * Called to assemble this task's input. This method is run in the parent process before the child * is spawned. It should not execute user code, only system code. */ public boolean prepare() throws IOException { return true; } /** * Called when this task's output is no longer needed. This method is run in the parent process * after the child exits. It should not execute user code, only system code. */ public void close() throws IOException {} private static String stringifyPathArray(Path[] p) { if (p == null) { return null; } StringBuffer str = new StringBuffer(p[0].toString()); for (int i = 1; i < p.length; i++) { str.append(","); str.append(p[i].toString()); } return str.toString(); } @Override public final void run() { try { // before preparing the job localize // all the archives TaskAttemptID taskid = t.getTaskID(); LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir"); File jobCacheDir = null; if (conf.getJar() != null) { jobCacheDir = new File(new Path(conf.getJar()).getParent().toString()); } File workDir = new File( lDirAlloc .getLocalPathToRead( TaskTracker.getJobCacheSubdir() + Path.SEPARATOR + t.getJobID() + Path.SEPARATOR + t.getTaskID() + Path.SEPARATOR + MRConstants.WORKDIR, conf) .toString()); URI[] archives = DistributedCache.getCacheArchives(conf); URI[] files = DistributedCache.getCacheFiles(conf); FileStatus fileStatus; FileSystem fileSystem; Path localPath; String baseDir; if ((archives != null) || (files != null)) { if (archives != null) { String[] archivesTimestamps = DistributedCache.getArchiveTimestamps(conf); Path[] p = new Path[archives.length]; for (int i = 0; i < archives.length; i++) { fileSystem = FileSystem.get(archives[i], conf); fileStatus = fileSystem.getFileStatus(new Path(archives[i].getPath())); String cacheId = DistributedCache.makeRelative(archives[i], conf); String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId; if (lDirAlloc.ifExists(cachePath, conf)) { localPath = lDirAlloc.getLocalPathToRead(cachePath, conf); } else { localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf); } baseDir = localPath.toString().replace(cacheId, ""); p[i] = DistributedCache.getLocalCache( archives[i], conf, new Path(baseDir), fileStatus, true, Long.parseLong(archivesTimestamps[i]), new Path(workDir.getAbsolutePath()), false); } DistributedCache.setLocalArchives(conf, stringifyPathArray(p)); } if ((files != null)) { String[] fileTimestamps = DistributedCache.getFileTimestamps(conf); Path[] p = new Path[files.length]; for (int i = 0; i < files.length; i++) { fileSystem = FileSystem.get(files[i], conf); fileStatus = fileSystem.getFileStatus(new Path(files[i].getPath())); String cacheId = DistributedCache.makeRelative(files[i], conf); String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId; if (lDirAlloc.ifExists(cachePath, conf)) { localPath = lDirAlloc.getLocalPathToRead(cachePath, conf); } else { localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf); } baseDir = localPath.toString().replace(cacheId, ""); p[i] = DistributedCache.getLocalCache( files[i], conf, new Path(baseDir), fileStatus, false, Long.parseLong(fileTimestamps[i]), new Path(workDir.getAbsolutePath()), false); } DistributedCache.setLocalFiles(conf, stringifyPathArray(p)); } Path localTaskFile = new Path(t.getJobFile()); FileSystem localFs = FileSystem.getLocal(conf); localFs.delete(localTaskFile, true); OutputStream out = localFs.create(localTaskFile); try { conf.writeXml(out); } finally { out.close(); } } if (!prepare()) { return; } String sep = System.getProperty("path.separator"); StringBuffer classPath = new StringBuffer(); // start with same classpath as parent process classPath.append(System.getProperty("java.class.path")); classPath.append(sep); if (!workDir.mkdirs()) { if (!workDir.isDirectory()) { LOG.fatal("Mkdirs failed to create " + workDir.toString()); } } String jar = conf.getJar(); if (jar != null) { // if jar exists, it into workDir File[] libs = new File(jobCacheDir, "lib").listFiles(); if (libs != null) { for (int i = 0; i < libs.length; i++) { classPath.append(sep); // add libs from jar to classpath classPath.append(libs[i]); } } classPath.append(sep); classPath.append(new File(jobCacheDir, "classes")); classPath.append(sep); classPath.append(jobCacheDir); } // include the user specified classpath // archive paths Path[] archiveClasspaths = DistributedCache.getArchiveClassPaths(conf); if (archiveClasspaths != null && archives != null) { Path[] localArchives = DistributedCache.getLocalCacheArchives(conf); if (localArchives != null) { for (int i = 0; i < archives.length; i++) { for (int j = 0; j < archiveClasspaths.length; j++) { if (archives[i].getPath().equals(archiveClasspaths[j].toString())) { classPath.append(sep); classPath.append(localArchives[i].toString()); } } } } } // file paths Path[] fileClasspaths = DistributedCache.getFileClassPaths(conf); if (fileClasspaths != null && files != null) { Path[] localFiles = DistributedCache.getLocalCacheFiles(conf); if (localFiles != null) { for (int i = 0; i < files.length; i++) { for (int j = 0; j < fileClasspaths.length; j++) { if (files[i].getPath().equals(fileClasspaths[j].toString())) { classPath.append(sep); classPath.append(localFiles[i].toString()); } } } } } classPath.append(sep); classPath.append(workDir); // Build exec child jmv args. Vector<String> vargs = new Vector<String>(8); File jvm = // use same jvm as parent new File(new File(System.getProperty("java.home"), "bin"), "java"); vargs.add(jvm.toString()); // Add child (task) java-vm options. // // The following symbols if present in mapred.child.java.opts value are // replaced: // + @taskid@ is interpolated with value of TaskID. // Other occurrences of @ will not be altered. // // Example with multiple arguments and substitutions, showing // jvm GC logging, and start of a passwordless JVM JMX agent so can // connect with jconsole and the likes to watch child memory, threads // and get thread dumps. // // <property> // <name>mapred.child.java.opts</name> // <value>-verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // String javaOpts = conf.get("mapred.child.java.opts", "-Xmx200m"); javaOpts = javaOpts.replace("@taskid@", taskid.toString()); String[] javaOptsSplit = javaOpts.split(" "); // Add java.library.path; necessary for loading native libraries. // // 1. To support native-hadoop library i.e. libhadoop.so, we add the // parent processes' java.library.path to the child. // 2. We also add the 'cwd' of the task to it's java.library.path to help // users distribute native libraries via the DistributedCache. // 3. The user can also specify extra paths to be added to the // java.library.path via mapred.child.java.opts. // String libraryPath = System.getProperty("java.library.path"); if (libraryPath == null) { libraryPath = workDir.getAbsolutePath(); } else { libraryPath += sep + workDir; } boolean hasUserLDPath = false; for (int i = 0; i < javaOptsSplit.length; i++) { if (javaOptsSplit[i].startsWith("-Djava.library.path=")) { javaOptsSplit[i] += sep + libraryPath; hasUserLDPath = true; break; } } if (!hasUserLDPath) { vargs.add("-Djava.library.path=" + libraryPath); } for (int i = 0; i < javaOptsSplit.length; i++) { vargs.add(javaOptsSplit[i]); } // add java.io.tmpdir given by mapred.child.tmp String tmp = conf.get("mapred.child.tmp", "./tmp"); Path tmpDir = new Path(tmp); // if temp directory path is not absolute // prepend it with workDir. if (!tmpDir.isAbsolute()) { tmpDir = new Path(workDir.toString(), tmp); } FileSystem localFs = FileSystem.getLocal(conf); if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) { throw new IOException("Mkdirs failed to create " + tmpDir.toString()); } vargs.add("-Djava.io.tmpdir=" + tmpDir.toString()); // Add classpath. vargs.add("-classpath"); vargs.add(classPath.toString()); // Setup the log4j prop long logSize = TaskLog.getTaskLogLength(conf); vargs.add( "-Dhadoop.log.dir=" + new File(System.getProperty("hadoop.log.dir")).getAbsolutePath()); vargs.add("-Dhadoop.root.logger=INFO,TLA"); vargs.add("-Dhadoop.tasklog.taskid=" + taskid); vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize); if (conf.getProfileEnabled()) { if (conf.getProfileTaskRange(t.isMapTask()).isIncluded(t.getPartition())) { File prof = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.PROFILE); vargs.add(String.format(conf.getProfileParams(), prof.toString())); } } // Add main class and its arguments vargs.add(Child.class.getName()); // main of Child // pass umbilical address InetSocketAddress address = tracker.getTaskTrackerReportAddress(); vargs.add(address.getAddress().getHostAddress()); vargs.add(Integer.toString(address.getPort())); vargs.add(taskid.toString()); // pass task identifier String pidFile = null; if (tracker.isTaskMemoryManagerEnabled()) { pidFile = lDirAlloc .getLocalPathForWrite( (TaskTracker.getPidFilesSubdir() + Path.SEPARATOR + taskid), this.conf) .toString(); } // set memory limit using ulimit if feasible and necessary ... String[] ulimitCmd = Shell.getUlimitMemoryCommand(conf); List<String> setup = null; if (ulimitCmd != null) { setup = new ArrayList<String>(); for (String arg : ulimitCmd) { setup.add(arg); } } // Set up the redirection of the task's stdout and stderr streams File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT); File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR); stdout.getParentFile().mkdirs(); tracker.getTaskTrackerInstrumentation().reportTaskLaunch(taskid, stdout, stderr); Map<String, String> env = new HashMap<String, String>(); StringBuffer ldLibraryPath = new StringBuffer(); ldLibraryPath.append(workDir.toString()); String oldLdLibraryPath = null; oldLdLibraryPath = System.getenv("LD_LIBRARY_PATH"); if (oldLdLibraryPath != null) { ldLibraryPath.append(sep); ldLibraryPath.append(oldLdLibraryPath); } env.put("LD_LIBRARY_PATH", ldLibraryPath.toString()); jvmManager.launchJvm( this, jvmManager.constructJvmEnv( setup, vargs, stdout, stderr, logSize, workDir, env, pidFile, conf)); synchronized (lock) { while (!done) { lock.wait(); } } tracker.getTaskTrackerInstrumentation().reportTaskEnd(t.getTaskID()); if (exitCodeSet) { if (!killed && exitCode != 0) { if (exitCode == 65) { tracker.getTaskTrackerInstrumentation().taskFailedPing(t.getTaskID()); } throw new IOException("Task process exit with nonzero status of " + exitCode + "."); } } } catch (FSError e) { LOG.fatal("FSError", e); try { tracker.fsError(t.getTaskID(), e.getMessage()); } catch (IOException ie) { LOG.fatal(t.getTaskID() + " reporting FSError", ie); } } catch (Throwable throwable) { LOG.warn(t.getTaskID() + " Child Error", throwable); ByteArrayOutputStream baos = new ByteArrayOutputStream(); throwable.printStackTrace(new PrintStream(baos)); try { tracker.reportDiagnosticInfo(t.getTaskID(), baos.toString()); } catch (IOException e) { LOG.warn(t.getTaskID() + " Reporting Diagnostics", e); } } finally { try { URI[] archives = DistributedCache.getCacheArchives(conf); URI[] files = DistributedCache.getCacheFiles(conf); if (archives != null) { for (int i = 0; i < archives.length; i++) { DistributedCache.releaseCache(archives[i], conf); } } if (files != null) { for (int i = 0; i < files.length; i++) { DistributedCache.releaseCache(files[i], conf); } } } catch (IOException ie) { LOG.warn("Error releasing caches : Cache files might not have been cleaned up"); } tracker.reportTaskFinished(t.getTaskID(), false); if (t.isMapTask()) { tracker.addFreeMapSlot(); } else { tracker.addFreeReduceSlot(); } } } // Mostly for setting up the symlinks. Note that when we setup the distributed // cache, we didn't create the symlinks. This is done on a per task basis // by the currently executing task. public static void setupWorkDir(JobConf conf) throws IOException { File workDir = new File(".").getAbsoluteFile(); FileUtil.fullyDelete(workDir); if (DistributedCache.getSymlink(conf)) { URI[] archives = DistributedCache.getCacheArchives(conf); URI[] files = DistributedCache.getCacheFiles(conf); Path[] localArchives = DistributedCache.getLocalCacheArchives(conf); Path[] localFiles = DistributedCache.getLocalCacheFiles(conf); if (archives != null) { for (int i = 0; i < archives.length; i++) { String link = archives[i].getFragment(); if (link != null) { link = workDir.toString() + Path.SEPARATOR + link; File flink = new File(link); if (!flink.exists()) { FileUtil.symLink(localArchives[i].toString(), link); } } } } if (files != null) { for (int i = 0; i < files.length; i++) { String link = files[i].getFragment(); if (link != null) { link = workDir.toString() + Path.SEPARATOR + link; File flink = new File(link); if (!flink.exists()) { FileUtil.symLink(localFiles[i].toString(), link); } } } } } File jobCacheDir = null; if (conf.getJar() != null) { jobCacheDir = new File(new Path(conf.getJar()).getParent().toString()); } // create symlinks for all the files in job cache dir in current // workingdir for streaming try { DistributedCache.createAllSymlink(conf, jobCacheDir, workDir); } catch (IOException ie) { // Do not exit even if symlinks have not been created. LOG.warn(StringUtils.stringifyException(ie)); } // add java.io.tmpdir given by mapred.child.tmp String tmp = conf.get("mapred.child.tmp", "./tmp"); Path tmpDir = new Path(tmp); // if temp directory path is not absolute // prepend it with workDir. if (!tmpDir.isAbsolute()) { tmpDir = new Path(workDir.toString(), tmp); FileSystem localFs = FileSystem.getLocal(conf); if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) { throw new IOException("Mkdirs failed to create " + tmpDir.toString()); } } } /** Kill the child process */ public void kill() { killed = true; jvmManager.taskKilled(this); signalDone(); } public void signalDone() { synchronized (lock) { done = true; lock.notify(); } } public void setExitCode(int exitCode) { this.exitCodeSet = true; this.exitCode = exitCode; } }
/** * Created with IntelliJ IDEA. User: ciobi Date: 2013-06-15 Time: 09:56 * * <p> */ public class ReaderHandler extends WebAppContext { public static final Log LOG = LogFactory.getLog(ReaderHandler.class); // ttt1 option that on http only redirects to https, for all paths public static final String ACTION_LOGIN = "******"; public static final String ACTION_SIGNUP = "signup"; public static final String ACTION_CHANGE_PASSWORD = "******"; public static final String ACTION_CHANGE_SETTINGS = "change_settings"; public static final String ACTION_ADD_FEED = "add_feed"; public static final String ACTION_REMOVE_FEED = "remove_feed"; public static final String ACTION_UPDATE_FEED_LIST = "update_feed_list"; // for ordering, //ttt2 public static final String PATH_LOGIN = "******" + ACTION_LOGIN; public static final String PATH_CHANGE_PASSWORD = "******" + ACTION_CHANGE_PASSWORD; public static final String PATH_CHANGE_SETTINGS = "/" + ACTION_CHANGE_SETTINGS; public static final String PATH_SIGNUP = "/" + ACTION_SIGNUP; public static final String PATH_ADD_FEED = "/" + ACTION_ADD_FEED; public static final String PATH_REMOVE_FEED = "/" + ACTION_REMOVE_FEED; public static final String PATH_UPDATE_FEED_LIST = "/" + ACTION_UPDATE_FEED_LIST; public static final String PATH_ERROR = "/error"; public static final String PATH_LOGOUT = "/logout"; public static final String PATH_SETTINGS = "/settings"; public static final String PATH_FEEDS = "/feeds"; public static final String PATH_FEED = "/feed"; public static final String PATH_ADMIN = "/admin"; public static final String PATH_FEED_ADMIN = "/feed_admin"; public static final String PATH_OPEN_ARTICLE = "/open_article/"; // !!! it's easier to end this one with a slash // params we use to send strings to the JSPs or to get user input in POST, via // request.getParameter(), or both public static final String PARAM_USER_ID = "userId"; public static final String PARAM_USER_NAME = "name"; public static final String PARAM_EMAIL = "email"; public static final String PARAM_CURRENT_PASSWORD = "******"; public static final String PARAM_PASSWORD = "******"; public static final String PARAM_PASSWORD_CONFIRM = "passwordConfirm"; public static final String PARAM_PATH = "path"; // public static final String PARAM_ERROR = "error"; public static final String PARAM_REMEMBER_ACCOUNT = "rememberAccount"; public static final String PARAM_NEW_FEED_URL = "feedUrl"; public static final String PARAM_FEED_ID = "feedId"; public static final String PARAM_ITEMS_PER_PAGE = "itemsPerPage"; public static final String PARAM_STYLE = "style"; public static final String PARAM_FEED_DATE_FORMAT = "feedDateFormat"; // variable names, used to give JSPs access to Java objects in the handler via // request.getAttribute(() public static final String VAR_USER = "******"; public static final String VAR_LOGIN_INFO = "loginInfo"; public static final String VAR_USER_DB = "userDb"; public static final String VAR_FEED_DB = "feedDb"; public static final String VAR_ARTICLE_DB = "articleDb"; public static final String VAR_READ_ARTICLES_COLL_DB = "readArticlesCollDb"; public static final String BROWSER_ID = "browserId"; public static final String SESSION_ID = "sessionId"; private LoginInfo.DB loginInfoDb; private User.DB userDb; private Feed.DB feedDb; private Article.DB articleDb; private ReadArticlesColl.DB readArticlesCollDb; private UserHelpers userHelpers; private boolean isInJar = Utils.isInJar(); private static class ReaderErrorHandler extends ErrorHandler { @Override // !!! note that this gets called for missing pages, but not if exceptions are thrown; // exceptions are handled separately public void handle( String target, Request request, HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) throws IOException { request.setHandled(true); httpServletResponse .getWriter() .println( String.format("<h1>Page doesn't exist: %s</h1>", request.getUri().getDecodedPath())); } } private static HashMap<String, String> PATH_MAPPING = new HashMap<>(); static { PATH_MAPPING.put("", "home_page"); PATH_MAPPING.put(PATH_LOGIN, "login"); PATH_MAPPING.put(PATH_LOGOUT, "login"); // !!! after logout we get redirected to /login PATH_MAPPING.put(PATH_SIGNUP, "signup"); PATH_MAPPING.put(PATH_ERROR, "error"); PATH_MAPPING.put(PATH_FEED_ADMIN, "feed_admin"); PATH_MAPPING.put(PATH_SETTINGS, "settings"); PATH_MAPPING.put(PATH_FEEDS, "feeds"); PATH_MAPPING.put(PATH_FEED + "/*", "feed"); PATH_MAPPING.put(PATH_ADMIN, "admin"); } public ReaderHandler(LowLevelDbAccess lowLevelDbAccess, String webDir) { loginInfoDb = new LoginInfo.DB(lowLevelDbAccess); userDb = new User.DB(lowLevelDbAccess); feedDb = new Feed.DB(lowLevelDbAccess); articleDb = new Article.DB(lowLevelDbAccess); readArticlesCollDb = new ReadArticlesColl.DB(lowLevelDbAccess); userHelpers = new UserHelpers(loginInfoDb, userDb); setContextPath("/"); File warPath = new File(webDir); setWar(warPath.getAbsolutePath()); if (isInJar) { for (Map.Entry<String, String> entry : PATH_MAPPING.entrySet()) { addPrebuiltJsp(entry.getKey(), "jsp." + entry.getValue().replaceAll("_", "_005f") + "_jsp"); } } else { for (Map.Entry<String, String> entry : PATH_MAPPING.entrySet()) { addServlet( new ServletHolder(new RedirectServlet("/" + entry.getValue() + ".jsp")), entry.getKey()); } } setErrorHandler(new ReaderErrorHandler()); } private void addPrebuiltJsp(String path, String className) { try { Class clazz = Class.forName( className); // ttt2 see if possible to not use this, preferably without doing // redirections like RedirectServlet Object obj = clazz.newInstance(); addServlet(new ServletHolder((Servlet) obj), path); LOG.info("Added prebuilt JSP: " + obj.toString()); } catch (Exception e) { LOG.fatal(String.format("Failed to load prebuilt JSP for %s and %s", path, className), e); } } @Override public void doHandle( String target, Request request, HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) throws IOException, ServletException { LOG.info("handling " + target); // !!! doHandle() is called twice for a request when using redirectiion, first time with // request.getPathInfo() // set to the URI and target set to the path, then with request.getPathInfo() set to null and // target set to the .jsp try { // request.setHandled(true); boolean secured; if (request.getScheme().equals("https")) { secured = true; } else if (request.getScheme().equals("http")) { secured = false; } else { httpServletResponse .getWriter() .println( String.format( "<h1>Unknown scheme %s at %s</h1>", request.getScheme(), request.getUri().getDecodedPath())); return; } if (request.getMethod().equals("GET")) { if (isInJar || target.endsWith(".jsp")) { // !!! when not in jar there's no need to do anything about params if it's not a .jsp, // as this will get called again for the corresponding .jsp if (prepareForJspGet(target, request, httpServletResponse, secured)) { return; } } if (target.startsWith(PATH_OPEN_ARTICLE)) { handleOpenArticle(request, httpServletResponse, target); return; } super.doHandle(target, request, httpServletRequest, httpServletResponse); LOG.info("handling of " + target + " went to super"); // httpServletResponse.setDateHeader("Date", System.currentTimeMillis()); //ttt2 review // these, probably not use // httpServletResponse.setDateHeader("Expires", System.currentTimeMillis() + 60000); return; } if (request.getMethod().equals("POST")) { if (request.getUri().getDecodedPath().equals(PATH_LOGIN)) { handleLoginPost(request, httpServletResponse, secured); } else if (request.getUri().getDecodedPath().equals(PATH_SIGNUP)) { handleSignupPost(request, httpServletResponse); } else if (request.getUri().getDecodedPath().equals(PATH_CHANGE_PASSWORD)) { handleChangePasswordPost(request, httpServletResponse); } else if (request.getUri().getDecodedPath().equals(PATH_UPDATE_FEED_LIST)) { handleUpdateFeedListPost(request, httpServletResponse); } else if (request.getUri().getDecodedPath().equals(PATH_ADD_FEED)) { handleAddFeedPost(request, httpServletResponse); } else if (request.getUri().getDecodedPath().equals(PATH_REMOVE_FEED)) { handleRemoveFeedPost(request, httpServletResponse); } else if (request.getUri().getDecodedPath().equals(PATH_CHANGE_SETTINGS)) { handleChangeSettingsPost(request, httpServletResponse); } } /*{ // for tests only; httpServletResponse.getWriter().println(String.format("<h1>Unable to process request %s</h1>", request.getUri().getDecodedPath())); request.setHandled(true); }*/ } catch (Exception e) { LOG.error("Error processing request", e); try { // redirectToError(e.toString(), request, httpServletResponse); //!!! redirectToError leads // to infinite loop, probably related to // the fact that we get 2 calls for a regular request when redirecting httpServletResponse .getWriter() .println( String.format( "<h1>Unable to process request %s</h1>", // ttt1 generate some HTML request.getUri().getDecodedPath())); request.setHandled(true); } catch (Exception e1) { LOG.error("Error redirecting", e1); } } } /** * Normally sets the path and a few attributes that the JSPs are likely to need. Also verifies the * login information. If necessary, just redirects to the login page. * * @param target * @param request * @param httpServletResponse * @param secured * @return true if the request is already handled so the .jsp shouldn't get called * @throws Exception */ private boolean prepareForJspGet( String target, Request request, HttpServletResponse httpServletResponse, boolean secured) throws Exception { LoginInfo.SessionInfo sessionInfo = UserHelpers.getSessionInfo(request); LOG.info( String.format( "hndl - %s ; %s; %s ; %s", target, request.getPathInfo(), request.getMethod(), secured ? "secured" : "not secured")); String path = request.getUri().getDecodedPath(); boolean redirectToLogin = path.equals(PATH_LOGOUT); LoginInfo loginInfo = null; if (sessionInfo.isNull()) { redirectToLogin = true; LOG.info("Null session info. Logging in again."); } else { loginInfo = loginInfoDb.get( sessionInfo.browserId, sessionInfo.sessionId); // ttt2 use a cache, to avoid going to DB if (loginInfo == null || loginInfo.expiresOn < System.currentTimeMillis()) { LOG.info("Session has expired. Logging in again. Info: " + loginInfo); redirectToLogin = true; } } if (!path.equals(PATH_LOGIN) && !path.equals(PATH_SIGNUP) && !path.equals(PATH_ERROR)) { if (redirectToLogin) { // ttt2 perhaps store URI, to return to it after login logOut(sessionInfo.browserId); addLoginParams(request, loginInfo); httpServletResponse.sendRedirect(PATH_LOGIN); return true; } User user = userDb.get(loginInfo.userId); if (user == null) { WebUtils.redirectToError("Unknown user", request, httpServletResponse); return true; } if (!user.active) { WebUtils.redirectToError("Account is not active", request, httpServletResponse); return true; } request.setAttribute(VAR_FEED_DB, feedDb); request.setAttribute(VAR_USER_DB, userDb); request.setAttribute(VAR_ARTICLE_DB, articleDb); request.setAttribute(VAR_READ_ARTICLES_COLL_DB, readArticlesCollDb); request.setAttribute(VAR_USER, user); request.setAttribute(VAR_LOGIN_INFO, loginInfo); MultiMap<String> params = new MultiMap<>(); params.put(PARAM_PATH, path); request.setParameters(params); } if (path.equals(PATH_LOGIN)) { addLoginParams(request, loginInfo); } return false; } private void handleOpenArticle( Request request, HttpServletResponse httpServletResponse, String target) throws Exception { try { int k1 = target.indexOf('/', 1); int k2 = target.indexOf('/', k1 + 1); String feedId = target.substring(k1 + 1, k2); String strSeq = target.substring(k2 + 1); int seq = Integer.parseInt(strSeq); Article article = articleDb.get(feedId, seq); LoginInfo loginInfo = userHelpers.getLoginInfo(request); // ttt2 using the link from a non-authenticated browser causes a NPE; maybe do something // better, e.g. sign up ReadArticlesColl readArticlesColl = readArticlesCollDb.get(loginInfo.userId, feedId); if (readArticlesColl == null) { readArticlesColl = new ReadArticlesColl(loginInfo.userId, feedId); } if (!readArticlesColl.isRead(seq)) { readArticlesColl.markRead(seq, Config.getConfig().maxSizeForReadArticles); readArticlesCollDb.add(readArticlesColl); } String s = URIUtil.encodePath(article.url) .replace("%3F", "?") .replace("%23", "#"); // ttt2 see how to do this right httpServletResponse.sendRedirect(s); } catch (Exception e) { WebUtils.showResult( String.format("Failed to get article for path %s. %s", target, e), "/", request, httpServletResponse); } } private void handleSignupPost(Request request, HttpServletResponse httpServletResponse) throws Exception { String userId = request.getParameter(PARAM_USER_ID); String userName = request.getParameter(PARAM_USER_NAME); String email = request.getParameter(PARAM_EMAIL); String stringPassword = request.getParameter(PARAM_PASSWORD); String stringPasswordConfirm = request.getParameter(PARAM_PASSWORD_CONFIRM); if (!stringPassword.equals(stringPasswordConfirm)) { WebUtils.redirectToError( "Mismatch between password and password confirmation", request, httpServletResponse); return; } SecureRandom secureRandom = new SecureRandom(); String salt = "" + secureRandom.nextLong(); byte[] password = User.computeHashedPassword(stringPassword, salt); User user = userDb.get(userId); if (user != null) { WebUtils.redirectToError( "There already exists a user with the ID " + userId, request, httpServletResponse); return; } user = new User( userId, userName, password, salt, email, new ArrayList<String>(), Config.getConfig().activateAccountsAtCreation, false); // ttt2 add confirmation by email, captcha, ... List<String> fieldErrors = user.checkFields(); if (!fieldErrors.isEmpty()) { StringBuilder bld = new StringBuilder("Invalid values when trying to create user with ID ") .append(userId) .append("<br/>"); for (String s : fieldErrors) { bld.append(s).append("<br/>"); } WebUtils.redirectToError(bld.toString(), request, httpServletResponse); return; } // ttt2 2 clients can add the same userId simultaneously userDb.add(user); httpServletResponse.sendRedirect("/"); } private void handleChangePasswordPost(Request request, HttpServletResponse httpServletResponse) throws Exception { LoginInfo loginInfo = userHelpers.getLoginInfo(request); if (loginInfo == null) { WebUtils.redirectToError("Couldn't determine the current user", request, httpServletResponse); return; } String userId = loginInfo.userId; String stringCrtPassword = request.getParameter(PARAM_CURRENT_PASSWORD); String stringNewPassword = request.getParameter(PARAM_PASSWORD); String stringNewPasswordConfirm = request.getParameter(PARAM_PASSWORD_CONFIRM); if (!stringNewPassword.equals(stringNewPasswordConfirm)) { showResult( "Mismatch between password and password confirmation", PATH_SETTINGS, request, httpServletResponse); return; } User user = userDb.get( userId); // ttt1 crashes for wrong ID; 2013.07.20 - no longer have an idea what this is // about if (user == null) { WebUtils.redirectToError("Couldn't find the current user", request, httpServletResponse); return; } if (!user.checkPassword(stringCrtPassword)) { showResult("Incorrect current password", PATH_SETTINGS, request, httpServletResponse); return; } SecureRandom secureRandom = new SecureRandom(); String salt = "" + secureRandom.nextLong(); byte[] password = User.computeHashedPassword(stringNewPassword, salt); user.salt = salt; user.password = password; // ttt3 2 clients can change the password simultaneously userDb.add(user); // httpServletResponse.sendRedirect(PATH_SETTINGS); showResult("Password changed", PATH_SETTINGS, request, httpServletResponse); } private void handleChangeSettingsPost(Request request, HttpServletResponse httpServletResponse) throws Exception { LoginInfo loginInfo = userHelpers.getLoginInfo(request); if (loginInfo == null) { WebUtils.redirectToError("Couldn't determine the current user", request, httpServletResponse); return; } String stringItemsPerPage = request.getParameter(PARAM_ITEMS_PER_PAGE); try { loginInfo.itemsPerPage = Integer.parseInt(stringItemsPerPage); } catch (Exception e) { showResult( "Error trying to set the items per page. Expected integer value but got " + stringItemsPerPage, PATH_SETTINGS, request, httpServletResponse); return; } loginInfo.style = request.getParameter(PARAM_STYLE); loginInfo.feedDateFormat = request.getParameter(PARAM_FEED_DATE_FORMAT); // ttt2 validate, better in JSP loginInfoDb.add(loginInfo); // httpServletResponse.sendRedirect(PATH_SETTINGS); showResult("Settings changed", "/", request, httpServletResponse); } private void handleUpdateFeedListPost(Request request, HttpServletResponse httpServletResponse) throws Exception { LOG.info("updating feed list"); // ttt2 implement httpServletResponse.sendRedirect(PATH_FEED_ADMIN); } private void handleAddFeedPost(Request request, HttpServletResponse httpServletResponse) throws Exception { LOG.info("adding feed"); User user = userHelpers.getUser(request); try { if (user == null) { LOG.error("User not found"); return; } String url = request.getParameter(PARAM_NEW_FEED_URL); // ttt1 add some validation; probably best try to actually get data, set the title, ... if (url == null || url.equals("")) { LOG.error("New feed not specified"); // ttt1 show some error return; } MessageDigest digest = MessageDigest.getInstance("MD5"); String feedId = PrintUtils.byteArrayAsUrlString(digest.digest(url.getBytes("UTF-8"))); feedId = feedId.substring(0, Config.getConfig().feedIdSize); Feed feed = feedDb.get(feedId); if (feed == null) { feed = new Feed(feedId, url); feedDb.add(feed); } if (user.feedIds.contains(feedId)) { LOG.error(String.format("Trying to add existing feed %s to user %s", feedId, user)); } else { user.feedIds.add(feedId); userDb.updateFeeds(user); } } finally { httpServletResponse.sendRedirect(PATH_FEED_ADMIN); } } private void handleRemoveFeedPost(Request request, HttpServletResponse httpServletResponse) throws Exception { LOG.info("removing feed"); User user = userHelpers.getUser(request); try { if (user == null) { LOG.error("User not found"); return; } String feedId = request.getParameter(PARAM_FEED_ID); LOG.info(String.format("Removing feed %s for user %s", feedId, user)); // ttt1 add some validation; probably best try to actually get data, set the title, ... if (feedId == null || feedId.equals("")) { LOG.error("feed not specified"); // ttt1 show some error return; } if (user.feedIds.remove( feedId)) { // ttt2 clean up the global feed table; that's probably better done if nobody // accesses a feed for 3 months or so userDb.updateFeeds(user); LOG.info(String.format("Removed feed %s for user %s", feedId, user)); } else { LOG.info(String.format("No feed found with ID %s for user %s", feedId, user)); } } finally { httpServletResponse.sendRedirect(PATH_FEED_ADMIN); } } private void handleLoginPost( Request request, HttpServletResponse httpServletResponse, boolean secured) throws Exception { String userId = request.getParameter(PARAM_USER_ID); String password = request.getParameter(PARAM_PASSWORD); String rememberAccountStr = request.getParameter(PARAM_REMEMBER_ACCOUNT); boolean rememberAccount = Boolean.parseBoolean(rememberAccountStr); LoginInfo.SessionInfo sessionInfo = UserHelpers.getSessionInfo(request); logOut(sessionInfo.browserId); User user = userDb.get(userId); if (user == null) { WebUtils.redirectToError("User " + userId + " not found", request, httpServletResponse); return; } if (!user.checkPassword(password)) { WebUtils.redirectToError("Invalid password", request, httpServletResponse); return; } if (!user.active) { WebUtils.redirectToError( "Account for User " + userId + " needs to be activated", request, httpServletResponse); return; } LOG.info("Logged in user " + userId); sessionInfo.sessionId = null; if (sessionInfo.browserId == null) { sessionInfo.browserId = getRandomId(); } else { for (LoginInfo loginInfo : loginInfoDb.getLoginsForBrowser(sessionInfo.browserId)) { if (userId.equals(loginInfo.userId)) { sessionInfo.sessionId = loginInfo.sessionId; break; } } } long expireOn = System.currentTimeMillis() + Config.getConfig().loginExpireInterval; if (sessionInfo.sessionId == null) { sessionInfo.sessionId = getRandomId(); Config config = Config.getConfig(); loginInfoDb.add( new LoginInfo( sessionInfo.browserId, sessionInfo.sessionId, userId, expireOn, rememberAccount, config.defaultStyle, config.defaultItemsPerPage, config.defaultFeedDateFormat)); LOG.info(String.format("Logging in in a new session. User: %s", user)); } else { loginInfoDb.updateExpireTime(sessionInfo.browserId, sessionInfo.sessionId, expireOn); LOG.info(String.format("Logging in in an existing session. User: %s", user)); } WebUtils.saveCookies( httpServletResponse, secured, sessionInfo.browserId, sessionInfo.sessionId); httpServletResponse.sendRedirect("/"); } private String getRandomId() { SecureRandom secureRandom = new SecureRandom(); return "" + secureRandom.nextLong(); } private void addLoginParams(Request request, LoginInfo loginInfo) { MultiMap<String> params = new MultiMap<>(); if (loginInfo != null && loginInfo.rememberAccount) { params.put(PARAM_USER_ID, loginInfo.userId); } request.setParameters(params); } private void logOut(String browserId) throws Exception { // ttt2 the right way to do it is to go through all the sessions of the current browser, which // would require a new field and a new index; // not sure if it's worth it, but this would work: A logs in, forgets to log out, B delets the // cookies, logs in, A sees B is logged in, then B // restores the cookies and uses A's account if (browserId == null) { return; } List<LoginInfo> loginInfos = loginInfoDb.getLoginsForBrowser(browserId); long expireTarget = System.currentTimeMillis() - Utils.ONE_DAY; for (LoginInfo loginInfo : loginInfos) { if (loginInfo.expiresOn <= expireTarget) { LOG.info(String.format("LoginInfo %s is enough in the past", loginInfo)); } else { LOG.info(String.format("Logging out: %s", loginInfo)); loginInfoDb.updateExpireTime(browserId, loginInfo.sessionId, expireTarget); } } } public static class FeedInfo { public String feedId; public int maxSeq; public FeedInfo(String feedId, int maxSeq) { this.feedId = feedId; this.maxSeq = maxSeq; } } // !!! IDEA reports this as unused, but it is called from JSP public static FeedInfo getFeedInfo(String feedPath) { if (feedPath.startsWith(PATH_FEED + "/")) { try { if (feedPath.endsWith("/")) { feedPath = feedPath.substring(0, feedPath.length() - 1); } int k = PATH_FEED.length() + 1; int p = feedPath.indexOf('/', k); return p >= 0 ? new FeedInfo(feedPath.substring(k, p), Integer.parseInt(feedPath.substring(p + 1))) : new FeedInfo(feedPath.substring(k), -1); } catch (Exception e) { LOG.error("Exception trying to parse the feed info", e); } } LOG.error("Invalid path from feed: " + feedPath); return new FeedInfo("INVALID", -1); } // !!! IDEA reports this as unused, but it is called from JSP public static String getStyle(LoginInfo loginInfo) { StringBuilder bld = new StringBuilder(); bld.append("<style media=\"screen\" type=\"text/css\">\n\n"); if (loginInfo == null) { bld.append(Config.getConfig().defaultStyle); } else { bld.append(loginInfo.style); // ttt3 detect broken styles and return default } bld.append("</style>\n"); return bld.toString(); } /* private void jspCodeCheck() throws Exception { Article.DB articleDb; Request request; String path = ""; String feedId = ReaderHandler.getFeedId(path); int maxSeq = ReaderHandler.getSeq(path); Feed.DB feedDb = (Feed.DB)request.getAttribute(ReaderHandler.VAR_FEED_DB); Feed feed = feedDb.get(feedId); if (feed == null) { out.println("Feed " + feedId + " not found"); } else { if (maxSeq == -1) { maxSeq = feed.maxSeq; } if (maxSeq < 0) { out.println("Feed " + feedId + " is empty"); } else { ++maxSeq; LoginInfo loginInfo = (LoginInfo)request.getAttribute(ReaderHandler.VAR_LOGIN_INFO); int minSeq = Math.max(maxSeq - loginInfo.itemsPerPage, 0); List<Article> articles = articleDb.get(feedId, minSeq, maxSeq); for (Article article : articles) { out.println("<a href=\"" + article.url + "\">" + article.title + "</a><br/>"); } } } } //*/ }
public class IdentitySchema { private static final String IDENTITY_TABLE_PREFIX = "JBPM_ID_"; Configuration configuration = null; Properties properties = null; Dialect dialect = null; Mapping mapping = null; String[] createSql = null; String[] dropSql = null; String[] cleanSql = null; ConnectionProvider connectionProvider = null; Connection connection = null; Statement statement = null; public IdentitySchema(Configuration configuration) { this.configuration = configuration; this.properties = configuration.getProperties(); this.dialect = Dialect.getDialect(properties); try { // get the mapping field via reflection :-( Field mappingField = Configuration.class.getDeclaredField("mapping"); mappingField.setAccessible(true); this.mapping = (Mapping) mappingField.get(configuration); } catch (Exception e) { throw new RuntimeException("couldn't get the hibernate mapping", e); } } // scripts lazy initializations ///////////////////////////////////////////// public String[] getCreateSql() { if (createSql == null) { createSql = configuration.generateSchemaCreationScript(dialect); } return createSql; } public String[] getDropSql() { if (dropSql == null) { dropSql = configuration.generateDropSchemaScript(dialect); } return dropSql; } public String[] getCleanSql() { if (cleanSql == null) { // loop over all foreign key constraints List dropForeignKeysSql = new ArrayList(); List createForeignKeysSql = new ArrayList(); Iterator iter = configuration.getTableMappings(); while (iter.hasNext()) { Table table = (Table) iter.next(); if (table.isPhysicalTable()) { Iterator subIter = table.getForeignKeyIterator(); while (subIter.hasNext()) { ForeignKey fk = (ForeignKey) subIter.next(); if (fk.isPhysicalConstraint()) { // collect the drop key constraint dropForeignKeysSql.add( fk.sqlDropString( dialect, properties.getProperty(Environment.DEFAULT_CATALOG), properties.getProperty(Environment.DEFAULT_SCHEMA))); createForeignKeysSql.add( fk.sqlCreateString( dialect, mapping, properties.getProperty(Environment.DEFAULT_CATALOG), properties.getProperty(Environment.DEFAULT_SCHEMA))); } } } } List deleteSql = new ArrayList(); iter = configuration.getTableMappings(); while (iter.hasNext()) { Table table = (Table) iter.next(); deleteSql.add("delete from " + table.getName()); } List cleanSqlList = new ArrayList(); cleanSqlList.addAll(dropForeignKeysSql); cleanSqlList.addAll(deleteSql); cleanSqlList.addAll(createForeignKeysSql); cleanSql = (String[]) cleanSqlList.toArray(new String[cleanSqlList.size()]); } return cleanSql; } // runtime table detection ////////////////////////////////////////////////// public boolean hasIdentityTables() { return (getIdentityTables().size() > 0); } public List getIdentityTables() { // delete all the data in the jbpm tables List jbpmTableNames = new ArrayList(); try { createConnection(); ResultSet resultSet = connection.getMetaData().getTables("", "", null, null); while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if ((tableName != null) && (tableName.length() > 5) && (IDENTITY_TABLE_PREFIX.equalsIgnoreCase(tableName.substring(0, 5)))) { jbpmTableNames.add(tableName); } } } catch (SQLException e) { throw new RuntimeException("couldn't get the jbpm table names"); } finally { closeConnection(); } return jbpmTableNames; } // script execution methods ///////////////////////////////////////////////// public void dropSchema() { execute(getDropSql()); } public void createSchema() { execute(getCreateSql()); } public void cleanSchema() { execute(getCleanSql()); } public void saveSqlScripts(String dir, String prefix) { try { new File(dir).mkdirs(); saveSqlScript(dir + "/" + prefix + ".drop.sql", getDropSql()); saveSqlScript(dir + "/" + prefix + ".create.sql", getCreateSql()); saveSqlScript(dir + "/" + prefix + ".clean.sql", getCleanSql()); new SchemaExport(configuration) .setDelimiter(getSqlDelimiter()) .setOutputFile(dir + "/" + prefix + ".drop.create.sql") .create(true, false); } catch (Exception e) { throw new RuntimeException("couldn't generate scripts", e); } } // main ///////////////////////////////////////////////////////////////////// public static void main(String[] args) { try { if ((args != null) && (args.length == 1) && ("create".equalsIgnoreCase(args[0]))) { new IdentitySchema(IdentitySessionFactory.createConfiguration()).createSchema(); } else if ((args != null) && (args.length == 1) && ("drop".equalsIgnoreCase(args[0]))) { new IdentitySchema(IdentitySessionFactory.createConfiguration()).dropSchema(); } else if ((args != null) && (args.length == 1) && ("clean".equalsIgnoreCase(args[0]))) { new IdentitySchema(IdentitySessionFactory.createConfiguration()).cleanSchema(); } else if ((args != null) && (args.length == 3) && ("scripts".equalsIgnoreCase(args[0]))) { new IdentitySchema(IdentitySessionFactory.createConfiguration()) .saveSqlScripts(args[1], args[2]); } else { System.err.println("syntax: JbpmSchema create"); System.err.println("syntax: JbpmSchema drop"); System.err.println("syntax: JbpmSchema clean"); System.err.println("syntax: JbpmSchema scripts <dir> <prefix>"); } } catch (Exception e) { e.printStackTrace(); throw new RuntimeException(e); } } private void saveSqlScript(String fileName, String[] sql) throws FileNotFoundException { FileOutputStream fileOutputStream = new FileOutputStream(fileName); PrintStream printStream = new PrintStream(fileOutputStream); for (int i = 0; i < sql.length; i++) { printStream.println(sql[i] + getSqlDelimiter()); } } // sql script execution ///////////////////////////////////////////////////// public void execute(String[] sqls) { String sql = null; String showSqlText = properties.getProperty("hibernate.show_sql"); boolean showSql = ("true".equalsIgnoreCase(showSqlText)); try { createConnection(); statement = connection.createStatement(); for (int i = 0; i < sqls.length; i++) { sql = sqls[i]; String delimitedSql = sql + getSqlDelimiter(); if (showSql) log.debug(delimitedSql); statement.executeUpdate(delimitedSql); } } catch (SQLException e) { e.printStackTrace(); throw new RuntimeException("couldn't execute sql '" + sql + "'", e); } finally { closeConnection(); } } private void closeConnection() { try { if (statement != null) statement.close(); if (connection != null) { JDBCExceptionReporter.logWarnings(connection.getWarnings()); connection.clearWarnings(); connectionProvider.closeConnection(connection); connectionProvider.close(); } } catch (Exception e) { System.err.println("Could not close connection"); e.printStackTrace(); } } private void createConnection() throws SQLException { connectionProvider = ConnectionProviderFactory.newConnectionProvider(properties); connection = connectionProvider.getConnection(); if (!connection.getAutoCommit()) { connection.commit(); connection.setAutoCommit(true); } } public Properties getProperties() { return properties; } // sql delimiter //////////////////////////////////////////////////////////// private static String sqlDelimiter = null; private synchronized String getSqlDelimiter() { if (sqlDelimiter == null) { sqlDelimiter = properties.getProperty("jbpm.sql.delimiter", ";"); } return sqlDelimiter; } // logger /////////////////////////////////////////////////////////////////// private static final Log log = LogFactory.getLog(IdentitySchema.class); }
/** * MDC Connector * * @author yjiang */ public abstract class MDCConnector extends IoHandlerAdapter { static final Log log = LogFactory.getLog(MDCConnector.class); /** the the max size of a packet, 32KB */ static int MAX_SIZE = MDCServer.MAX_SIZE; protected Selector selector; protected IoConnector connector; protected static Configuration _conf; protected static boolean inited = false; /** Close. */ public void close() { if (selector != null) { selector.wakeup(); try { selector.close(); } catch (IOException e1) { log.warn("close selector fails", e1); } finally { selector = null; } } if (connector != null) { connector.dispose(); connector = null; } } /** Instantiates a new MDC connector. */ protected MDCConnector() {} /** Inits the. */ public static synchronized void init() { if (inited) { return; } _conf = Config.getConfig(); /** initialize app command */ Command.init(); /** initialize the RSA key, hardcode 2048 bits */ TConn.pub_key = SystemConfig.s("pub_key", null); if (TConn.pub_key == null) { Key k = RSA.generate(2048); TConn.pri_key = k.pri_key; TConn.pub_key = k.pub_key; /** set back in database */ SystemConfig.setConfig("pri_key", TConn.pri_key); SystemConfig.setConfig("pub_key", TConn.pub_key); } else { /** get from the database */ TConn.pri_key = SystemConfig.s("pri_key", null); } inited = true; } /** * Service. * * @param o the o * @param session the session */ void service(IoBuffer o, IoSession session) { try { // System.out.println(o.remaining() + "/" + o.capacity()); session.setAttribute("last", System.currentTimeMillis()); SimpleIoBuffer in = (SimpleIoBuffer) session.getAttribute("buf"); if (in == null) { in = SimpleIoBuffer.create(4096); session.setAttribute("buf", in); } byte[] data = new byte[o.remaining()]; o.get(data); in.append(data); // log.debug("recv: " + data.length + ", " + // session.getRemoteAddress()); while (in.length() > 5) { in.mark(); /** * Byte 1: head of the package<br> * bit 7-6: "01", indicator of MDC<br> * bit 5: encrypt indicator, "0": no; "1": encrypted<br> * bit 4: zip indicator, "0": no, "1": ziped<br> * bit 0-3: reserved<br> * Byte 2-5: length of data<br> * Byte[…]: data array<br> */ byte head = in.read(); /** test the head indicator, if not correct close it */ if ((head & 0xC0) != 0x40) { log.info("flag is not correct! flag:" + head + ",from: " + session.getRemoteAddress()); session.close(true); return; } int len = in.getInt(); if (len <= 0 || len > MAX_SIZE) { log.error( "mdcconnector.Wrong lendth: " + len + "/" + MAX_SIZE + " - " + session.getRemoteAddress()); session.close(true); break; } if (in.length() < len) { in.reset(); break; } else { // do it // log.info("stub.package.size: " + len); byte[] b = new byte[len]; in.read(b); if (TConn.DEBUG) { log.debug("recv: " + Bean.toString(b)); } /** test the zip flag */ if ((head & 0x10) > 0) { b = Zip.unzip(b); } final TConn d = (TConn) session.getAttribute("conn"); if (d != null) { /** test the encrypted flag */ if ((head & 0x20) > 0) { b = DES.decode(b, d.deskey); } final byte[] bb = b; /** test if the packet is for mdc or app */ new WorkerTask() { @Override public void onExecute() { d.process(bb); } }.schedule(0); session.setAttribute("last", System.currentTimeMillis()); } } } } catch (Throwable e) { log.error("closing stub: " + session.getRemoteAddress(), e); session.close(true); } } /* * (non-Javadoc) * * @see * org.apache.mina.core.service.IoHandlerAdapter#sessionCreated(org.apache * .mina.core.session.IoSession) */ public void sessionCreated(IoSession session) throws Exception { String remote = session.getRemoteAddress().toString(); log.info("stub created:" + remote); /** check the allow ip */ if (TConn.ALLOW_IP == null || "*".equals(TConn.ALLOW_IP) || remote.matches(TConn.ALLOW_IP)) { TConn d = new TConn(session); session.setAttribute("conn", d); } else { log.warn("deny the connection:" + remote + ", allow ip:" + TConn.ALLOW_IP); session.close(true); } } /* * (non-Javadoc) * * @see * org.apache.mina.core.service.IoHandlerAdapter#sessionClosed(org.apache * .mina.core.session.IoSession) */ public void sessionClosed(IoSession session) throws Exception { log.debug("closed stub: " + session.getRemoteAddress()); TConn d = (TConn) session.getAttribute("conn"); if (d != null) { d.close(); } } /* * (non-Javadoc) * * @see * org.apache.mina.core.service.IoHandlerAdapter#sessionIdle(org.apache. * mina.core.session.IoSession, org.apache.mina.core.session.IdleStatus) */ public void sessionIdle(IoSession session, IdleStatus status) throws Exception { if (IdleStatus.BOTH_IDLE.equals(status)) { Long l = (Long) session.getAttribute("last"); if (l != null && System.currentTimeMillis() - l > 60 * 1000) { session.close(true); } } } /* * (non-Javadoc) * * @see * org.apache.mina.core.service.IoHandlerAdapter#messageReceived(org.apache * .mina.core.session.IoSession, java.lang.Object) */ public void messageReceived(IoSession session, Object message) throws Exception { // System.out.println(message); if (message instanceof IoBuffer) { service((IoBuffer) message, session); } } private static MDCConnector tcpconnector; private static MDCConnector udpconnector; /** * @param host * @param port * @return TConn */ public static synchronized TConn connectByTcp(String host, int port) { return connectByTcp(host, port, X.AMINUTE); } /** * Connect by tcp. * * @param host the host * @param port the port * @return the t conn */ public static synchronized TConn connectByTcp(String host, int port, long timeout) { TimeStamp t = TimeStamp.create(); try { if (tcpconnector == null) { tcpconnector = new TDCConnector(); } tcpconnector.connector.setConnectTimeoutMillis(timeout); ConnectFuture connFuture = tcpconnector.connector.connect(new InetSocketAddress(host, port)); connFuture.awaitUninterruptibly(timeout); IoSession session = connFuture.getSession(); TConn c = new TConn(session); session.setAttribute("conn", c); return c; } catch (Exception e) { log.error( "error, [" + host + ":" + port + "], cost: " + t.past() + "ms, timeout=" + timeout, e); } return null; } /** * Connect by udp. * * @param host the host * @param port the port * @return the t conn */ public static synchronized TConn connectByUdp(String host, int port) { try { if (udpconnector == null) { udpconnector = new UDCConnector(); } ConnectFuture connFuture = udpconnector.connector.connect(new InetSocketAddress(host, port)); connFuture.awaitUninterruptibly(); IoSession session = connFuture.getSession(); TConn c = new TConn(session); session.setAttribute("conn", c); return c; } catch (Exception e) { log.error("[" + host + ":" + port + "]", e); } return null; } /* * (non-Javadoc) * * @see * org.apache.mina.core.service.IoHandlerAdapter#exceptionCaught(org.apache * .mina.core.session.IoSession, java.lang.Throwable) */ @Override public void exceptionCaught(IoSession session, Throwable cause) throws Exception { log.error(cause.getMessage(), cause); } }
/** @author Johan Lindquist */ public class BaseController { private Log _log = LogFactory.getLog(BaseController.class); @Autowired protected JahSpotifyService _jahSpotifyService; @Value(value = "${jahspotify.web.controller.default-media-expires-duration}") private int _defaultMediaExpirationTime; protected void writeResponse( final HttpServletResponse httpServletResponse, final SimpleStatusResponse simpleStatusResponse) { Gson gson = new Gson(); try { httpServletResponse.setContentType("application/json; charset=utf-8"); _log.debug("Serializing: " + simpleStatusResponse); final PrintWriter writer = httpServletResponse.getWriter(); gson.toJson(simpleStatusResponse.getResponseStatus(), writer); writer.flush(); writer.close(); } catch (Exception e) { _log.error("Error while writing response: " + e.getMessage(), e); } } protected void writeMediaNotReadable(final HttpServletResponse httpServletResponse) { SimpleStatusResponse simpleStatusResponse = new SimpleStatusResponse(); simpleStatusResponse.setResponseStatus(ResponseStatus.RESOURCE_NOT_FOUND); writeResponse(httpServletResponse, simpleStatusResponse); } protected void writeResponseGeneric( final HttpServletResponse httpServletResponse, final Object object) { this.writeResponseGenericWithDate(httpServletResponse, null, object); } protected void writeResponseGenericWithDate( final HttpServletResponse httpServletResponse, final Date lastModified, final Object object) { writeResponseGenericWithDate( httpServletResponse, lastModified, _defaultMediaExpirationTime, object); } protected void writeResponseGenericWithDate( final HttpServletResponse httpServletResponse, final Date lastModified, final int expirationTime, final Object object) { Gson gson = new Gson(); try { httpServletResponse.setContentType("application/json; charset=utf-8"); if (lastModified != null) { httpServletResponse.addHeader("Expires", createDateHeader(expirationTime)); httpServletResponse.addHeader("Last-Modified", toHttpDate(lastModified)); } _log.debug("Serializing: " + object); final PrintWriter writer = httpServletResponse.getWriter(); gson.toJson(object, writer); writer.flush(); writer.close(); } catch (Exception e) { _log.error("Error while writing response: " + e.getMessage(), e); } } protected String createDateHeader(int expires) { final Calendar utc = GregorianCalendar.getInstance(TimeZone.getTimeZone("UTC")); utc.add(Calendar.SECOND, expires); return toHttpDate(utc.getTime()); } protected String toHttpDate(Date date) { SimpleDateFormat simpleDateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z"); return simpleDateFormat.format(date); } protected Link retrieveLink(final HttpServletRequest httpServletRequest) { String uri = httpServletRequest .getRequestURI() .substring(httpServletRequest.getRequestURI().lastIndexOf("/") + 1); _log.debug("Extracted URI: " + uri); return Link.create(uri); } protected <T> T readRequest(final HttpServletRequest httpServletRequest, final Class<T> classOfT) throws IOException { final BufferedReader br = new BufferedReader(httpServletRequest.getReader()); Gson gson = new Gson(); return gson.fromJson(br, classOfT); } public void writeErrorResponse(final HttpServletResponse httpServletResponse, final Exception e) { SimpleStatusResponse simpleStatusResponse = new SimpleStatusResponse(); simpleStatusResponse.setResponseStatus(ResponseStatus.INTERNAL_ERROR); writeResponse(httpServletResponse, simpleStatusResponse); } }
/** * A simple RPC mechanism. * * <p>A <i>protocol</i> is a Java interface. All parameters and return types must be one of: * * <ul> * <li>a primitive type, <code>boolean</code>, <code>byte</code>, <code>char</code>, <code>short * </code>, <code>int</code>, <code>long</code>, <code>float</code>, <code>double</code>, or * <code>void</code>; or * <li>a {@link String}; or * <li>a {@link Writable}; or * <li>an array of the above types * </ul> * * All methods in the protocol should throw only IOException. No field data of the protocol instance * is transmitted. */ @InterfaceAudience.LimitedPrivate(value = {"Common", "HDFS", "MapReduce", "Yarn"}) @InterfaceStability.Evolving public class RPC { static final int RPC_SERVICE_CLASS_DEFAULT = 0; public enum RpcKind { RPC_BUILTIN((short) 1), // Used for built in calls by tests RPC_WRITABLE((short) 2), // Use WritableRpcEngine RPC_PROTOCOL_BUFFER((short) 3); // Use ProtobufRpcEngine static final short MAX_INDEX = RPC_PROTOCOL_BUFFER.value; // used for array size public final short value; // TODO make it private RpcKind(short val) { this.value = val; } } interface RpcInvoker { /** * Process a client call on the server side * * @param server the server within whose context this rpc call is made * @param protocol - the protocol name (the class of the client proxy used to make calls to the * rpc server. * @param rpcRequest - deserialized * @param receiveTime time at which the call received (for metrics) * @return the call's return * @throws IOException */ public Writable call(Server server, String protocol, Writable rpcRequest, long receiveTime) throws Exception; } static final Log LOG = LogFactory.getLog(RPC.class); /** * Get all superInterfaces that extend VersionedProtocol * * @param childInterfaces * @return the super interfaces that extend VersionedProtocol */ static Class<?>[] getSuperInterfaces(Class<?>[] childInterfaces) { List<Class<?>> allInterfaces = new ArrayList<Class<?>>(); for (Class<?> childInterface : childInterfaces) { if (VersionedProtocol.class.isAssignableFrom(childInterface)) { allInterfaces.add(childInterface); allInterfaces.addAll(Arrays.asList(getSuperInterfaces(childInterface.getInterfaces()))); } else { LOG.warn( "Interface " + childInterface + " ignored because it does not extend VersionedProtocol"); } } return allInterfaces.toArray(new Class[allInterfaces.size()]); } /** * Get all interfaces that the given protocol implements or extends which are assignable from * VersionedProtocol. */ static Class<?>[] getProtocolInterfaces(Class<?> protocol) { Class<?>[] interfaces = protocol.getInterfaces(); return getSuperInterfaces(interfaces); } /** * Get the protocol name. If the protocol class has a ProtocolAnnotation, then get the protocol * name from the annotation; otherwise the class name is the protocol name. */ public static String getProtocolName(Class<?> protocol) { if (protocol == null) { return null; } ProtocolInfo anno = protocol.getAnnotation(ProtocolInfo.class); return (anno == null) ? protocol.getName() : anno.protocolName(); } /** * Get the protocol version from protocol class. If the protocol class has a ProtocolAnnotation, * then get the protocol name from the annotation; otherwise the class name is the protocol name. */ public static long getProtocolVersion(Class<?> protocol) { if (protocol == null) { throw new IllegalArgumentException("Null protocol"); } long version; ProtocolInfo anno = protocol.getAnnotation(ProtocolInfo.class); if (anno != null) { version = anno.protocolVersion(); if (version != -1) return version; } try { Field versionField = protocol.getField("versionID"); versionField.setAccessible(true); return versionField.getLong(protocol); } catch (NoSuchFieldException ex) { throw new RuntimeException(ex); } catch (IllegalAccessException ex) { throw new RuntimeException(ex); } } private RPC() {} // no public ctor // cache of RpcEngines by protocol private static final Map<Class<?>, RpcEngine> PROTOCOL_ENGINES = new HashMap<Class<?>, RpcEngine>(); private static final String ENGINE_PROP = "rpc.engine"; /** * Set a protocol to use a non-default RpcEngine. * * @param conf configuration to use * @param protocol the protocol interface * @param engine the RpcEngine impl */ public static void setProtocolEngine(Configuration conf, Class<?> protocol, Class<?> engine) { conf.setClass(ENGINE_PROP + "." + protocol.getName(), engine, RpcEngine.class); } // return the RpcEngine configured to handle a protocol static synchronized RpcEngine getProtocolEngine(Class<?> protocol, Configuration conf) { RpcEngine engine = PROTOCOL_ENGINES.get(protocol); if (engine == null) { Class<?> impl = conf.getClass(ENGINE_PROP + "." + protocol.getName(), WritableRpcEngine.class); engine = (RpcEngine) ReflectionUtils.newInstance(impl, conf); PROTOCOL_ENGINES.put(protocol, engine); } return engine; } /** A version mismatch for the RPC protocol. */ public static class VersionMismatch extends RpcServerException { private static final long serialVersionUID = 0; private String interfaceName; private long clientVersion; private long serverVersion; /** * Create a version mismatch exception * * @param interfaceName the name of the protocol mismatch * @param clientVersion the client's version of the protocol * @param serverVersion the server's version of the protocol */ public VersionMismatch(String interfaceName, long clientVersion, long serverVersion) { super( "Protocol " + interfaceName + " version mismatch. (client = " + clientVersion + ", server = " + serverVersion + ")"); this.interfaceName = interfaceName; this.clientVersion = clientVersion; this.serverVersion = serverVersion; } /** * Get the interface name * * @return the java class name (eg. org.apache.hadoop.mapred.InterTrackerProtocol) */ public String getInterfaceName() { return interfaceName; } /** Get the client's preferred version */ public long getClientVersion() { return clientVersion; } /** Get the server's agreed to version. */ public long getServerVersion() { return serverVersion; } /** get the rpc status corresponding to this exception */ public RpcStatusProto getRpcStatusProto() { return RpcStatusProto.ERROR; } /** get the detailed rpc status corresponding to this exception */ public RpcErrorCodeProto getRpcErrorCodeProto() { return RpcErrorCodeProto.ERROR_RPC_VERSION_MISMATCH; } } /** * Get a proxy connection to a remote server * * @param protocol protocol class * @param clientVersion client version * @param addr remote address * @param conf configuration to use * @return the proxy * @throws IOException if the far end through a RemoteException */ public static <T> T waitForProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { return waitForProtocolProxy(protocol, clientVersion, addr, conf).getProxy(); } /** * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods * that are supported by the server * * @param protocol protocol class * @param clientVersion client version * @param addr remote address * @param conf configuration to use * @return the protocol proxy * @throws IOException if the far end through a RemoteException */ public static <T> ProtocolProxy<T> waitForProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { return waitForProtocolProxy(protocol, clientVersion, addr, conf, Long.MAX_VALUE); } /** * Get a proxy connection to a remote server * * @param protocol protocol class * @param clientVersion client version * @param addr remote address * @param conf configuration to use * @param connTimeout time in milliseconds before giving up * @return the proxy * @throws IOException if the far end through a RemoteException */ public static <T> T waitForProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, long connTimeout) throws IOException { return waitForProtocolProxy(protocol, clientVersion, addr, conf, connTimeout).getProxy(); } /** * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods * that are supported by the server * * @param protocol protocol class * @param clientVersion client version * @param addr remote address * @param conf configuration to use * @param connTimeout time in milliseconds before giving up * @return the protocol proxy * @throws IOException if the far end through a RemoteException */ public static <T> ProtocolProxy<T> waitForProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, long connTimeout) throws IOException { return waitForProtocolProxy(protocol, clientVersion, addr, conf, 0, null, connTimeout); } /** * Get a proxy connection to a remote server * * @param protocol protocol class * @param clientVersion client version * @param addr remote address * @param conf configuration to use * @param rpcTimeout timeout for each RPC * @param timeout time in milliseconds before giving up * @return the proxy * @throws IOException if the far end through a RemoteException */ public static <T> T waitForProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout, long timeout) throws IOException { return waitForProtocolProxy(protocol, clientVersion, addr, conf, rpcTimeout, null, timeout) .getProxy(); } /** * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods * that are supported by the server * * @param protocol protocol class * @param clientVersion client version * @param addr remote address * @param conf configuration to use * @param rpcTimeout timeout for each RPC * @param timeout time in milliseconds before giving up * @return the proxy * @throws IOException if the far end through a RemoteException */ public static <T> ProtocolProxy<T> waitForProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout, RetryPolicy connectionRetryPolicy, long timeout) throws IOException { long startTime = Time.now(); IOException ioe; while (true) { try { return getProtocolProxy( protocol, clientVersion, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout, connectionRetryPolicy); } catch (ConnectException se) { // namenode has not been started LOG.info("Server at " + addr + " not available yet, Zzzzz..."); ioe = se; } catch (SocketTimeoutException te) { // namenode is busy LOG.info("Problem connecting to server: " + addr); ioe = te; } catch (NoRouteToHostException nrthe) { // perhaps a VIP is failing over LOG.info("No route to host for server: " + addr); ioe = nrthe; } // check if timed out if (Time.now() - timeout >= startTime) { throw ioe; } // wait for retry try { Thread.sleep(1000); } catch (InterruptedException ie) { // IGNORE } } } /** * Construct a client-side proxy object that implements the named protocol, talking to a server at * the named address. * * @param <T> */ public static <T> T getProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, SocketFactory factory) throws IOException { return getProtocolProxy(protocol, clientVersion, addr, conf, factory).getProxy(); } /** * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods * that are supported by the server * * @param protocol protocol class * @param clientVersion client version * @param addr remote address * @param conf configuration to use * @param factory socket factory * @return the protocol proxy * @throws IOException if the far end through a RemoteException */ public static <T> ProtocolProxy<T> getProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf, SocketFactory factory) throws IOException { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); return getProtocolProxy(protocol, clientVersion, addr, ugi, conf, factory); } /** * Construct a client-side proxy object that implements the named protocol, talking to a server at * the named address. * * @param <T> */ public static <T> T getProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory) throws IOException { return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory).getProxy(); } /** * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods * that are supported by the server * * @param protocol protocol class * @param clientVersion client version * @param addr remote address * @param ticket user group information * @param conf configuration to use * @param factory socket factory * @return the protocol proxy * @throws IOException if the far end through a RemoteException */ public static <T> ProtocolProxy<T> getProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory) throws IOException { return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory, 0, null); } /** * Construct a client-side proxy that implements the named protocol, talking to a server at the * named address. * * @param <T> * @param protocol protocol * @param clientVersion client's version * @param addr server address * @param ticket security ticket * @param conf configuration * @param factory socket factory * @param rpcTimeout max time for each rpc; 0 means no timeout * @return the proxy * @throws IOException if any error occurs */ public static <T> T getProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory, int rpcTimeout) throws IOException { return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout, null) .getProxy(); } /** * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods * that are supported by the server * * @param protocol protocol * @param clientVersion client's version * @param addr server address * @param ticket security ticket * @param conf configuration * @param factory socket factory * @param rpcTimeout max time for each rpc; 0 means no timeout * @return the proxy * @throws IOException if any error occurs */ public static <T> ProtocolProxy<T> getProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy) throws IOException { if (UserGroupInformation.isSecurityEnabled()) { SaslRpcServer.init(conf); } return getProtocolEngine(protocol, conf) .getProxy( protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout, connectionRetryPolicy); } /** * Construct a client-side proxy object with the default SocketFactory * * @param <T> * @param protocol * @param clientVersion * @param addr * @param conf * @return a proxy instance * @throws IOException */ public static <T> T getProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { return getProtocolProxy(protocol, clientVersion, addr, conf).getProxy(); } /** Returns the server address for a given proxy. */ public static InetSocketAddress getServerAddress(Object proxy) { return getConnectionIdForProxy(proxy).getAddress(); } /** * Return the connection ID of the given object. If the provided object is in fact a protocol * translator, we'll get the connection ID of the underlying proxy object. * * @param proxy the proxy object to get the connection ID of. * @return the connection ID for the provided proxy object. */ public static ConnectionId getConnectionIdForProxy(Object proxy) { if (proxy instanceof ProtocolTranslator) { proxy = ((ProtocolTranslator) proxy).getUnderlyingProxyObject(); } RpcInvocationHandler inv = (RpcInvocationHandler) Proxy.getInvocationHandler(proxy); return inv.getConnectionId(); } /** * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods * that are supported by the server * * @param protocol * @param clientVersion * @param addr * @param conf * @return a protocol proxy * @throws IOException */ public static <T> ProtocolProxy<T> getProtocolProxy( Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { return getProtocolProxy( protocol, clientVersion, addr, conf, NetUtils.getDefaultSocketFactory(conf)); } /** * Stop the proxy. Proxy must either implement {@link Closeable} or must have associated {@link * RpcInvocationHandler}. * * @param proxy the RPC proxy object to be stopped * @throws HadoopIllegalArgumentException if the proxy does not implement {@link Closeable} * interface or does not have closeable {@link InvocationHandler} */ public static void stopProxy(Object proxy) { if (proxy == null) { throw new HadoopIllegalArgumentException("Cannot close proxy since it is null"); } try { if (proxy instanceof Closeable) { ((Closeable) proxy).close(); return; } else { InvocationHandler handler = Proxy.getInvocationHandler(proxy); if (handler instanceof Closeable) { ((Closeable) handler).close(); return; } } } catch (IOException e) { LOG.error("Closing proxy or invocation handler caused exception", e); } catch (IllegalArgumentException e) { LOG.error("RPC.stopProxy called on non proxy: class=" + proxy.getClass().getName(), e); } // If you see this error on a mock object in a unit test you're // developing, make sure to use MockitoUtil.mockProtocol() to // create your mock. throw new HadoopIllegalArgumentException( "Cannot close proxy - is not Closeable or " + "does not provide closeable invocation handler " + proxy.getClass()); } /** Class to construct instances of RPC server with specific options. */ public static class Builder { private Class<?> protocol = null; private Object instance = null; private String bindAddress = "0.0.0.0"; private int port = 0; private int numHandlers = 1; private int numReaders = -1; private int queueSizePerHandler = -1; private boolean verbose = false; private final Configuration conf; private SecretManager<? extends TokenIdentifier> secretManager = null; private String portRangeConfig = null; public Builder(Configuration conf) { this.conf = conf; } /** Mandatory field */ public Builder setProtocol(Class<?> protocol) { this.protocol = protocol; return this; } /** Mandatory field */ public Builder setInstance(Object instance) { this.instance = instance; return this; } /** Default: 0.0.0.0 */ public Builder setBindAddress(String bindAddress) { this.bindAddress = bindAddress; return this; } /** Default: 0 */ public Builder setPort(int port) { this.port = port; return this; } /** Default: 1 */ public Builder setNumHandlers(int numHandlers) { this.numHandlers = numHandlers; return this; } /** Default: -1 */ public Builder setnumReaders(int numReaders) { this.numReaders = numReaders; return this; } /** Default: -1 */ public Builder setQueueSizePerHandler(int queueSizePerHandler) { this.queueSizePerHandler = queueSizePerHandler; return this; } /** Default: false */ public Builder setVerbose(boolean verbose) { this.verbose = verbose; return this; } /** Default: null */ public Builder setSecretManager(SecretManager<? extends TokenIdentifier> secretManager) { this.secretManager = secretManager; return this; } /** Default: null */ public Builder setPortRangeConfig(String portRangeConfig) { this.portRangeConfig = portRangeConfig; return this; } /** * Build the RPC Server. * * @throws IOException on error * @throws HadoopIllegalArgumentException when mandatory fields are not set */ public Server build() throws IOException, HadoopIllegalArgumentException { if (this.conf == null) { throw new HadoopIllegalArgumentException("conf is not set"); } if (this.protocol == null) { throw new HadoopIllegalArgumentException("protocol is not set"); } if (this.instance == null) { throw new HadoopIllegalArgumentException("instance is not set"); } return getProtocolEngine(this.protocol, this.conf) .getServer( this.protocol, this.instance, this.bindAddress, this.port, this.numHandlers, this.numReaders, this.queueSizePerHandler, this.verbose, this.conf, this.secretManager, this.portRangeConfig); } } /** An RPC Server. */ public abstract static class Server extends org.apache.hadoop.ipc.Server { boolean verbose; static String classNameBase(String className) { String[] names = className.split("\\.", -1); if (names == null || names.length == 0) { return className; } return names[names.length - 1]; } /** Store a map of protocol and version to its implementation */ /** The key in Map */ static class ProtoNameVer { final String protocol; final long version; ProtoNameVer(String protocol, long ver) { this.protocol = protocol; this.version = ver; } @Override public boolean equals(Object o) { if (o == null) return false; if (this == o) return true; if (!(o instanceof ProtoNameVer)) return false; ProtoNameVer pv = (ProtoNameVer) o; return ((pv.protocol.equals(this.protocol)) && (pv.version == this.version)); } @Override public int hashCode() { return protocol.hashCode() * 37 + (int) version; } } /** The value in map */ static class ProtoClassProtoImpl { final Class<?> protocolClass; final Object protocolImpl; ProtoClassProtoImpl(Class<?> protocolClass, Object protocolImpl) { this.protocolClass = protocolClass; this.protocolImpl = protocolImpl; } } ArrayList<Map<ProtoNameVer, ProtoClassProtoImpl>> protocolImplMapArray = new ArrayList<Map<ProtoNameVer, ProtoClassProtoImpl>>(RpcKind.MAX_INDEX); Map<ProtoNameVer, ProtoClassProtoImpl> getProtocolImplMap(RPC.RpcKind rpcKind) { if (protocolImplMapArray.size() == 0) { // initialize for all rpc kinds for (int i = 0; i <= RpcKind.MAX_INDEX; ++i) { protocolImplMapArray.add(new HashMap<ProtoNameVer, ProtoClassProtoImpl>(10)); } } return protocolImplMapArray.get(rpcKind.ordinal()); } // Register protocol and its impl for rpc calls void registerProtocolAndImpl(RpcKind rpcKind, Class<?> protocolClass, Object protocolImpl) { String protocolName = RPC.getProtocolName(protocolClass); long version; try { version = RPC.getProtocolVersion(protocolClass); } catch (Exception ex) { LOG.warn("Protocol " + protocolClass + " NOT registered as cannot get protocol version "); return; } getProtocolImplMap(rpcKind) .put( new ProtoNameVer(protocolName, version), new ProtoClassProtoImpl(protocolClass, protocolImpl)); LOG.debug( "RpcKind = " + rpcKind + " Protocol Name = " + protocolName + " version=" + version + " ProtocolImpl=" + protocolImpl.getClass().getName() + " protocolClass=" + protocolClass.getName()); } static class VerProtocolImpl { final long version; final ProtoClassProtoImpl protocolTarget; VerProtocolImpl(long ver, ProtoClassProtoImpl protocolTarget) { this.version = ver; this.protocolTarget = protocolTarget; } } VerProtocolImpl[] getSupportedProtocolVersions(RPC.RpcKind rpcKind, String protocolName) { VerProtocolImpl[] resultk = new VerProtocolImpl[getProtocolImplMap(rpcKind).size()]; int i = 0; for (Map.Entry<ProtoNameVer, ProtoClassProtoImpl> pv : getProtocolImplMap(rpcKind).entrySet()) { if (pv.getKey().protocol.equals(protocolName)) { resultk[i++] = new VerProtocolImpl(pv.getKey().version, pv.getValue()); } } if (i == 0) { return null; } VerProtocolImpl[] result = new VerProtocolImpl[i]; System.arraycopy(resultk, 0, result, 0, i); return result; } VerProtocolImpl getHighestSupportedProtocol(RpcKind rpcKind, String protocolName) { Long highestVersion = 0L; ProtoClassProtoImpl highest = null; if (LOG.isDebugEnabled()) { LOG.debug("Size of protoMap for " + rpcKind + " =" + getProtocolImplMap(rpcKind).size()); } for (Map.Entry<ProtoNameVer, ProtoClassProtoImpl> pv : getProtocolImplMap(rpcKind).entrySet()) { if (pv.getKey().protocol.equals(protocolName)) { if ((highest == null) || (pv.getKey().version > highestVersion)) { highest = pv.getValue(); highestVersion = pv.getKey().version; } } } if (highest == null) { return null; } return new VerProtocolImpl(highestVersion, highest); } protected Server( String bindAddress, int port, Class<? extends Writable> paramClass, int handlerCount, int numReaders, int queueSizePerHandler, Configuration conf, String serverName, SecretManager<? extends TokenIdentifier> secretManager, String portRangeConfig) throws IOException { super( bindAddress, port, paramClass, handlerCount, numReaders, queueSizePerHandler, conf, serverName, secretManager, portRangeConfig); initProtocolMetaInfo(conf); } private void initProtocolMetaInfo(Configuration conf) { RPC.setProtocolEngine(conf, ProtocolMetaInfoPB.class, ProtobufRpcEngine.class); ProtocolMetaInfoServerSideTranslatorPB xlator = new ProtocolMetaInfoServerSideTranslatorPB(this); BlockingService protocolInfoBlockingService = ProtocolInfoService.newReflectiveBlockingService(xlator); addProtocol( RpcKind.RPC_PROTOCOL_BUFFER, ProtocolMetaInfoPB.class, protocolInfoBlockingService); } /** * Add a protocol to the existing server. * * @param protocolClass - the protocol class * @param protocolImpl - the impl of the protocol that will be called * @return the server (for convenience) */ public Server addProtocol(RpcKind rpcKind, Class<?> protocolClass, Object protocolImpl) { registerProtocolAndImpl(rpcKind, protocolClass, protocolImpl); return this; } @Override public Writable call( RPC.RpcKind rpcKind, String protocol, Writable rpcRequest, long receiveTime) throws Exception { return getRpcInvoker(rpcKind).call(this, protocol, rpcRequest, receiveTime); } } }
public class MilterRequestHandler implements RequestHandler, JilterHandler, StopBlockTarget { protected static Log logger = LogFactory.getLog(MilterRequestHandler.class.getName()); protected SocketChannel socket = null; protected ArrayList<String> rcpts = null; protected FetchMessageCallback callback; protected String host = ""; protected JilterStatus status = null; protected ByteArrayOutputStream bos = new ByteArrayOutputStream(); protected static Pattern headerPattern1 = Pattern.compile("^cc|^to|^bcc"); protected static Pattern headerPattern2 = Pattern.compile(".*<([-.+_\\d\\w]*@[-.+_\\d\\w]*)>"); protected static Pattern headerPattern3 = Pattern.compile("([-.+_\\d\\w]*@[-.+_\\d\\w]*)"); private static final int IDLE_TIMEOUT = 300000; // 5 minutes protected boolean includeBCC = false; public void handleRequest(SocketChannel socket, FetchMessageCallback callback) { this.socket = socket; this.callback = callback; includeBCC = false; rcpts = new ArrayList<String>(); bos = new ByteArrayOutputStream(); InetAddress address = socket.socket().getInetAddress(); boolean isAllowed = Config.getConfig().getAgent().isAllowed(address); if (!isAllowed) { logger.debug( "attempted milter connection from disallowed address. force disconnect {address='" + address.getHostAddress() + "'}"); try { socket.close(); } catch (IOException io) { logger.error("failed to close milter socket.", io); } return; } ByteBuffer dataBuffer = ByteBuffer.allocateDirect(4096); JilterProcessor processor = new JilterProcessor(this); try { while (processor.process(socket, (ByteBuffer) dataBuffer.flip())) { dataBuffer.compact(); if (this.socket.read(dataBuffer) == -1) { logger.debug("socket reports EOF, exiting read loop"); break; } } } catch (IOException e) { logger.debug("Unexpected exception, connection will be closed", e); } finally { logger.debug("closing processor"); processor.close(); logger.debug("processor closed"); try { logger.debug("closing socket"); this.socket.close(); logger.debug("socket closed"); } catch (IOException e) { logger.debug("Unexpected exception", e); } } } public JilterStatus abort() { logger.debug("abort"); return JilterStatus.SMFIS_CONTINUE; } public JilterStatus body(ByteBuffer bodyp) { logger.debug("jilter body()"); long maxMessageSizeMB = Config.getConfig().getArchiver().getMaxMessageSize(); long maxMessageSizeBytes = maxMessageSizeMB * 1024 * 1024; if (bodyp.array().length > maxMessageSizeBytes) { logger.warn( "milter maximum message size exceeded { size='" + bodyp.array().length + " bytes'}"); return JilterStatus.SMFIS_REJECT; } try { bos.write("\n".getBytes()); bos.write(bodyp.array()); } catch (IOException io) { logger.error("jilter failed to write milter body data to byte buffer", io); } logger.debug("jilter body written"); return JilterStatus.SMFIS_CONTINUE; } public JilterStatus close() { logger.debug("jilter close()"); return JilterStatus.SMFIS_CONTINUE; } public JilterStatus connect(String hostname, InetAddress hostaddr, Properties properties) { rcpts = new ArrayList<String>(); if (hostaddr != null) { host = hostaddr.toString(); } else if (host != null) { host = hostname; } else { host = "localhost"; } logger.debug("jilter connect() {from='" + hostname + "',host='" + host + "'}"); return JilterStatus.SMFIS_CONTINUE; } public JilterStatus envfrom(String[] argv, Properties properties) { for (int i = 0; i < argv.length; i++) { logger.debug("jilter envfrom() {from='" + argv[i] + "'}"); } return JilterStatus.SMFIS_CONTINUE; } public JilterStatus envrcpt(String[] argv, Properties properties) { for (int i = 0; i < argv.length; i++) { String strRecipient = argv[i]; boolean orcptFlag = strRecipient.toLowerCase(Locale.ENGLISH).trim().contains("orcpt="); if (!orcptFlag) { logger.debug("jilter envrcpt() {to='" + strRecipient + "'}"); String recipient = strRecipient.toLowerCase(Locale.ENGLISH).trim().replaceAll("<", "").replaceAll(">", ""); rcpts.add(recipient); logger.debug("jilter add recipient {recipient='" + recipient + "'}"); } } return JilterStatus.SMFIS_CONTINUE; } protected boolean shouldIgnoreBCCAddress(String address) { MilterServerService milterService = Config.getConfig().getMilterServerService(); List<String> ignoreAddresses = milterService.getIgnoreBCCAddress(); Matcher m = headerPattern2.matcher(address.toLowerCase(Locale.ENGLISH).trim()); if (m.matches()) { String mailAddress = m.group(1); for (String ignoreAddress : ignoreAddresses) { if (ignoreAddress.equalsIgnoreCase(mailAddress)) return true; } } else { m = headerPattern3.matcher(address.toLowerCase(Locale.ENGLISH).trim()); if (m.matches()) { String mailAddress = m.group(1); for (String ignoreAddress : ignoreAddresses) { if (ignoreAddress.equalsIgnoreCase(mailAddress)) return true; } } } return false; } public JilterStatus eoh() { logger.debug("jilter eoh()"); // includeBCC is false if RCPT TO does not contain at least one field in TO, FROM and CC // this is a safety check as sometimes, RCPT TO is something differently entirely // and does not contain the actual recipients in the email MilterServerService milterService = Config.getConfig().getMilterServerService(); if (milterService.getIncludeBCC() && includeBCC) { logger.debug("including BCC addresses"); // check to see if address is flagged to ignore if (rcpts.size() > 0) { Iterator<String> i = rcpts.iterator(); while (i.hasNext()) { String rcpt = i.next(); if (shouldIgnoreBCCAddress(rcpt)) { logger.debug("ignore include bcc address {address='" + rcpt + "'}"); i.remove(); } } } if (rcpts.size() > 0) { try { for (int j = 0; j < rcpts.size(); j++) { if (j == 0) { bos.write("bcc: ".getBytes()); } else { bos.write(",".getBytes()); } bos.write(rcpts.get(j).getBytes()); } bos.write("\n".getBytes()); } catch (IOException io) { logger.error("jilter failed to write end of header data", io); } } } return JilterStatus.SMFIS_CONTINUE; } public JilterStatus eom(JilterEOMActions eomActions, Properties properties) { logger.debug("jilter eom()"); try { bos.close(); // close stream } catch (IOException io) { logger.error("jilter failed to close io stream during eom", io); } byte[] messageBytes = bos.toByteArray(); bos = new ByteArrayOutputStream(); ByteArrayInputStream bis = new ByteArrayInputStream(messageBytes); try { logger.debug("jilter store callback execute"); Config.getStopBlockFactory() .detectBlock("milter server", Thread.currentThread(), this, IDLE_TIMEOUT); callback.store(bis, host); logger.debug("jilter store callback finished"); } catch (ArchiveException e) { logger.error("failed to store the message via milter", e); if (e.getRecoveryDirective() == ArchiveException.RecoveryDirective.REJECT) { logger.debug("jilter reject"); return JilterStatus.SMFIS_REJECT; } else if (e.getRecoveryDirective() == ArchiveException.RecoveryDirective.RETRYLATER) { logger.debug("jilter temp fail"); return JilterStatus.SMFIS_TEMPFAIL; } } catch (Throwable oome) { logger.error("failed to store message:" + oome.getMessage(), oome); return JilterStatus.SMFIS_REJECT; } finally { Config.getStopBlockFactory().endDetectBlock(Thread.currentThread()); } return JilterStatus.SMFIS_CONTINUE; } public int getRequiredModifications() { logger.debug("jilter requiredmodifications()"); return SMFIF_NONE; } public int getSupportedProcesses() { logger.debug("jilter getsupportedprocesses()"); return PROCESS_CONNECT | PROCESS_ENVRCPT | PROCESS_HEADER | PROCESS_BODY; } public JilterStatus header(String headerf, String headerv) { logger.debug("jilter header {name='" + headerf + "',value='" + headerv + "'}"); StringBuffer header = new StringBuffer(); header.append(headerf); header.append(": "); header.append(headerv); header.append("\n"); try { bos.write(header.toString().getBytes()); } catch (IOException io) { logger.error("jilter failed to write header field", io); } Matcher m = headerPattern1.matcher(headerf.toLowerCase(Locale.ENGLISH).trim()); if (m.matches()) { logger.debug("jilter found to/bcc/cc header"); String[] addresses = headerv.split(","); for (int i = 0; i < addresses.length; i++) { includeBCC = includeBCC | rcpts.remove(addresses[i].toLowerCase(Locale.ENGLISH).trim()); logger.debug("jilter del recipient {recipient='" + addresses[i] + "'}"); m = headerPattern2.matcher(addresses[i].toLowerCase(Locale.ENGLISH).trim()); if (m.matches()) { String mailAddress = m.group(1); includeBCC = includeBCC | rcpts.remove(mailAddress); logger.debug("jilter del recipient {recipient='" + mailAddress + "'}"); } else { m = headerPattern3.matcher(addresses[i].toLowerCase(Locale.ENGLISH).trim()); if (m.matches()) { String mailAddress = m.group(1); includeBCC = includeBCC | rcpts.remove(mailAddress); logger.debug("jilter del recipient {recipient='" + mailAddress + "'}"); } } } } return JilterStatus.SMFIS_CONTINUE; } public JilterStatus helo(String helohost, Properties properties) { logger.debug("jilter helo() " + helohost); return JilterStatus.SMFIS_CONTINUE; } public void handleBlock(Thread thread) { try { if (socket != null) { logger.debug("close socket()"); socket.close(); } } catch (Exception e) { // ignored } synchronized (this) { if (thread != null) { logger.debug("interrupt thread()"); thread.interrupt(); } } } }