Exemple #1
0
  public FileChunk get() throws IOException {
    if (useLocalFile) {
      startTime = System.currentTimeMillis();
      finishTime = System.currentTimeMillis();
      state = TajoProtos.FetcherState.FETCH_FINISHED;
      return fileChunk;
    }

    this.startTime = System.currentTimeMillis();
    this.state = TajoProtos.FetcherState.FETCH_FETCHING;
    ChannelFuture future = null;
    try {
      future =
          bootstrap
              .clone()
              .connect(new InetSocketAddress(host, port))
              .addListener(ChannelFutureListener.CLOSE_ON_FAILURE);

      // Wait until the connection attempt succeeds or fails.
      Channel channel = future.awaitUninterruptibly().channel();
      if (!future.isSuccess()) {
        state = TajoProtos.FetcherState.FETCH_FAILED;
        throw new IOException(future.cause());
      }

      String query = uri.getPath() + (uri.getRawQuery() != null ? "?" + uri.getRawQuery() : "");
      // Prepare the HTTP request.
      HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, query);
      request.headers().set(HttpHeaders.Names.HOST, host);
      request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.CLOSE);
      request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP);

      LOG.info("Status: " + getState() + ", URI:" + uri);
      // Send the HTTP request.
      channel.writeAndFlush(request);

      // Wait for the server to close the connection. throw exception if failed
      channel.closeFuture().syncUninterruptibly();

      fileChunk.setLength(fileChunk.getFile().length());
      return fileChunk;
    } finally {
      if (future != null && future.channel().isOpen()) {
        // Close the channel to exit.
        future.channel().close().awaitUninterruptibly();
      }

      this.finishTime = System.currentTimeMillis();
      LOG.info(
          "Fetcher finished:" + (finishTime - startTime) + " ms, " + getState() + ", URI:" + uri);
    }
  }
  /**
   * Flush the response contents to the output channel
   *
   * @param channel Channel to be used
   */
  private void writeResponse(Channel channel) {
    // Convert the response content to a ChannelBuffer.
    ByteBuf buf = copiedBuffer(responseContent.toString(), CharsetUtil.UTF_8);
    responseContent.setLength(0);

    // Decide whether to close the connection or not.
    boolean close =
        HttpHeaders.Values.CLOSE.equalsIgnoreCase(request.headers().get(CONNECTION))
            || request.getProtocolVersion().equals(HttpVersion.HTTP_1_0)
                && !HttpHeaders.Values.KEEP_ALIVE.equalsIgnoreCase(
                    request.headers().get(CONNECTION));

    // Build the response object.
    FullHttpResponse response =
        new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK, buf);
    response.headers().set(CONTENT_TYPE, "application/json; charset=UTF-8");

    if (!close) {
      // There's no need to add 'Content-Length' header
      // if this is the last response.
      response.headers().set(CONTENT_LENGTH, buf.readableBytes());
    }

    try {
      // Write the response.
      ChannelFuture future = channel.writeAndFlush(response);
      // Close the connection after the write operation is done if necessary.
      if (close) {
        future.addListener(ChannelFutureListener.CLOSE);
      }
    } catch (Exception e) {
      logger.error("{}", e);
    }
  }
Exemple #3
0
    @Override
    protected void initChannel(Channel channel) throws Exception {
      ChannelPipeline pipeline = channel.pipeline();

      int maxChunkSize = conf.getIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_CHUNK_MAX_SIZE);
      int readTimeout = conf.getIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_READ_TIMEOUT);

      pipeline.addLast("codec", new HttpClientCodec(4096, 8192, maxChunkSize));
      pipeline.addLast("inflater", new HttpContentDecompressor());
      pipeline.addLast("timeout", new ReadTimeoutHandler(readTimeout, TimeUnit.SECONDS));
      pipeline.addLast("handler", new HttpClientHandler(file));
    }
  public void channelRead0(final ChannelHandlerContext ctx, final FullHttpRequest nettyRequest)
      throws Exception {
    if (!nettyRequest.getDecoderResult().isSuccess()) {
      sendError(ctx, HttpResponseStatus.BAD_REQUEST);
      return;
    }

    final long startTime = System.nanoTime();

    final Request request =
        new DefaultRequest(
            new NettyHeadersBackedHeaders(nettyRequest.headers()),
            nettyRequest.getMethod().name(),
            nettyRequest.getUri(),
            nettyRequest.content());

    final Channel channel = ctx.channel();

    final DefaultMutableStatus responseStatus = new DefaultMutableStatus();
    final HttpHeaders httpHeaders = new DefaultHttpHeaders(false);
    final MutableHeaders responseHeaders = new NettyHeadersBackedMutableHeaders(httpHeaders);
    FileHttpTransmitter fileHttpTransmitter =
        new DefaultFileHttpTransmitter(
            nettyRequest,
            httpHeaders,
            channel,
            compressResponses,
            addResponseTimeHeader ? startTime : -1);

    final DefaultEventController<RequestOutcome> requestOutcomeEventController =
        new DefaultEventController<>();

    // We own the lifecycle
    nettyRequest.content().retain();

    final Response response =
        new DefaultResponse(
            responseStatus,
            responseHeaders,
            fileHttpTransmitter,
            ctx.alloc(),
            new Action<ByteBuf>() {
              @Override
              public void execute(final ByteBuf byteBuf) throws Exception {
                final HttpResponse nettyResponse =
                    new CustomHttpResponse(responseStatus.getResponseStatus(), httpHeaders);

                nettyRequest.content().release();
                responseHeaders.set(HttpHeaderConstants.CONTENT_LENGTH, byteBuf.writerIndex());
                boolean shouldClose = true;
                if (channel.isOpen()) {
                  if (isKeepAlive(nettyRequest)) {
                    responseHeaders.set(
                        HttpHeaderConstants.CONNECTION, HttpHeaderConstants.KEEP_ALIVE);
                    shouldClose = false;
                  }

                  long stopTime = System.nanoTime();
                  if (addResponseTimeHeader) {
                    responseHeaders.set(
                        "X-Response-Time", NumberUtil.toMillisDiffString(startTime, stopTime));
                  }

                  execController.getExecution().complete();

                  channel.writeAndFlush(nettyResponse);
                  channel.write(new DefaultHttpContent(byteBuf));
                  ChannelFuture future = channel.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);

                  if (requestOutcomeEventController.isHasListeners()) {
                    Headers headers = new DelegatingHeaders(responseHeaders);
                    Status status =
                        new DefaultStatus(responseStatus.getCode(), responseStatus.getMessage());
                    SentResponse sentResponse = new DefaultSentResponse(headers, status);
                    RequestOutcome requestOutcome =
                        new DefaultRequestOutcome(
                            request, sentResponse, System.currentTimeMillis());
                    requestOutcomeEventController.fire(requestOutcome);
                  }
                  if (shouldClose) {
                    future.addListener(ChannelFutureListener.CLOSE);
                  }
                }
              }
            });

    InetSocketAddress socketAddress = (InetSocketAddress) channel.localAddress();
    final BindAddress bindAddress = new InetSocketAddressBackedBindAddress(socketAddress);

    Action<Action<Object>> subscribeHandler =
        new Action<Action<Object>>() {
          @Override
          public void execute(Action<Object> thing) throws Exception {
            channelSubscriptions.put(channel, thing);
            channel
                .closeFuture()
                .addListener(
                    new ChannelFutureListener() {
                      @Override
                      public void operationComplete(ChannelFuture future) throws Exception {
                        channelSubscriptions.remove(channel);
                      }
                    });
          }
        };

    final DirectChannelAccess directChannelAccess =
        new DefaultDirectChannelAccess(channel, subscribeHandler);

    execController.start(
        new Action<Execution>() {
          @Override
          public void execute(Execution execution) throws Exception {
            DefaultContext.RequestConstants requestConstants =
                new DefaultContext.RequestConstants(
                    applicationConstants,
                    bindAddress,
                    request,
                    response,
                    directChannelAccess,
                    requestOutcomeEventController.getRegistry(),
                    execution);

            new DefaultContext(requestConstants, registry, handlers, 0, return404).next();
          }
        });
  }
  public void channelRead0(final ChannelHandlerContext ctx, final FullHttpRequest nettyRequest)
      throws Exception {
    if (!nettyRequest.getDecoderResult().isSuccess()) {
      sendError(ctx, HttpResponseStatus.BAD_REQUEST);
      return;
    }

    final long startTime = addResponseTimeHeader ? System.nanoTime() : 0;
    final Request request =
        new DefaultRequest(
            new NettyHeadersBackedHeaders(nettyRequest.headers()),
            nettyRequest.getMethod().name(),
            nettyRequest.getUri(),
            nettyRequest.content());
    final Channel channel = ctx.channel();
    final DefaultMutableStatus responseStatus = new DefaultMutableStatus();
    final HttpHeaders nettyHeaders = new DefaultHttpHeaders(false);
    final MutableHeaders responseHeaders = new NettyHeadersBackedMutableHeaders(nettyHeaders);
    final MimeTypes mimeTypes = registry.get(MimeTypes.class);
    final DefaultEventController<RequestOutcome> requestOutcomeEventController =
        new DefaultEventController<>();
    final AtomicBoolean transmitted = new AtomicBoolean(false);

    final ResponseTransmitter responseTransmitter =
        new DefaultResponseTransmitter(
            transmitted,
            channel,
            nettyRequest,
            request,
            nettyHeaders,
            responseStatus,
            requestOutcomeEventController,
            startTime);
    final Action<Action<? super ResponseTransmitter>> responseTransmitterWrapper =
        Actions.wrap(responseTransmitter);

    final FileHttpTransmitter fileHttpTransmitter =
        new DefaultFileHttpTransmitter(
            nettyHeaders,
            mimeTypes,
            compressResponses,
            compressionMinSize,
            compressionMimeTypeWhiteList,
            compressionMimeTypeBlackList,
            responseTransmitterWrapper);
    StreamTransmitter streamTransmitter =
        new DefaultStreamTransmitter(nettyRequest, nettyHeaders, channel);

    final Response response =
        new DefaultResponse(
            responseStatus,
            responseHeaders,
            fileHttpTransmitter,
            streamTransmitter,
            ctx.alloc(),
            new Action<ByteBuf>() {
              @Override
              public void execute(final ByteBuf byteBuf) throws Exception {
                responseTransmitterWrapper.execute(
                    new Action<ResponseTransmitter>() {
                      @Override
                      public void execute(ResponseTransmitter responseTransmitter)
                          throws Exception {
                        nettyHeaders.set(HttpHeaders.Names.CONTENT_LENGTH, byteBuf.writerIndex());
                        responseTransmitter.transmit(new DefaultHttpContent(byteBuf));
                      }
                    });
              }
            });

    InetSocketAddress socketAddress = (InetSocketAddress) channel.localAddress();
    final BindAddress bindAddress = new InetSocketAddressBackedBindAddress(socketAddress);

    Action<Action<Object>> subscribeHandler =
        new Action<Action<Object>>() {
          @Override
          public void execute(Action<Object> thing) throws Exception {
            transmitted.set(true);
            channelSubscriptions.put(channel, thing);
            channel
                .closeFuture()
                .addListener(
                    new ChannelFutureListener() {
                      @Override
                      public void operationComplete(ChannelFuture future) throws Exception {
                        channelSubscriptions.remove(channel);
                      }
                    });
          }
        };

    final DirectChannelAccess directChannelAccess =
        new DefaultDirectChannelAccess(channel, subscribeHandler);

    final DefaultContext.RequestConstants requestConstants =
        new DefaultContext.RequestConstants(
            applicationConstants,
            bindAddress,
            request,
            response,
            directChannelAccess,
            requestOutcomeEventController.getRegistry());

    DefaultContext.start(
        execController.getControl(),
        requestConstants,
        registry,
        handlers,
        return404,
        new Action<Execution>() {
          @Override
          public void execute(Execution execution) throws Exception {
            if (!transmitted.get()) {
              Handler lastHandler = requestConstants.handler;
              StringBuilder description = new StringBuilder();
              description
                  .append("No response sent for ")
                  .append(request.getMethod().getName())
                  .append(" request to ")
                  .append(request.getUri())
                  .append(" (last handler: ");

              if (lastHandler instanceof DescribingHandler) {
                ((DescribingHandler) lastHandler).describeTo(description);
              } else {
                DescribingHandlers.describeTo(lastHandler, description);
              }

              description.append(")");
              String message = description.toString();
              LOGGER.warn(message);

              response.status(500);

              if (launchConfig.isDevelopment()) {
                response.send(message);
              } else {
                response.send();
              }
            }
          }
        });
  }