private ChannelFuture sendFile(ChannelHandlerContext ctx, Channel ch, FileChunk file)
      throws IOException {
    RandomAccessFile raf;
    try {
      raf = new RandomAccessFile(file.getFile(), "r");
    } catch (FileNotFoundException fnfe) {
      return null;
    }

    ChannelFuture writeFuture;
    if (ch.getPipeline().get(SslHandler.class) != null) {
      // Cannot use zero-copy with HTTPS.
      writeFuture = ch.write(new ChunkedFile(raf, file.startOffset(), file.length(), 8192));
    } else {
      // No encryption - use zero-copy.
      final FileRegion region =
          new DefaultFileRegion(raf.getChannel(), file.startOffset(), file.length());
      writeFuture = ch.write(region);
      writeFuture.addListener(
          new ChannelFutureListener() {
            public void operationComplete(ChannelFuture future) {
              region.releaseExternalResources();
            }
          });
    }

    return writeFuture;
  }
 private ChannelFuture sendFile(ChannelHandlerContext ctx, Channel ch, FileChunk file)
     throws IOException {
   RandomAccessFile spill;
   try {
     spill = new RandomAccessFile(file.getFile(), "r");
   } catch (FileNotFoundException e) {
     LOG.info(file.getFile() + " not found");
     return null;
   }
   ChannelFuture writeFuture;
   if (ch.getPipeline().get(SslHandler.class) == null) {
     final FadvisedFileRegion partition =
         new FadvisedFileRegion(
             spill,
             file.startOffset,
             file.length(),
             manageOsCache,
             readaheadLength,
             readaheadPool,
             file.getFile().getAbsolutePath());
     writeFuture = ch.write(partition);
     writeFuture.addListener(
         new ChannelFutureListener() {
           // TODO error handling; distinguish IO/connection failures,
           //      attribute to appropriate spill output
           @Override
           public void operationComplete(ChannelFuture future) {
             partition.releaseExternalResources();
           }
         });
   } else {
     // HTTPS cannot be done with zero copy.
     final FadvisedChunkedFile chunk =
         new FadvisedChunkedFile(
             spill,
             file.startOffset,
             file.length,
             sslFileBufferSize,
             manageOsCache,
             readaheadLength,
             readaheadPool,
             file.getFile().getAbsolutePath());
     writeFuture = ch.write(chunk);
   }
   metrics.shuffleConnections.incr();
   metrics.shuffleOutputBytes.incr(file.length); // optimistic
   return writeFuture;
 }
Esempio n. 3
0
  private void sendHttpResponse(Channel channel, HttpRequest request, HttpResponse response) {
    if (!channel.isOpen()) {
      return;
    }

    // response的内容已在各Listener中填充
    response.headers().set(Names.CONTENT_LENGTH, response.getContent().readableBytes());
    response.headers().set(Names.SERVER, "NAVI/1.1.4(UNIX)");

    if (!HttpHeaders.isKeepAlive(request)
        || response.getStatus() != HttpResponseStatus.OK
        || ServerConfigure.isChannelClose()) {
      response.headers().set(Names.CONNECTION, "close");
      channel.setAttachment(WRITING);
      ChannelFuture f = channel.write(response);
      f.addListener(ChannelFutureListener.CLOSE);
      f.addListener(
          new ChannelFutureListener() {

            public void operationComplete(ChannelFuture f) throws Exception {
              if (!f.isSuccess()) {
                log.error(f.getCause().getMessage(), f.getCause());
              }
            }
          });
    } else {
      if (request.getProtocolVersion() == HttpVersion.HTTP_1_0) {
        response.headers().add(Names.CONNECTION, "Keep-Alive");
      }
      channel.setAttachment(WRITING);
      ChannelFuture f = channel.write(response);
      f.addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
      f.addListener(
          new ChannelFutureListener() {

            public void operationComplete(ChannelFuture f) throws Exception {
              if (!f.isSuccess()) {
                log.error(f.getCause().getMessage(), f.getCause());
              }
            }
          });
    }
  }
Esempio n. 4
0
  @Override
  public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
    Channel channel = ctx.getChannel();
    if (channel.getAttachment() != null
        && (Error.class.isAssignableFrom(channel.getAttachment().getClass())
            || WRITING.equals(channel.getAttachment().toString()))) {
      return;
    }

    if (!isReceived(ctx)) {
      return;
    }

    try {
      log.error(e.getCause().getMessage(), e.getCause());
      channel.setAttachment(new Error());
      DefaultHttpResponse response =
          new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR);
      response.setContent(
          ChannelBuffers.copiedBuffer(
              response.getStatus().toString() + ":" + e.getCause().getMessage(),
              CharsetUtil.UTF_8));
      response.headers().set(Names.CONTENT_LENGTH, response.getContent().readableBytes());
      response.headers().set(Names.SERVER, "NAVI/1.1.4(UNIX)");
      response.headers().set(Names.CONNECTION, "close");
      ChannelFuture f = channel.write(response);
      f.addListener(ChannelFutureListener.CLOSE);
      f.addListener(
          new ChannelFutureListener() {

            public void operationComplete(ChannelFuture f) throws Exception {
              if (!f.isSuccess()) {
                log.error(f.getCause().getMessage(), f.getCause());
              }
            }
          });
    } catch (Exception ex) {
      log.error(e.getCause().getMessage(), e.getCause());
      e.getFuture().addListener(ChannelFutureListener.CLOSE);
    }
  }
Esempio n. 5
0
  public File get() throws IOException {
    ClientBootstrap bootstrap =
        new ClientBootstrap(
            new NioClientSocketChannelFactory(
                Executors.newCachedThreadPool(), Executors.newCachedThreadPool()));
    bootstrap.setOption("connectTimeoutMillis", 5000L); // set 5 sec
    bootstrap.setOption("receiveBufferSize", 1048576); // set 1M
    ChannelPipelineFactory factory = new HttpClientPipelineFactory(file);
    bootstrap.setPipelineFactory(factory);

    ChannelFuture future = bootstrap.connect(new InetSocketAddress(host, port));

    // Wait until the connection attempt succeeds or fails.
    Channel channel = future.awaitUninterruptibly().getChannel();
    if (!future.isSuccess()) {
      bootstrap.releaseExternalResources();
      throw new IOException(future.getCause());
    }

    String query = uri.getPath() + (uri.getQuery() != null ? "?" + uri.getQuery() : "");
    // Prepare the HTTP request.
    HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, query);
    request.setHeader(HttpHeaders.Names.HOST, host);
    LOG.info("Fetch: " + request.getUri());
    request.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.CLOSE);
    request.setHeader(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP);

    // Send the HTTP request.
    channel.write(request);

    // Wait for the server to close the connection.
    channel.getCloseFuture().awaitUninterruptibly();

    // Shut down executor threads to exit.
    bootstrap.releaseExternalResources();

    return file;
  }
    @Override
    public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {

      HttpRequest request = (HttpRequest) e.getMessage();
      if (request.getMethod() != GET) {
        sendError(ctx, METHOD_NOT_ALLOWED);
        return;
      }

      // Parsing the URL into key-values
      final Map<String, List<String>> params =
          new QueryStringDecoder(request.getUri()).getParameters();
      final List<String> types = params.get("type");
      final List<String> taskIdList = params.get("ta");
      final List<String> subQueryIds = params.get("sid");
      final List<String> partitionIds = params.get("p");

      if (types == null || taskIdList == null || subQueryIds == null || partitionIds == null) {
        sendError(ctx, "Required type, taskIds, subquery Id, and partition id", BAD_REQUEST);
        return;
      }

      if (types.size() != 1 || subQueryIds.size() != 1) {
        sendError(ctx, "Required type, taskIds, subquery Id, and partition id", BAD_REQUEST);
        return;
      }

      final List<FileChunk> chunks = Lists.newArrayList();

      String repartitionType = types.get(0);
      String sid = subQueryIds.get(0);
      String partitionId = partitionIds.get(0);
      List<String> taskIds = splitMaps(taskIdList);

      // the working dir of tajo worker for each query
      String queryBaseDir = queryId + "/output" + "/";

      LOG.info(
          "PullServer request param: repartitionType="
              + repartitionType
              + ", sid="
              + sid
              + ", partitionId="
              + partitionId
              + ", taskIds="
              + taskIdList);

      String taskLocalDir = conf.get(ConfVars.WORKER_TEMPORAL_DIR.varname);
      if (taskLocalDir == null || taskLocalDir.equals("")) {
        LOG.error("Tajo local directory should be specified.");
      }
      LOG.info("PullServer baseDir: " + taskLocalDir + "/" + queryBaseDir);

      // if a subquery requires a range partitioning
      if (repartitionType.equals("r")) {
        String ta = taskIds.get(0);
        Path path =
            localFS.makeQualified(
                lDirAlloc.getLocalPathToRead(
                    queryBaseDir + "/" + sid + "/" + ta + "/output/", conf));

        String startKey = params.get("start").get(0);
        String endKey = params.get("end").get(0);
        boolean last = params.get("final") != null;

        FileChunk chunk;
        try {
          chunk = getFileCunks(path, startKey, endKey, last);
        } catch (Throwable t) {
          LOG.error("ERROR Request: " + request.getUri(), t);
          sendError(ctx, "Cannot get file chunks to be sent", BAD_REQUEST);
          return;
        }
        if (chunk != null) {
          chunks.add(chunk);
        }

        // if a subquery requires a hash repartition
      } else if (repartitionType.equals("h")) {
        for (String ta : taskIds) {
          Path path =
              localFS.makeQualified(
                  lDirAlloc.getLocalPathToRead(
                      queryBaseDir + "/" + sid + "/" + ta + "/output/" + partitionId, conf));
          File file = new File(path.toUri());
          FileChunk chunk = new FileChunk(file, 0, file.length());
          chunks.add(chunk);
        }
      } else {
        LOG.error("Unknown repartition type: " + repartitionType);
        return;
      }

      // Write the content.
      Channel ch = e.getChannel();
      if (chunks.size() == 0) {
        HttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT);
        ch.write(response);
        if (!isKeepAlive(request)) {
          ch.close();
        }
      } else {
        FileChunk[] file = chunks.toArray(new FileChunk[chunks.size()]);
        HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
        long totalSize = 0;
        for (FileChunk chunk : file) {
          totalSize += chunk.length();
        }
        setContentLength(response, totalSize);

        // Write the initial line and the header.
        ch.write(response);

        ChannelFuture writeFuture = null;

        for (FileChunk chunk : file) {
          writeFuture = sendFile(ctx, ch, chunk);
          if (writeFuture == null) {
            sendError(ctx, NOT_FOUND);
            return;
          }
        }

        // Decide whether to close the connection or not.
        if (!isKeepAlive(request)) {
          // Close the connection when the whole content is written out.
          writeFuture.addListener(ChannelFutureListener.CLOSE);
        }
      }
    }
Esempio n. 7
0
  public void serveStatic(
      RenderStatic renderStatic,
      ChannelHandlerContext ctx,
      Request request,
      Response response,
      HttpRequest nettyRequest,
      MessageEvent e) {
    Logger.trace("serveStatic: begin");
    HttpResponse nettyResponse =
        new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.valueOf(response.status));
    if (exposePlayServer) {
      nettyResponse.setHeader(SERVER, signature);
    }
    try {
      VirtualFile file = Play.getVirtualFile(renderStatic.file);
      if (file != null && file.exists() && file.isDirectory()) {
        file = file.child("index.html");
        if (file != null) {
          renderStatic.file = file.relativePath();
        }
      }
      if ((file == null || !file.exists())) {
        serve404(
            new NotFound("The file " + renderStatic.file + " does not exist"),
            ctx,
            request,
            nettyRequest);
      } else {
        boolean raw = false;
        for (PlayPlugin plugin : Play.plugins) {
          if (plugin.serveStatic(file, Request.current(), Response.current())) {
            raw = true;
            break;
          }
        }
        if (raw) {
          copyResponse(ctx, request, response, nettyRequest);
        } else {
          final File localFile = file.getRealFile();
          final boolean keepAlive = isKeepAlive(nettyRequest);
          nettyResponse = addEtag(nettyRequest, nettyResponse, localFile);

          if (nettyResponse.getStatus().equals(HttpResponseStatus.NOT_MODIFIED)) {

            Channel ch = e.getChannel();

            // Write the initial line and the header.
            ChannelFuture writeFuture = ch.write(nettyResponse);
            if (!keepAlive) {
              // Write the content.
              writeFuture.addListener(ChannelFutureListener.CLOSE);
            }
          } else {

            final RandomAccessFile raf = new RandomAccessFile(localFile, "r");
            try {
              long fileLength = raf.length();

              Logger.trace("keep alive " + keepAlive);
              Logger.trace(
                  "content type " + (MimeTypes.getContentType(localFile.getName(), "text/plain")));

              if (keepAlive && !nettyResponse.getStatus().equals(HttpResponseStatus.NOT_MODIFIED)) {
                // Add 'Content-Length' header only for a keep-alive connection.
                Logger.trace("file length " + fileLength);
                setContentLength(nettyResponse, fileLength);
              }

              nettyResponse.setHeader(
                  CONTENT_TYPE, (MimeTypes.getContentType(localFile.getName(), "text/plain")));

              Channel ch = e.getChannel();

              // Write the initial line and the header.
              ChannelFuture writeFuture = ch.write(nettyResponse);

              // Write the content.
              if (!nettyRequest.getMethod().equals(HttpMethod.HEAD)) {
                writeFuture = ch.write(new ChunkedFile(raf, 0, fileLength, 8192));
              } else {
                raf.close();
              }

              if (!keepAlive) {
                // Close the connection when the whole content is written out.
                writeFuture.addListener(ChannelFutureListener.CLOSE);
              }
            } catch (Throwable exx) {
              try {
                raf.close();
              } catch (Throwable ex) {
                /* Left empty */
              }
              try {
                ctx.getChannel().close();
              } catch (Throwable ex) {
                /* Left empty */
              }
            }
          }
        }
      }
    } catch (Throwable ez) {
      Logger.error(ez, "serveStatic for request %s", request.method + " " + request.url);
      try {
        HttpResponse errorResponse =
            new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR);
        ChannelBuffer buf =
            ChannelBuffers.copiedBuffer("Internal Error (check logs)".getBytes("utf-8"));
        errorResponse.setContent(buf);
        ChannelFuture future = ctx.getChannel().write(errorResponse);
        future.addListener(ChannelFutureListener.CLOSE);
      } catch (Exception ex) {
        Logger.error(ez, "serveStatic for request %s", request.method + " " + request.url);
      }
    }
    Logger.trace("serveStatic: end");
  }
Esempio n. 8
0
  public void copyResponse(
      ChannelHandlerContext ctx, Request request, Response response, HttpRequest nettyRequest)
      throws Exception {
    Logger.trace("copyResponse: begin");
    // response.out.flush();

    // Decide whether to close the connection or not.

    HttpResponse nettyResponse =
        new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.valueOf(response.status));
    if (exposePlayServer) {
      nettyResponse.setHeader(SERVER, signature);
    }

    if (response.contentType != null) {
      nettyResponse.setHeader(
          CONTENT_TYPE,
          response.contentType
              + (response.contentType.startsWith("text/")
                      && !response.contentType.contains("charset")
                  ? "; charset=utf-8"
                  : ""));
    } else {
      nettyResponse.setHeader(CONTENT_TYPE, "text/plain; charset=utf-8");
    }

    addToResponse(response, nettyResponse);

    final Object obj = response.direct;
    File file = null;
    ChunkedInput stream = null;
    InputStream is = null;
    if (obj instanceof File) {
      file = (File) obj;
    } else if (obj instanceof InputStream) {
      is = (InputStream) obj;
    } else if (obj instanceof ChunkedInput) {
      stream = (ChunkedInput) obj;
    }

    final boolean keepAlive = isKeepAlive(nettyRequest);
    if (file != null && file.isFile()) {
      try {
        nettyResponse = addEtag(nettyRequest, nettyResponse, file);
        if (nettyResponse.getStatus().equals(HttpResponseStatus.NOT_MODIFIED)) {

          Channel ch = ctx.getChannel();

          // Write the initial line and the header.
          ChannelFuture writeFuture = ch.write(nettyResponse);

          if (!keepAlive) {
            // Close the connection when the whole content is written out.
            writeFuture.addListener(ChannelFutureListener.CLOSE);
          }
        } else {
          nettyResponse.setHeader(
              CONTENT_TYPE, MimeTypes.getContentType(file.getName(), "text/plain"));
          final RandomAccessFile raf = new RandomAccessFile(file, "r");
          try {
            long fileLength = raf.length();

            if (keepAlive) {
              // Add 'Content-Length' header only for a keep-alive connection.
              Logger.trace("file length is [" + fileLength + "]");
              setContentLength(nettyResponse, fileLength);
            }

            Channel ch = ctx.getChannel();

            // Write the initial line and the header.
            ChannelFuture writeFuture = ch.write(nettyResponse);

            // Write the content.
            // If it is not a HEAD
            if (!nettyRequest.getMethod().equals(HttpMethod.HEAD)) {
              writeFuture = ch.write(new ChunkedFile(raf, 0, fileLength, 8192));
            } else {
              raf.close();
            }
            if (!keepAlive) {
              // Close the connection when the whole content is written out.
              writeFuture.addListener(ChannelFutureListener.CLOSE);
            }
          } catch (Throwable exx) {
            try {
              raf.close();
            } catch (Throwable ex) {
              /* Left empty */
            }
            try {
              ctx.getChannel().close();
            } catch (Throwable ex) {
              /* Left empty */
            }
          }
        }
      } catch (Exception e) {
        throw e;
      }
    } else if (is != null) {
      ChannelFuture writeFuture = ctx.getChannel().write(nettyResponse);
      if (!nettyRequest.getMethod().equals(HttpMethod.HEAD)
          && !nettyResponse.getStatus().equals(HttpResponseStatus.NOT_MODIFIED)) {
        writeFuture = ctx.getChannel().write(new ChunkedStream(is));
      } else {
        is.close();
      }
      if (!keepAlive) {
        writeFuture.addListener(ChannelFutureListener.CLOSE);
      }
    } else if (stream != null) {
      ChannelFuture writeFuture = ctx.getChannel().write(nettyResponse);
      if (!nettyRequest.getMethod().equals(HttpMethod.HEAD)
          && !nettyResponse.getStatus().equals(HttpResponseStatus.NOT_MODIFIED)) {
        writeFuture = ctx.getChannel().write(stream);
      } else {
        stream.close();
      }
      if (!keepAlive) {
        writeFuture.addListener(ChannelFutureListener.CLOSE);
      }
    } else {
      writeResponse(ctx, response, nettyResponse, nettyRequest);
    }
    Logger.trace("copyResponse: end");
  }
  @Override
  public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
    HttpRequest request = (HttpRequest) e.getMessage();
    if (request.getMethod() != GET) {
      sendError(ctx, METHOD_NOT_ALLOWED);
      return;
    }

    String base =
        ContainerLocalizer.USERCACHE
            + "/"
            + userName
            + "/"
            + ContainerLocalizer.APPCACHE
            + "/"
            + appId
            + "/output"
            + "/";

    final Map<String, List<String>> params =
        new QueryStringDecoder(request.getUri()).getParameters();

    List<FileChunk> chunks = Lists.newArrayList();
    List<String> taskIds = splitMaps(params.get("ta"));
    int sid = Integer.valueOf(params.get("sid").get(0));
    int partitionId = Integer.valueOf(params.get("p").get(0));
    for (String ta : taskIds) {

      File file = new File(base + "/" + sid + "/" + ta + "/output/" + partitionId);
      FileChunk chunk = new FileChunk(file, 0, file.length());
      chunks.add(chunk);
    }

    FileChunk[] file = chunks.toArray(new FileChunk[chunks.size()]);
    //    try {
    //      file = retriever.handle(ctx, request);
    //    } catch (FileNotFoundException fnf) {
    //      LOG.error(fnf);
    //      sendError(ctx, NOT_FOUND);
    //      return;
    //    } catch (IllegalArgumentException iae) {
    //      LOG.error(iae);
    //      sendError(ctx, BAD_REQUEST);
    //      return;
    //    } catch (FileAccessForbiddenException fafe) {
    //      LOG.error(fafe);
    //      sendError(ctx, FORBIDDEN);
    //      return;
    //    } catch (IOException ioe) {
    //      LOG.error(ioe);
    //      sendError(ctx, INTERNAL_SERVER_ERROR);
    //      return;
    //    }

    // Write the content.
    Channel ch = e.getChannel();
    if (file == null) {
      HttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT);
      ch.write(response);
      if (!isKeepAlive(request)) {
        ch.close();
      }
    } else {
      HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
      long totalSize = 0;
      for (FileChunk chunk : file) {
        totalSize += chunk.length();
      }
      setContentLength(response, totalSize);

      // Write the initial line and the header.
      ch.write(response);

      ChannelFuture writeFuture = null;

      for (FileChunk chunk : file) {
        writeFuture = sendFile(ctx, ch, chunk);
        if (writeFuture == null) {
          sendError(ctx, NOT_FOUND);
          return;
        }
      }

      // Decide whether to close the connection or not.
      if (!isKeepAlive(request)) {
        // Close the connection when the whole content is written out.
        writeFuture.addListener(ChannelFutureListener.CLOSE);
      }
    }
  }