コード例 #1
0
ファイル: Downloader.java プロジェクト: ttyunix/crawl
  public void sendRequest(MetaInfo meta) {
    System.out.println(Thread.currentThread().getId() + " start sendRequest");
    URI uri = null;
    try {
      System.out.println(meta.getParams());
      uri = new URI(meta.getUrl());
    } catch (URISyntaxException e) {
      e.printStackTrace(); // To change body of catch statement use File | Settings | File
      // Templates.
    }
    String host = uri.getHost();
    int port = 80;

    HttpRequest request =
        new DefaultHttpRequest(
            HttpVersion.HTTP_1_1, HttpMethod.valueOf(meta.getMethod()), uri.toASCIIString());
    meta.buildHttpRequestHeader(request);

    ChannelFuture future = bootstrap.connect(new InetSocketAddress(host, port));
    Channel channel = future.getChannel();
    channel.getPipeline().addLast("handler", new DownloaderHandler());
    GlobalVar.metaInfoVar.set(channel, meta);

    future.addListener(new ConnectOk(request));
    channel.getCloseFuture().awaitUninterruptibly().addListener(new ConnectClose());
    System.out.println(Thread.currentThread().getId() + " end sendRequest");
  }
コード例 #2
0
  private ChannelFuture sendFile(ChannelHandlerContext ctx, Channel ch, FileChunk file)
      throws IOException {
    RandomAccessFile raf;
    try {
      raf = new RandomAccessFile(file.getFile(), "r");
    } catch (FileNotFoundException fnfe) {
      return null;
    }

    ChannelFuture writeFuture;
    if (ch.getPipeline().get(SslHandler.class) != null) {
      // Cannot use zero-copy with HTTPS.
      writeFuture = ch.write(new ChunkedFile(raf, file.startOffset(), file.length(), 8192));
    } else {
      // No encryption - use zero-copy.
      final FileRegion region =
          new DefaultFileRegion(raf.getChannel(), file.startOffset(), file.length());
      writeFuture = ch.write(region);
      writeFuture.addListener(
          new ChannelFutureListener() {
            public void operationComplete(ChannelFuture future) {
              region.releaseExternalResources();
            }
          });
    }

    return writeFuture;
  }
コード例 #3
0
 private ChannelFuture sendFile(ChannelHandlerContext ctx, Channel ch, FileChunk file)
     throws IOException {
   RandomAccessFile spill;
   try {
     spill = new RandomAccessFile(file.getFile(), "r");
   } catch (FileNotFoundException e) {
     LOG.info(file.getFile() + " not found");
     return null;
   }
   ChannelFuture writeFuture;
   if (ch.getPipeline().get(SslHandler.class) == null) {
     final FadvisedFileRegion partition =
         new FadvisedFileRegion(
             spill,
             file.startOffset,
             file.length(),
             manageOsCache,
             readaheadLength,
             readaheadPool,
             file.getFile().getAbsolutePath());
     writeFuture = ch.write(partition);
     writeFuture.addListener(
         new ChannelFutureListener() {
           // TODO error handling; distinguish IO/connection failures,
           //      attribute to appropriate spill output
           @Override
           public void operationComplete(ChannelFuture future) {
             partition.releaseExternalResources();
           }
         });
   } else {
     // HTTPS cannot be done with zero copy.
     final FadvisedChunkedFile chunk =
         new FadvisedChunkedFile(
             spill,
             file.startOffset,
             file.length,
             sslFileBufferSize,
             manageOsCache,
             readaheadLength,
             readaheadPool,
             file.getFile().getAbsolutePath());
     writeFuture = ch.write(chunk);
   }
   metrics.shuffleConnections.incr();
   metrics.shuffleOutputBytes.incr(file.length); // optimistic
   return writeFuture;
 }