Exemple #1
0
    private void sendAck() {
      // Send results back
      DTask dt, origDt = _dt; // _dt can go null the instant it is send over wire
      assert origDt != null; // Freed after completion
      while ((dt = _dt) != null) { // Retry loop for broken TCP sends
        AutoBuffer ab = null;
        try {
          // Start the ACK with results back to client.  If the client is
          // asking for a class/id mapping (or any job running at FETCH_ACK
          // priority) then return a udp.fetchack byte instead of a udp.ack.
          // The receiver thread then knows to handle the mapping at the higher
          // priority.

          UDP.udp udp = dt.priority() == H2O.FETCH_ACK_PRIORITY ? UDP.udp.fetchack : UDP.udp.ack;
          ab = new AutoBuffer(_client, udp._prior).putTask(udp, _tsknum).put1(SERVER_UDP_SEND);
          assert ab.position() == 1 + 2 + 4 + 1;
          dt.write(ab); // Write the DTask - could be very large write
          dt._repliedTcp = ab.hasTCP(); // Resends do not need to repeat TCP result
          ab.close(); // Then close; send final byte
          _computedAndReplied = true; // After the final handshake, set computed+replied bit
          break; // Break out of retry loop
        } catch (AutoBuffer.AutoBufferException e) {
          if (!_client._heartbeat._client) // Report on servers only; clients allowed to be flaky
          Log.info(
                "IOException during ACK, "
                    + e._ioe.getMessage()
                    + ", t#"
                    + _tsknum
                    + " AB="
                    + ab
                    + ", waiting and retrying...");
          ab.drainClose();
          if (_client._heartbeat._client) // Dead client will not accept a TCP ACK response?
          this.CAS_DT(dt, null); // cancel the ACK
          try {
            Thread.sleep(100);
          } catch (InterruptedException ignore) {
          }
        } catch (Exception e) { // Custom serializer just barfed?
          Log.err(e); // Log custom serializer exception
          ab.drainClose();
        }
      } // end of while(true)
      if (dt == null)
        Log.info(
            "Cancelled remote task#"
                + _tsknum
                + " "
                + origDt.getClass()
                + " to "
                + _client
                + " has been cancelled by remote");
      else {
        if (dt instanceof MRTask && dt.logVerbose())
          Log.debug("Done remote task#" + _tsknum + " " + dt.getClass() + " to " + _client);
        _client.record_task_answer(this); // Setup for retrying Ack & AckAck, if not canceled
      }
    }
Exemple #2
0
 // TCP large RECEIVE of results.  Note that 'this' is NOT the RPC object
 // that is hoping to get the received object, nor is the current thread the
 // RPC thread blocking for the object.  The current thread is the TCP
 // reader thread.
 static void tcp_ack(final AutoBuffer ab) throws IOException {
   // Get the RPC we're waiting on
   int task = ab.getTask();
   RPC rpc = ab._h2o.taskGet(task);
   // Race with canceling a large RPC fetch: Task is already dead.  Do not
   // bother reading from the TCP socket, just bail out & close socket.
   if (rpc == null || rpc._done) {
     ab.drainClose();
   } else {
     assert rpc._tasknum == task;
     assert !rpc._done;
     // Here we have the result, and we're on the correct Node but wrong
     // Thread.  If we just return, the TCP reader thread will close the
     // remote, the remote will UDP ACK the RPC back, and back on the current
     // Node but in the correct Thread, we'd wake up and realize we received a
     // large result.
     try {
       rpc.response(ab);
     } catch (AutoBuffer.AutoBufferException e) {
       // If TCP fails, we will have done a short-read crushing the original
       // _dt object, and be unable to resend.  This is fatal right now.
       // Really: an unimplemented feature; fix is to notice that a partial
       // TCP read means that the server (1) got our remote_exec request, (2)
       // has computed an answer and was trying to send it to us, (3) failed
       // sending via TCP hence the server knows it failed and will send again
       // without any further work from us.  We need to disable all the resend
       // & retry logic, and wait for the server to re-send our result.
       // Meanwhile the _dt object is crushed with half-read crap, and cannot
       // be trusted except in the base fields.
       throw Log.throwErr(e._ioe);
     }
   }
   // ACKACK the remote, telling him "we got the answer"
   new AutoBuffer(ab._h2o, H2O.ACK_ACK_PRIORITY).putTask(UDP.udp.ackack.ordinal(), task).close();
 }
Exemple #3
0
  protected AutoBuffer response(AutoBuffer ab) {

    assert _tasknum == ab.getTask();
    if (_done) {
      if (!ab.hasTCP()) return ackack(ab, _tasknum); // Ignore duplicate response packet
      ab.drainClose();
    } else {
      int flag = ab.getFlag(); // Must read flag also, to advance ab
      if (flag == SERVER_TCP_SEND) return ackack(ab, _tasknum); // Ignore UDP packet for a TCP reply
      assert flag == SERVER_UDP_SEND : "flag = " + flag;
      synchronized (this) { // Install the answer under lock
        if (_done) {
          if (!ab.hasTCP()) return ackack(ab, _tasknum); // Ignore duplicate response packet
          ab.drainClose();
        } else {
          //          UDPTimeOutThread.PENDING.remove(_tasknum);
          _dt.read(ab); // Read the answer (under lock?)
          _size_rez = ab.size(); // Record received size
          ab
              .close(); // Also finish the read (under lock?  even if canceled, since need to drain
                        // TCP)
          if (!isCancelled()) // Can be canceled already (locally by MRTask while recieving remote
                              // answer)
          _dt.onAck(); // One time only execute (before sending ACKACK)
          _done = true; // Only read one (of many) response packets
          ab._h2o.taskRemove(_tasknum); // Flag as task-completed, even if the result is null
          notifyAll(); // And notify in any case
        }
        if (!isCancelled()) // Can be canceled already
        doAllCompletions(); // Send all tasks needing completion to the work queues
      }
    }
    // AckAck back on a fresh AutoBuffer, since actually closed() the incoming one
    return new AutoBuffer(ab._h2o, H2O.ACK_ACK_PRIORITY)
        .putTask(UDP.udp.ackack.ordinal(), _tasknum);
  }
Exemple #4
0
 // TCP large RECEIVE of results.  Note that 'this' is NOT the RPC object
 // that is hoping to get the received object, nor is the current thread the
 // RPC thread blocking for the object.  The current thread is the TCP
 // reader thread.
 static void tcp_ack(final AutoBuffer ab) {
   // Get the RPC we're waiting on
   int task = ab.getTask();
   RPC rpc = TASKS.get(task);
   // Race with canceling a large RPC fetch: Task is already dead.  Do not
   // bother reading from the TCP socket, just bail out & close socket.
   if (rpc == null) {
     ab.drainClose();
   } else {
     assert rpc._tasknum == task;
     assert !rpc._done;
     // Here we have the result, and we're on the correct Node but wrong
     // Thread.  If we just return, the TCP reader thread will close the
     // remote, the remote will UDP ACK the RPC back, and back on the current
     // Node but in the correct Thread, we'd wake up and realize we received a
     // large result.
     rpc.response(ab);
   }
   // ACKACK the remote, telling him "we got the answer"
   new AutoBuffer(ab._h2o).putTask(UDP.udp.ackack.ordinal(), task).close(true);
 }
Exemple #5
0
  // Handle traffic, from a client to this server asking for work to be done.
  // Called from either a F/J thread (generally with a UDP packet) or from the
  // TCPReceiver thread.
  static void remote_exec(AutoBuffer ab) {
    long lo = ab.get8(0), hi = ab._size >= 16 ? ab.get8(8) : 0;
    final int task = ab.getTask();
    final int flag = ab.getFlag();
    assert flag == CLIENT_UDP_SEND || flag == CLIENT_TCP_SEND; // Client-side send
    // Atomically record an instance of this task, one-time-only replacing a
    // null with an RPCCall, a placeholder while we work on a proper response -
    // and it serves to let us discard dup UDP requests.
    RPCCall old = ab._h2o.has_task(task);
    // This is a UDP packet requesting an answer back for a request sent via
    // TCP but the UDP packet has arrived ahead of the TCP.  Just drop the UDP
    // and wait for the TCP to appear.
    if (old == null && flag == CLIENT_TCP_SEND) {
      Log.warn(
          "got tcp with existing task #, FROM "
              + ab._h2o.toString()
              + " AB: " /* +  UDP.printx16(lo,hi)*/);
      assert !ab.hasTCP()
          : "ERROR: got tcp with existing task #, FROM "
              + ab._h2o.toString()
              + " AB: " /* + UDP.printx16(lo,hi)*/; // All the resends should be UDP only
      // DROP PACKET
    } else if (old == null) { // New task?
      RPCCall rpc;
      try {
        // Read the DTask Right Now.  If we are the TCPReceiver thread, then we
        // are reading in that thread... and thus TCP reads are single-threaded.
        rpc = new RPCCall(ab.get(water.DTask.class), ab._h2o, task);
      } catch (AutoBuffer.AutoBufferException e) {
        // Here we assume it's a TCP fail on read - and ignore the remote_exec
        // request.  The caller will send it again.  NOTE: this case is
        // indistinguishable from a broken short-writer/long-reader bug, except
        // that we'll re-send endlessly and fail endlessly.
        Log.info(
            "Network congestion OR short-writer/long-reader: TCP "
                + e._ioe.getMessage()
                + ",  AB="
                + ab
                + ", ignoring partial send");
        ab.drainClose();
        return;
      }
      RPCCall rpc2 = ab._h2o.record_task(rpc);
      if (rpc2 == null) { // Atomically insert (to avoid double-work)
        if (rpc._dt instanceof MRTask && rpc._dt.logVerbose())
          Log.debug("Start remote task#" + task + " " + rpc._dt.getClass() + " from " + ab._h2o);
        H2O.submitTask(rpc); // And execute!
      } else { // Else lost the task-insertion race
        if (ab.hasTCP()) ab.drainClose();
        // DROP PACKET
      }

    } else if (!old._computedAndReplied) {
      // This packet has not been fully computed.  Hence it's still a work-in-
      // progress locally.  We have no answer to reply but we do not want to
      // re-offer the packet for repeated work.  Send back a NACK, letting the
      // client know we're Working On It
      assert !ab.hasTCP()
          : "got tcp with existing task #, FROM "
              + ab._h2o.toString()
              + " AB: "
              + UDP.printx16(lo, hi)
              + ", position = "
              + ab._bb.position();
      ab.clearForWriting(udp.nack._prior).putTask(UDP.udp.nack.ordinal(), task);
      // DROP PACKET
    } else {
      // This is an old re-send of the same thing we've answered to before.
      // Send back the same old answer ACK.  If we sent via TCP before, then
      // we know the answer got there so just send a control-ACK back.  If we
      // sent via UDP, resend the whole answer.
      if (ab.hasTCP()) {
        Log.warn(
            "got tcp with existing task #, FROM "
                + ab._h2o.toString()
                + " AB: "
                + UDP.printx16(lo, hi)); // All the resends should be UDP only
        ab.drainClose();
      }
      if (old._dt != null) { // already ackacked
        ++old._ackResendCnt;
        if (old._ackResendCnt % 10 == 0)
          Log.err(
              "Possibly broken network, can not send ack through, got "
                  + old._ackResendCnt
                  + " for task # "
                  + old._tsknum
                  + ", dt == null?"
                  + (old._dt == null));
        old.resend_ack();
      }
    }
    ab.close();
  }
Exemple #6
0
  public synchronized RPC<V> call() {
    // Any Completer will not be carried over to remote; add it to the RPC call
    // so completion is signaled after the remote comes back.
    CountedCompleter cc = _dt.getCompleter();
    if (cc != null) handleCompleter(cc);

    // If running on self, just submit to queues & do locally
    if (_target == H2O.SELF) return handleLocal();

    // Keep a global record, for awhile
    if (_target != null) _target.taskPut(_tasknum, this);
    try {
      if (_nack) return this; // Racing Nack rechecked under lock; no need to send retry
      // We could be racing timeouts-vs-replies.  Blow off timeout if we have an answer.
      if (isDone()) {
        if (_target != null) _target.taskRemove(_tasknum);
        return this;
      }
      // Default strategy: (re)fire the packet and (re)start the timeout.  We
      // "count" exactly 1 failure: just whether or not we shipped via TCP ever
      // once.  After that we fearlessly (re)send UDP-sized packets until the
      // server replies.

      // Pack classloader/class & the instance data into the outgoing
      // AutoBuffer.  If it fits in a single UDP packet, ship it.  If not,
      // finish off the current AutoBuffer (which is now going TCP style), and
      // make a new UDP-sized packet.  On a re-send of a TCP-sized hunk, just
      // send the basic UDP control packet.
      if (!_sentTcp) {
        while (true) { // Retry loop for broken TCP sends
          AutoBuffer ab = new AutoBuffer(_target, _dt.priority());
          try {
            final boolean t;
            int offset = ab.position();
            ab.putTask(UDP.udp.exec, _tasknum).put1(CLIENT_UDP_SEND);
            ab.put(_dt);
            t = ab.hasTCP();
            assert sz_check(ab)
                : "Resend of "
                    + _dt.getClass()
                    + " changes size from "
                    + _size
                    + " to "
                    + ab.size()
                    + " for task#"
                    + _tasknum;
            ab.close(); // Then close; send final byte
            _sentTcp = t; // Set after close (and any other possible fail)
            break; // Break out of retry loop
          } catch (AutoBuffer.AutoBufferException e) {
            Log.info(
                "IOException during RPC call: "
                    + e._ioe.getMessage()
                    + ",  AB="
                    + ab
                    + ", for task#"
                    + _tasknum
                    + ", waiting and retrying...");
            ab.drainClose();
            try {
              Thread.sleep(500);
            } catch (InterruptedException ignore) {
            }
          }
        } // end of while(true)
      } else {
        // Else it was sent via TCP in a prior attempt, and we've timed out.
        // This means the caller's ACK/answer probably got dropped and we need
        // him to resend it (or else the caller is still processing our
        // request).  Send a UDP reminder - but with the CLIENT_TCP_SEND flag
        // instead of the UDP send, and no DTask (since it previously went via
        // TCP, no need to resend it).
        AutoBuffer ab = new AutoBuffer(_target, _dt.priority()).putTask(UDP.udp.exec, _tasknum);
        ab.put1(CLIENT_TCP_SEND).close();
      }
      // Double retry until we exceed existing age.  This is the time to delay
      // until we try again.  Note that we come here immediately on creation,
      // so the first doubling happens before anybody does any waiting.  Also
      // note the generous 5sec cap: ping at least every 5 sec.
      _retry += (_retry < MAX_TIMEOUT) ? _retry : MAX_TIMEOUT;
      // Put self on the "TBD" list of tasks awaiting Timeout.
      // So: dont really 'forget' but remember me in a little bit.
      //      UDPTimeOutThread.PENDING.put(_tasknum, this);
      return this;
    } catch (Throwable t) {
      t.printStackTrace();
      throw Log.throwErr(t);
    }
  }