private void sendAck() { // Send results back DTask dt, origDt = _dt; // _dt can go null the instant it is send over wire assert origDt != null; // Freed after completion while ((dt = _dt) != null) { // Retry loop for broken TCP sends AutoBuffer ab = null; try { // Start the ACK with results back to client. If the client is // asking for a class/id mapping (or any job running at FETCH_ACK // priority) then return a udp.fetchack byte instead of a udp.ack. // The receiver thread then knows to handle the mapping at the higher // priority. UDP.udp udp = dt.priority() == H2O.FETCH_ACK_PRIORITY ? UDP.udp.fetchack : UDP.udp.ack; ab = new AutoBuffer(_client, udp._prior).putTask(udp, _tsknum).put1(SERVER_UDP_SEND); assert ab.position() == 1 + 2 + 4 + 1; dt.write(ab); // Write the DTask - could be very large write dt._repliedTcp = ab.hasTCP(); // Resends do not need to repeat TCP result ab.close(); // Then close; send final byte _computedAndReplied = true; // After the final handshake, set computed+replied bit break; // Break out of retry loop } catch (AutoBuffer.AutoBufferException e) { if (!_client._heartbeat._client) // Report on servers only; clients allowed to be flaky Log.info( "IOException during ACK, " + e._ioe.getMessage() + ", t#" + _tsknum + " AB=" + ab + ", waiting and retrying..."); ab.drainClose(); if (_client._heartbeat._client) // Dead client will not accept a TCP ACK response? this.CAS_DT(dt, null); // cancel the ACK try { Thread.sleep(100); } catch (InterruptedException ignore) { } } catch (Exception e) { // Custom serializer just barfed? Log.err(e); // Log custom serializer exception ab.drainClose(); } } // end of while(true) if (dt == null) Log.info( "Cancelled remote task#" + _tsknum + " " + origDt.getClass() + " to " + _client + " has been cancelled by remote"); else { if (dt instanceof MRTask && dt.logVerbose()) Log.debug("Done remote task#" + _tsknum + " " + dt.getClass() + " to " + _client); _client.record_task_answer(this); // Setup for retrying Ack & AckAck, if not canceled } }
// Do the remote execution in a F/J thread & send a reply packet. // Caller must call 'tryComplete'. private static AutoBuffer remexec(DTask dt, H2ONode client, int task, AutoBuffer abold) { abold.close(); // Closing the old guy, returning a new guy // Now compute on it! dt.invoke(client); // Send results back AutoBuffer ab = new AutoBuffer(client).putTask(UDP.udp.ack, task).put1(SERVER_UDP_SEND); dt.write(ab); // Write the DTask dt._repliedTcp = ab.hasTCP(); // Resends do not need to repeat TCP result // Install answer so retries get this very answer client.record_task_answer(task, dt); return ab; }
// Re-send strictly the ack, because we're missing an AckAck final void resend_ack() { assert _computedAndReplied : "Found RPCCall not computed " + _tsknum; DTask dt = _dt; if (dt == null) return; // Received ACKACK already UDP.udp udp = dt.priority() == H2O.FETCH_ACK_PRIORITY ? UDP.udp.fetchack : UDP.udp.ack; AutoBuffer rab = new AutoBuffer(_client, dt.priority()).putTask(udp, _tsknum); boolean wasTCP = dt._repliedTcp; if (wasTCP) rab.put1(RPC.SERVER_TCP_SEND); // Original reply sent via TCP else { rab.put1(RPC.SERVER_UDP_SEND); // Original reply sent via UDP assert rab.position() == 1 + 2 + 4 + 1; dt.write(rab); } assert sz_check(rab) : "Resend of " + _dt.getClass() + " changes size from " + _size + " to " + rab.size(); assert dt._repliedTcp == wasTCP; rab.close(); dt._repliedTcp = wasTCP; // Double retry until we exceed existing age. This is the time to delay // until we try again. Note that we come here immediately on creation, // so the first doubling happens before anybody does any waiting. Also // note the generous 5sec cap: ping at least every 5 sec. _retry += (_retry < MAX_TIMEOUT) ? _retry : MAX_TIMEOUT; }