public void execute(Job job) {
// Get the input as a String
String data = ByteUtils.fromBytes(job.getData(), encoding);
// Perform the reversal
StringBuffer sb = new StringBuffer(data);
sb = sb.reverse();
// Store result as bytes
job.setResult(ByteUtils.toBytes(sb.toString(), encoding));
// Set the job state
job.setState(JobState.COMPLETE);
}
/**
* "\0REQ" == [ 00 52 45 51 ] == 5391697
*
* <p>"\0RES" == [ 00 52 45 53 ] == 5391699
*
* @param bytes array of bytes containing magic code for a packet
* @throws BadMagicException if byte array is not 4 bytes long
*/
public static GearmanPacketMagic fromBytes(byte[] bytes) {
if (bytes != null && bytes.length == 4) {
int magic = ByteUtils.fromBigEndian(bytes);
if (magic == ByteUtils.fromBigEndian(REQ.toBytes())) {
return REQ;
}
if (magic == ByteUtils.fromBigEndian(RES.toBytes())) {
return RES;
}
}
throw new BadMagicException(ByteUtils.toHex(bytes));
}
private GearmanPacket getPacketFromJob(GearmanJob job) {
int destPos = 0;
GearmanPacketMagic magic = GearmanPacketMagic.REQ;
GearmanPacketType type = null;
byte[] packetdata = null;
byte[] fnname = ByteUtils.toAsciiBytes(job.getFunctionName());
byte[] uid = job.getID();
byte[] data = job.getData();
if (job.getPriority().equals(GearmanJob.JobPriority.HIGH)) {
type =
job.isBackgroundJob()
? GearmanPacketType.SUBMIT_JOB_HIGH_BG
: GearmanPacketType.SUBMIT_JOB_HIGH;
}
if (job.getPriority().equals(GearmanJob.JobPriority.LOW)) {
type =
job.isBackgroundJob()
? GearmanPacketType.SUBMIT_JOB_LOW_BG
: GearmanPacketType.SUBMIT_JOB_LOW;
}
if (job.getPriority().equals(GearmanJob.JobPriority.NORMAL)) {
type = job.isBackgroundJob() ? GearmanPacketType.SUBMIT_JOB_BG : GearmanPacketType.SUBMIT_JOB;
}
packetdata = new byte[fnname.length + uid.length + data.length + 2];
System.arraycopy(fnname, 0, packetdata, destPos, fnname.length);
destPos += fnname.length;
packetdata[destPos++] = ByteUtils.NULL;
System.arraycopy(uid, 0, packetdata, destPos, uid.length);
destPos += uid.length;
packetdata[destPos++] = ByteUtils.NULL;
System.arraycopy(data, 0, packetdata, destPos, data.length);
return new GearmanPacketImpl(magic, type, packetdata);
}
public byte[] digest(byte[] input) {
String function = DigestFunction.class.getCanonicalName();
String uniqueId = null;
byte[] algoNameBytes = ALGO_NAME.getBytes();
byte[] digestParms = new byte[input.length + algoNameBytes.length + 1];
System.arraycopy(algoNameBytes, 0, digestParms, 0, algoNameBytes.length);
digestParms[algoNameBytes.length] = '\0';
System.arraycopy(input, 0, digestParms, algoNameBytes.length + 1, input.length);
GearmanJob job = GearmanJobImpl.createJob(function, digestParms, uniqueId);
Future<GearmanJobResult> f = client.submit(job);
GearmanJobResult jr = null;
try {
jr = f.get();
} catch (Exception e) {
e.printStackTrace(); // NOPMD
}
if (jr == null) {
return new byte[0];
} else {
if (jr.jobSucceeded()) {
return jr.getResults();
} else {
System.err.println(ByteUtils.fromUTF8Bytes(jr.getExceptions())); // NOPMD
return new byte[0];
}
}
}
private void addNewJob(GearmanSessionEvent event) {
byte[] handle, data, functionNameBytes, unique;
GearmanPacket p = event.getPacket();
GearmanJobServerSession sess = event.getSession();
String functionName;
handle = p.getDataComponentValue(GearmanPacket.DataComponentName.JOB_HANDLE);
functionNameBytes = p.getDataComponentValue(GearmanPacket.DataComponentName.FUNCTION_NAME);
data = p.getDataComponentValue(GearmanPacket.DataComponentName.DATA);
unique = p.getDataComponentValue(DataComponentName.UNIQUE_ID);
functionName = ByteUtils.fromUTF8Bytes(functionNameBytes);
FunctionDefinition def = functionMap.get(functionName);
if (def == null) {
GearmanTask gsr =
new GearmanTask(
new GearmanPacketImpl(GearmanPacketMagic.REQ, GearmanPacketType.WORK_FAIL, handle));
sess.submitTask(gsr);
} else {
GearmanFunction function = def.getFactory().getFunction();
function.setData(data);
function.setJobHandle(handle);
function.registerEventListener(sess);
if (unique != null && unique.length > 0) {
function.setUniqueId(unique);
}
functionList.add(function);
}
}
public boolean addServer(GearmanJobServerConnection conn)
throws IllegalArgumentException, IllegalStateException {
if (conn == null) {
throw new IllegalArgumentException("Connection can not be null");
}
// this is a sub-optimal way to look for dups, but addJobServer
// ops should be infrequent enough that this should be a big penalty
for (GearmanJobServerSession sess : sessionMap.values()) {
if (sess.getConnection().equals(conn)) {
return true;
}
}
GearmanJobServerSession session = new GearmanJobServerSession(conn);
if (ioAvailable == null) {
try {
ioAvailable = Selector.open();
} catch (IOException ioe) {
LOG.warn("Failed to connect to job server " + conn + ".", ioe);
return false;
}
}
try {
session.initSession(ioAvailable, this);
} catch (IOException ioe) {
LOG.warn("Failed to initialize session with job server " + conn + ".", ioe);
return false;
}
SelectionKey key = session.getSelectionKey();
if (key == null) {
String msg =
"Session "
+ session
+ " has a null "
+ "selection key. Server will not be added to worker.";
LOG.warn(msg);
throw new IllegalStateException(msg);
}
sessionMap.put(key, session);
GearmanPacket p =
new GearmanPacketImpl(
GearmanPacketMagic.REQ, GearmanPacketType.SET_CLIENT_ID, ByteUtils.toUTF8Bytes(id));
session.submitTask(new GearmanTask(p));
for (FunctionDefinition def : functionMap.values()) {
p = generateCanDoPacket(def);
session.submitTask(new GearmanTask(p)); // NOPMD
}
p = new GearmanPacketImpl(GearmanPacketMagic.REQ, getGrabJobPacketType(), new byte[0]);
GearmanTask gsr = new GearmanTask(new GrabJobEventHandler(session), p);
taskMap.put(session, gsr);
session.submitTask(gsr);
LOG.debug("Added server " + conn + " to worker " + this);
return true;
}
private GearmanPacket generateCanDoPacket(FunctionDefinition def) {
GearmanPacketType pt = GearmanPacketType.CAN_DO;
byte[] data = null;
byte[] name = ByteUtils.toUTF8Bytes(def.getFactory().getFunctionName());
long timeout = def.getTimeout();
if (timeout > 0) {
pt = GearmanPacketType.CAN_DO_TIMEOUT;
byte[] to = ByteUtils.toUTF8Bytes(String.valueOf(timeout));
data = new byte[name.length + to.length + 1];
System.arraycopy(name, 0, data, 0, name.length);
data[name.length] = ByteUtils.NULL;
System.arraycopy(to, 0, data, name.length + 1, to.length);
} else {
data = name;
}
return new GearmanPacketImpl(GearmanPacketMagic.REQ, pt, data);
}
public void unregisterFunction(String functionName) {
functionMap.remove(functionName);
sendToAll(
new GearmanPacketImpl(
GearmanPacketMagic.REQ,
GearmanPacketType.CANT_DO,
ByteUtils.toUTF8Bytes(functionName)));
LOG.debug("Worker " + this + " has unregistered function " + functionName);
}
public void setWorkerID(String id) throws IllegalArgumentException {
if (id == null) {
throw new IllegalArgumentException("Worker ID may not be null");
}
this.id = id;
sendToAll(
new GearmanPacketImpl(
GearmanPacketMagic.REQ, GearmanPacketType.SET_CLIENT_ID, ByteUtils.toUTF8Bytes(id)));
}
public static void main(String[] args) {
if (args.length == 0 || args.length > 3) {
usage();
return;
}
String host = Constants.GEARMAN_DEFAULT_TCP_HOST;
int port = Constants.GEARMAN_DEFAULT_TCP_PORT;
byte[] payload = ByteUtils.toUTF8Bytes(args[args.length - 1]);
for (String arg : args) {
if (arg.startsWith("-h")) {
host = arg.substring(2);
} else if (arg.startsWith("-p")) {
port = Integer.parseInt(arg.substring(2));
}
}
DigestClient dc = new DigestClient(host, port);
byte[] md5 = dc.digest(payload);
System.out.println(ByteUtils.toHex(md5)); // NOPMD
dc.shutdown();
}
public byte[] echo(byte[] data) throws IOException, GearmanException {
if (!runState.equals(state.RUNNING)) {
throw new IllegalStateException(CLIENT_NOT_ACTIVE);
}
GearmanPacket echoRequest =
new GearmanPacketImpl(GearmanPacketMagic.REQ, GearmanPacketType.ECHO_REQ, data);
GearmanEchoResponseHandler handler = new GearmanEchoResponseHandler();
GearmanTask t = new GearmanTask(handler, echoRequest);
GearmanJobServerSession session = getSessionForTask();
session.submitTask(t);
LOG.info(
"Client "
+ this
+ " has submitted echo request "
+ "(payload = "
+ ByteUtils.toHex(data)
+ " to session "
+ session);
if (!driveRequestTillState(t, GearmanTask.State.FINISHED)) {
throw new GearmanException("Failed to execute echo request " + t + " to session " + session);
}
LOG.info("Client " + this + " has completed echo request " + "to session " + session);
return handler.getResults();
}
private GearmanPacketType(int i) {
type = ByteUtils.toBigEndian(i);
code = i;
}
/*
* ISSUES/RFEs
* -several of the invoke/submit/execute methods throw OpNotSupported exception
* because they rely task cancelation, which we dont support
* -shutdownNow will always return an empty set, there is a few problems here
* 1) a gearmanJob is a callable, not a runnable
* 2) we track requests, not jobs inside the session object
* -selectUpdate should provide the ability to on select on certain events
*/
public class GearmanClientImpl implements GearmanClient, GearmanSessionEventHandler {
private static enum state {
RUNNING,
SHUTTINGDOWN,
TERMINATED
}
private static final String DESCRIPION_PREFIX = "GearmanClient";
private static final org.slf4j.Logger LOG =
org.slf4j.LoggerFactory.getLogger(Constants.GEARMAN_CLIENT_LOGGER_NAME);
private static final String CLIENT_NOT_ACTIVE = "Client is not active";
private static final long DEFAULT_DRIVE_REQUEST_TIMEOUT = 2000;
private static final byte[] EXCEPTIONS = ByteUtils.toAsciiBytes("exceptions");
private final String DESCRIPTION;
private Map<SelectionKey, GearmanJobServerSession> sessionsMap = null;
private Selector ioAvailable = null;
private state runState = state.RUNNING;
private final Map<GearmanJobServerSession, Map<JobHandle, GearmanJobImpl>> sessionJobsMap;
private GearmanJobImpl jobAwatingCreation;
private final Timer timer = new Timer();
private final GearmanJobServerIpConnectionFactory connFactory =
new GearmanNIOJobServerConnectionFactory();
private final long driveRequestTimeout;
private static class Alarm extends TimerTask {
private final AtomicBoolean timesUp = new AtomicBoolean(false);
@Override
public void run() {
timesUp.set(true);
}
public boolean hasFired() {
return timesUp.get();
}
}
private static class JobHandle {
private final byte[] jobHandle;
JobHandle(GearmanJobImpl job) {
this(job.getHandle());
}
JobHandle(byte[] handle) {
jobHandle = new byte[handle.length];
System.arraycopy(handle, 0, jobHandle, 0, handle.length);
}
@Override
public boolean equals(Object that) {
if (that == null) {
return false;
}
if (!(that instanceof JobHandle)) {
return false;
}
JobHandle thatHandle = (JobHandle) that;
return Arrays.equals(jobHandle, thatHandle.jobHandle);
}
@Override
public int hashCode() {
return Arrays.hashCode(jobHandle);
}
}
/**
* Create a new GearmanClient instance. Instances are not thread-safe and should not be shared
* across threads.
*/
public GearmanClientImpl() {
this(DEFAULT_DRIVE_REQUEST_TIMEOUT);
}
public GearmanClientImpl(long driveRequestTimeout) {
if (driveRequestTimeout < 0) {
throw new IllegalArgumentException("Drive request timeout must be 0" + " or greater.");
}
sessionsMap = new Hashtable<SelectionKey, GearmanJobServerSession>();
sessionJobsMap = new HashMap<GearmanJobServerSession, Map<JobHandle, GearmanJobImpl>>();
DESCRIPTION = DESCRIPION_PREFIX + ":" + Thread.currentThread().getId();
this.driveRequestTimeout = driveRequestTimeout;
}
public boolean addJobServer(String host, int port) {
return addJobServer(host, port, true);
}
// TODO move to interface?
public boolean addJobServer(String host, int port, boolean serverForwardsExceptions) {
return addJobServer(connFactory.createConnection(host, port), serverForwardsExceptions);
}
public boolean addJobServer(GearmanJobServerConnection newconn)
throws IllegalArgumentException, IllegalStateException {
return addJobServer(newconn, true);
}
public boolean addJobServer(GearmanJobServerConnection newconn, boolean serverForwardsExceptions)
throws IllegalArgumentException, IllegalStateException {
// TODO remove this restriction
if (!(newconn instanceof GearmanNIOJobServerConnection)) {
throw new IllegalArgumentException(
"Client currently only "
+ "supports "
+ GearmanNIOJobServerConnection.class.getName()
+ " connections.");
}
GearmanNIOJobServerConnection conn = (GearmanNIOJobServerConnection) newconn;
if (!runState.equals(state.RUNNING)) {
throw new RejectedExecutionException("Client has been shutdown");
}
GearmanJobServerSession session = new GearmanJobServerSession(conn);
if (sessionsMap.values().contains(session)) {
LOG.debug(
"The server "
+ newconn
+ " was previously "
+ "added to the client. Ignoring add request.");
return true;
}
try {
if (ioAvailable == null) {
ioAvailable = Selector.open();
}
session.initSession(ioAvailable, this);
SelectionKey key = session.getSelectionKey();
sessionsMap.put(key, session);
} catch (IOException ioe) {
LOG.warn("Failed to connect to job server " + newconn + ".", ioe);
return false;
}
if (serverForwardsExceptions && !setForwardExceptions(session)) {
return false;
}
sessionJobsMap.put(session, new HashMap<JobHandle, GearmanJobImpl>());
LOG.info("Added connection " + conn + " to client " + this);
return true;
}
public boolean hasConnection(GearmanJobServerConnection conn) {
for (GearmanJobServerSession sess : sessionsMap.values()) {
if (sess.getConnection().equals(conn)) {
return true;
}
}
return false;
}
public List<GearmanJobServerConnection> getSetOfJobServers() throws IllegalStateException {
if (!runState.equals(state.RUNNING)) {
throw new IllegalStateException(CLIENT_NOT_ACTIVE);
}
ArrayList<GearmanJobServerConnection> retSet = new ArrayList<GearmanJobServerConnection>();
for (GearmanJobServerSession sess : sessionsMap.values()) {
retSet.add(sess.getConnection());
}
return retSet;
}
public void removeJobServer(GearmanJobServerConnection conn)
throws IllegalArgumentException, IllegalStateException {
if (!runState.equals(state.RUNNING)) {
throw new IllegalStateException(
"JobServers can not be removed " + "once shutdown has been commenced.");
}
// TODO, make this better
Iterator<GearmanJobServerSession> iter = sessionsMap.values().iterator();
GearmanJobServerSession session = null;
boolean foundit = false;
while (iter.hasNext() && !foundit) {
session = iter.next();
if (session.getConnection().equals(conn)) {
foundit = true;
}
}
if (!foundit) {
throw new IllegalArgumentException(
"JobServer " + conn + " has not" + " been registered with this client.");
}
shutDownSession(session);
LOG.info("Removed job server " + conn + " from client " + this);
}
public <T> Future<T> submit(Callable<T> task) {
if (task == null) {
throw new IllegalStateException("Null task was submitted to " + "gearman client");
}
if (!runState.equals(state.RUNNING)) {
throw new RejectedExecutionException("Client has been shutdown");
}
if (!(task instanceof GearmanServerResponseHandler)) {
throw new RejectedExecutionException(
"Task must implement the "
+ GearmanServerResponseHandler.class
+ " interface to"
+ " submitted to this client");
}
GearmanJobImpl job = (GearmanJobImpl) task;
GearmanServerResponseHandler handler = (GearmanServerResponseHandler) job;
if (job.isDone()) {
throw new RejectedExecutionException("Task can not be resubmitted ");
}
GearmanJobServerSession session = null;
try {
session = getSessionForTask();
} catch (IOException ioe) {
throw new RejectedExecutionException(ioe);
}
job.setJobServerSession(session);
GearmanPacket submitRequest = getPacketFromJob(job);
GearmanTask submittedJob = new GearmanTask(handler, submitRequest);
session.submitTask(submittedJob);
LOG.info(
"Client "
+ this
+ " has submitted job "
+ job
+ // NOPMD
" to session "
+ session
+ ". Job has been added to the "
+ // NOPMD
"active job queue");
try {
jobAwatingCreation = job;
if (!(driveRequestTillState(submittedJob, GearmanTask.State.RUNNING))) {
throw new RejectedExecutionException(
"Timed out waiting for" + " submission of " + job + " to complete");
}
} catch (IOException ioe) {
LOG.warn("Client " + this + " encounted an " + "IOException while drivingIO", ioe);
throw new RejectedExecutionException(
"Failed to successfully " + "submit" + job + " due to IOException", ioe);
} finally {
jobAwatingCreation = null;
}
return (Future<T>) job;
}
public <T> Future<T> submit(Runnable task, T result) {
throw new UnsupportedOperationException(
"Client does not support " + "execution of non-GearmanJob objects");
}
public Future<?> submit(Runnable task) {
throw new UnsupportedOperationException(
"Client does not support " + "execution of non-GearmanJob objects");
}
public void execute(Runnable command) {
throw new UnsupportedOperationException(
"Client does not support " + "execution of non-GearmanJob objects");
}
// NOTE, there is a subtle difference between the ExecutorService invoke*
// method signatures in jdk1.5 and jdk1.6 that requires we implement these
// methods in their 'erasure' format (that is without the use of the
// specified generic types), otherwise we will not be able to compile this
// class using both compilers.
@SuppressWarnings("unchecked") // NOPMD
public List invokeAll(Collection tasks) throws InterruptedException {
ArrayList<Future> futures = new ArrayList<Future>();
Iterator<Callable<Future>> iter = tasks.iterator();
while (iter.hasNext()) {
Callable<Future> curTask = iter.next();
futures.add(this.submit(curTask));
}
for (Future results : futures) {
try {
results.get();
} catch (ExecutionException ee) {
LOG.warn("Failed to execute task " + results + ".", ee);
}
}
return futures;
}
@SuppressWarnings("unchecked")
public List invokeAll(Collection tasks, long timeout, TimeUnit unit) throws InterruptedException {
throw new UnsupportedOperationException("Not supported yet.");
}
@SuppressWarnings("unchecked")
public Future invokeAny(Collection tasks) throws InterruptedException, ExecutionException {
throw new UnsupportedOperationException("Not supported yet.");
}
@SuppressWarnings("unchecked")
public Future invokeAny(Collection tasks, long timeout, TimeUnit unit)
throws InterruptedException, ExecutionException, TimeoutException {
throw new UnsupportedOperationException("Not supported yet.");
}
public GearmanJobStatus getJobStatus(GearmanJob job)
throws IOException, GearmanException, IllegalStateException {
// we need to know what session to ask that information can not be
// gleened from the job object.
if (!(job instanceof GearmanJobImpl)) {
throw new IllegalArgumentException("job must be of type " + GearmanJobImpl.class);
}
GearmanJobImpl jobImpl = (GearmanJobImpl) job;
return updateJobStatus(jobImpl.getHandle(), jobImpl.getSession());
}
public byte[] echo(byte[] data) throws IOException, GearmanException {
if (!runState.equals(state.RUNNING)) {
throw new IllegalStateException(CLIENT_NOT_ACTIVE);
}
GearmanPacket echoRequest =
new GearmanPacketImpl(GearmanPacketMagic.REQ, GearmanPacketType.ECHO_REQ, data);
GearmanEchoResponseHandler handler = new GearmanEchoResponseHandler();
GearmanTask t = new GearmanTask(handler, echoRequest);
GearmanJobServerSession session = getSessionForTask();
session.submitTask(t);
LOG.info(
"Client "
+ this
+ " has submitted echo request "
+ "(payload = "
+ ByteUtils.toHex(data)
+ " to session "
+ session);
if (!driveRequestTillState(t, GearmanTask.State.FINISHED)) {
throw new GearmanException("Failed to execute echo request " + t + " to session " + session);
}
LOG.info("Client " + this + " has completed echo request " + "to session " + session);
return handler.getResults();
}
public int getNumberofActiveJobs() throws IllegalStateException {
if (runState.equals(state.TERMINATED)) {
throw new IllegalStateException(CLIENT_NOT_ACTIVE);
}
int size = 0;
for (Map<JobHandle, GearmanJobImpl> cur : sessionJobsMap.values()) {
size += cur.size();
}
return size;
}
public void handleSessionEvent(GearmanSessionEvent event)
throws IllegalArgumentException, IllegalStateException {
GearmanPacket p = event.getPacket();
GearmanJobServerSession s = event.getSession();
GearmanPacketType t = p.getPacketType();
Map<JobHandle, GearmanJobImpl> jobsMaps = sessionJobsMap.get(s);
switch (t) {
case JOB_CREATED:
if (jobAwatingCreation == null) {
throw new IllegalStateException(
"Recevied job creation " + "message but have not job awaiting submission.");
}
if (!jobAwatingCreation.isBackgroundJob()) {
jobsMaps.put(new JobHandle(jobAwatingCreation), jobAwatingCreation);
}
break;
case WORK_DATA:
case WORK_STATUS:
case WORK_WARNING:
case WORK_COMPLETE:
case WORK_FAIL:
case WORK_EXCEPTION:
JobHandle handle =
new JobHandle(p.getDataComponentValue(GearmanPacket.DataComponentName.JOB_HANDLE));
GearmanJobImpl job = jobsMaps.get(handle);
if (job == null) {
LOG.warn(
"Client received packet from server"
+ " for unknown job ( job_handle = "
+ handle
+ " packet = "
+ t
+ " )");
} else {
job.handleEvent(p);
if (job.isDone()) {
jobsMaps.remove(handle);
}
}
break;
case ERROR:
String errCode =
ByteUtils.fromUTF8Bytes(
p.getDataComponentValue(GearmanPacket.DataComponentName.ERROR_CODE));
String errMsg =
ByteUtils.fromUTF8Bytes(
p.getDataComponentValue(GearmanPacket.DataComponentName.ERROR_TEXT));
LOG.warn(
"Received error code "
+ errCode
+ "( "
+ errMsg
+ " )"
+ " from session "
+ s
+ ". Shutting session down");
shutDownSession(s);
if (sessionsMap.isEmpty()) {
shutdown();
}
break;
default:
LOG.warn(
"received un-expected packet from Job"
+ " Server Session: "
+ p
+ ". Shutting down session");
shutDownSession(s);
if (sessionsMap.isEmpty()) {
shutdown();
}
}
}
public void shutdown() {
if (!runState.equals(state.RUNNING)) {
return;
}
runState = state.SHUTTINGDOWN;
LOG.info("Commencing controlled shutdown of client: " + this);
try {
awaitTermination(-1, TimeUnit.SECONDS);
} catch (InterruptedException ie) {
LOG.info("Client shutdown interrupted while waiting" + " for jobs to terminate.");
}
shutdownNow();
LOG.info("Completed ontrolled shutdown of client: " + this);
}
public List<Runnable> shutdownNow() {
runState = state.SHUTTINGDOWN;
LOG.info("Commencing immediate shutdown of client: " + this);
timer.cancel();
Iterator<GearmanJobServerSession> sessions = sessionsMap.values().iterator();
while (sessions.hasNext()) {
GearmanJobServerSession curSession = sessions.next();
if (!curSession.isInitialized()) {
continue;
}
try {
curSession.closeSession();
} catch (Exception e) {
LOG.warn(
"Failed to closes session "
+ curSession
+ " while performing immediate shutdown of client "
+ this
+ ". Encountered the following exception "
+ e);
}
sessions.remove();
}
sessionsMap.clear();
sessionsMap = null;
runState = state.TERMINATED;
try {
ioAvailable.close();
} catch (IOException ioe) {
LOG.warn("Encountered IOException while closing selector for client ", ioe);
}
LOG.info("Completed shutdown of client: " + this);
return new ArrayList<Runnable>();
}
public boolean isShutdown() {
return !runState.equals(state.RUNNING);
}
public boolean isTerminated() {
return runState.equals(state.TERMINATED);
}
public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
TimeUnit sessionUnit = TimeUnit.MILLISECONDS;
long timeLeft = -1;
long timeOutInMills =
timeout < 0
? -1
: TimeUnit.MILLISECONDS.convert(timeout, unit) + System.currentTimeMillis();
if (getNumberofActiveJobs() == 0) {
return true;
}
for (GearmanJobServerSession curSession : sessionsMap.values()) {
if (!(curSession.isInitialized())) {
continue;
}
// negative timeout means block till completion
if (timeout >= 0) {
timeLeft = timeOutInMills - System.currentTimeMillis();
if (timeLeft <= 0) {
LOG.warn("awaitTermination exceeded timeout.");
break;
}
}
try {
curSession.waitForTasksToComplete(timeLeft, sessionUnit);
} catch (TimeoutException te) {
LOG.info("timed out waiting for all tasks to complete");
break;
}
}
return getNumberofActiveJobs() == 0;
}
@Override
public String toString() {
return DESCRIPTION;
}
private void driveClientIO() throws IOException, GearmanException {
for (GearmanJobServerSession sess : sessionsMap.values()) {
int interestOps = SelectionKey.OP_READ;
if (sess.sessionHasDataToWrite()) {
interestOps |= SelectionKey.OP_WRITE;
}
try {
sess.getSelectionKey().interestOps(interestOps);
} catch (IllegalStateException ise) {
LOG.warn("Unable to drive IO for session " + sess + "," + " skipping.", ise);
continue;
}
}
ioAvailable.selectNow();
Set<SelectionKey> keys = ioAvailable.selectedKeys();
LOG.trace(
"Driving IO for client "
+ this
+ ". "
+ keys.size()
+ " session(s) currently available for IO");
Iterator<SelectionKey> iter = keys.iterator();
while (iter.hasNext()) {
SelectionKey key = iter.next();
GearmanJobServerSession s = sessionsMap.get(key);
s.driveSessionIO();
}
}
private GearmanJobStatus updateJobStatus(byte[] jobhandle, GearmanJobServerSession session)
throws IOException, IllegalStateException, GearmanException {
if (!runState.equals(state.RUNNING)) {
throw new IllegalStateException(CLIENT_NOT_ACTIVE);
}
if (jobhandle == null || jobhandle.length == 0) {
throw new IllegalStateException("Invalid job handle. Handle must" + " not be null nor empty");
}
GearmanPacket statusRequest =
new GearmanPacketImpl(GearmanPacketMagic.REQ, GearmanPacketType.GET_STATUS, jobhandle);
GearmanServerResponseHandler handler =
(GearmanServerResponseHandler) new GearmanJobStatusImpl();
GearmanTask t = new GearmanTask(handler, statusRequest);
session.submitTask(t);
if (!driveRequestTillState(t, GearmanTask.State.FINISHED)) {
throw new GearmanException(
"Failed to execute jobstatus request " + t + " to session " + session);
}
return (GearmanJobStatus) handler;
}
private GearmanJobServerSession getSessionForTask() throws IOException {
if (sessionsMap.values().isEmpty()) {
throw new IOException("No servers registered with client");
}
ArrayList<GearmanJobServerSession> sessions = new ArrayList<GearmanJobServerSession>();
sessions.addAll(sessionsMap.values());
Random rand = new Random(System.currentTimeMillis());
int s = rand.nextInt(sessions.size());
GearmanJobServerSession session = sessions.get(s);
IOException ioe = new IOException("No servers available for client");
for (int i = 0; i < sessions.size(); i++) {
if (!session.isInitialized()) {
try {
session.initSession(ioAvailable, this);
sessionsMap.put(session.getSelectionKey(), session);
} catch (Exception e) {
if (sessions.size() > 1) {
// try next one
int prev = s;
s = (s + 1) % sessions.size();
session = sessions.get(s);
LOG.warn(
"Got exception attempting to retrieve "
+ "session for task. Try next server at "
+ "pos={}, previous pos={}",
s,
prev);
continue;
} else {
break;
}
}
}
return session;
}
throw ioe;
}
private boolean driveRequestTillState(GearmanTask r, GearmanTask.State state)
throws IOException, GearmanException {
Alarm alarm = new Alarm();
timer.schedule(alarm, driveRequestTimeout);
while (r.getState().compareTo(state) < 0 && !(alarm.hasFired())) {
driveClientIO();
}
return r.getState().compareTo(state) >= 0;
}
private GearmanPacket getPacketFromJob(GearmanJob job) {
int destPos = 0;
GearmanPacketMagic magic = GearmanPacketMagic.REQ;
GearmanPacketType type = null;
byte[] packetdata = null;
byte[] fnname = ByteUtils.toAsciiBytes(job.getFunctionName());
byte[] uid = job.getID();
byte[] data = job.getData();
if (job.getPriority().equals(GearmanJob.JobPriority.HIGH)) {
type =
job.isBackgroundJob()
? GearmanPacketType.SUBMIT_JOB_HIGH_BG
: GearmanPacketType.SUBMIT_JOB_HIGH;
}
if (job.getPriority().equals(GearmanJob.JobPriority.LOW)) {
type =
job.isBackgroundJob()
? GearmanPacketType.SUBMIT_JOB_LOW_BG
: GearmanPacketType.SUBMIT_JOB_LOW;
}
if (job.getPriority().equals(GearmanJob.JobPriority.NORMAL)) {
type = job.isBackgroundJob() ? GearmanPacketType.SUBMIT_JOB_BG : GearmanPacketType.SUBMIT_JOB;
}
packetdata = new byte[fnname.length + uid.length + data.length + 2];
System.arraycopy(fnname, 0, packetdata, destPos, fnname.length);
destPos += fnname.length;
packetdata[destPos++] = ByteUtils.NULL;
System.arraycopy(uid, 0, packetdata, destPos, uid.length);
destPos += uid.length;
packetdata[destPos++] = ByteUtils.NULL;
System.arraycopy(data, 0, packetdata, destPos, data.length);
return new GearmanPacketImpl(magic, type, packetdata);
}
private void shutDownSession(GearmanJobServerSession s) {
if (s.isInitialized()) {
SelectionKey k = s.getSelectionKey();
if (k != null) {
sessionsMap.remove(k);
k.cancel();
}
s.closeSession();
}
sessionJobsMap.remove(s);
}
private boolean setForwardExceptions(GearmanJobServerSession session) {
GearmanServerResponseHandler optionReqHandler =
new GearmanServerResponseHandler() {
boolean isDone = false;
public boolean isDone() {
return isDone;
}
public void handleEvent(GearmanPacket event) throws GearmanException {
GearmanPacketType type = event.getPacketType();
if (type.equals(GearmanPacketType.OPTION_RES)
&& Arrays.equals(EXCEPTIONS, event.getData())) {
isDone = true;
}
}
};
GearmanTask setExceptionsTask =
new GearmanTask(
optionReqHandler,
new GearmanPacketImpl(
GearmanPacketMagic.REQ, GearmanPacketType.OPTION_REQ, EXCEPTIONS));
session.submitTask(setExceptionsTask);
Exception e = null;
boolean success = false;
try {
success = driveRequestTillState(setExceptionsTask, GearmanTask.State.FINISHED);
} catch (IOException ioe) {
e = ioe;
}
if (!success) {
if (e != null) {
LOG.info("Failed to set forward-exceptions option to " + session.getConnection(), e);
} else {
LOG.info("Failed to set forward-exceptions option to " + session.getConnection());
}
return false;
}
return true;
}
}
public void handleSessionEvent(GearmanSessionEvent event)
throws IllegalArgumentException, IllegalStateException {
GearmanPacket p = event.getPacket();
GearmanJobServerSession s = event.getSession();
GearmanPacketType t = p.getPacketType();
Map<JobHandle, GearmanJobImpl> jobsMaps = sessionJobsMap.get(s);
switch (t) {
case JOB_CREATED:
if (jobAwatingCreation == null) {
throw new IllegalStateException(
"Recevied job creation " + "message but have not job awaiting submission.");
}
if (!jobAwatingCreation.isBackgroundJob()) {
jobsMaps.put(new JobHandle(jobAwatingCreation), jobAwatingCreation);
}
break;
case WORK_DATA:
case WORK_STATUS:
case WORK_WARNING:
case WORK_COMPLETE:
case WORK_FAIL:
case WORK_EXCEPTION:
JobHandle handle =
new JobHandle(p.getDataComponentValue(GearmanPacket.DataComponentName.JOB_HANDLE));
GearmanJobImpl job = jobsMaps.get(handle);
if (job == null) {
LOG.warn(
"Client received packet from server"
+ " for unknown job ( job_handle = "
+ handle
+ " packet = "
+ t
+ " )");
} else {
job.handleEvent(p);
if (job.isDone()) {
jobsMaps.remove(handle);
}
}
break;
case ERROR:
String errCode =
ByteUtils.fromUTF8Bytes(
p.getDataComponentValue(GearmanPacket.DataComponentName.ERROR_CODE));
String errMsg =
ByteUtils.fromUTF8Bytes(
p.getDataComponentValue(GearmanPacket.DataComponentName.ERROR_TEXT));
LOG.warn(
"Received error code "
+ errCode
+ "( "
+ errMsg
+ " )"
+ " from session "
+ s
+ ". Shutting session down");
shutDownSession(s);
if (sessionsMap.isEmpty()) {
shutdown();
}
break;
default:
LOG.warn(
"received un-expected packet from Job"
+ " Server Session: "
+ p
+ ". Shutting down session");
shutDownSession(s);
if (sessionsMap.isEmpty()) {
shutdown();
}
}
}
public void work() {
if (!state.equals(State.IDLE)) {
throw new IllegalStateException(
"Can not call work while worker " + "is running or shutting down");
}
state = State.RUNNING;
// a map keeping track of sessions with connection errors
// (to avoid printing an error about them in every reconnect attempt)
Map<GearmanJobServerSession, Boolean> havingConnectionError =
new HashMap<GearmanJobServerSession, Boolean>();
while (isRunning()) {
// look for sessions which have been disconnected and attempt to reconnect.
for (Iterator<GearmanJobServerSession> iter = sessionMap.values().iterator();
iter.hasNext(); ) {
GearmanJobServerSession sess = iter.next();
if (!sess.isInitialized()) {
try {
// reconnect, unregister old selection key and register new one
SelectionKey oldKey = sess.isInitialized() ? sess.getSelectionKey() : null;
sess.initSession(ioAvailable, this);
if (oldKey != null) {
iter.remove();
}
sessionMap.put(sess.getSelectionKey(), sess);
// register all functions with the newly reconnected server
for (FunctionDefinition d : functionMap.values()) {
GearmanTask gsr = new GearmanTask(null, generateCanDoPacket(d));
sess.submitTask(gsr);
}
GearmanPacket p =
new GearmanPacketImpl(
GearmanPacketMagic.REQ,
GearmanPacketType.SET_CLIENT_ID,
ByteUtils.toUTF8Bytes(id));
sess.submitTask(new GearmanTask(p));
GearmanTask sessTask =
new GearmanTask(
new GrabJobEventHandler(sess),
new GearmanPacketImpl(
GearmanPacketMagic.REQ, getGrabJobPacketType(), new byte[0]));
sess.submitTask(sessTask);
sess.driveSessionIO();
// log reconnection message
if (havingConnectionError.get(sess)) {
LOG.info("Re-established connection to " + sess.getConnection().toString());
}
havingConnectionError.put(sess, false);
} catch (IOException e) {
if (!havingConnectionError.get(sess)) {
LOG.warn("Error connecting to " + sess + ", will keep trying..");
}
havingConnectionError.put(sess, true);
try {
Thread.sleep(50);
} catch (InterruptedException e1) {
}
}
} else {
havingConnectionError.put(sess, false);
}
}
for (GearmanJobServerSession sess : sessionMap.values()) {
// if still disconnected, skip
if (!sess.isInitialized()) {
continue;
}
int interestOps = SelectionKey.OP_READ;
if (sess.sessionHasDataToWrite()) {
interestOps |= SelectionKey.OP_WRITE;
}
sess.getSelectionKey().interestOps(interestOps);
}
try {
ioAvailable.select(1);
} catch (IOException io) {
LOG.warn("Receieved IOException while" + " selecting for IO", io);
}
for (SelectionKey key : ioAvailable.selectedKeys()) {
GearmanJobServerSession sess = sessionMap.get(key);
if (sess == null) {
LOG.warn("Worker does not have " + "session for key " + key);
continue;
}
if (!sess.isInitialized()) {
continue;
}
try {
GearmanTask sessTask = taskMap.get(sess);
if (sessTask == null) {
sessTask =
new GearmanTask( // NOPMD
new GrabJobEventHandler(sess),
new GearmanPacketImpl(
GearmanPacketMagic.REQ, getGrabJobPacketType(), new byte[0]));
taskMap.put(sess, sessTask);
sess.submitTask(sessTask);
LOG.debug(
"Worker: "
+ this
+ " submitted a "
+ sessTask.getRequestPacket().getPacketType()
+ " to session: "
+ sess);
}
sess.driveSessionIO();
// For the time being we will execute the jobs synchronously
// in the future, I expect to change this.
if (!functionList.isEmpty()) {
GearmanFunction fun = functionList.remove();
submitFunction(fun);
statistics();
}
} catch (IOException ioe) {
LOG.warn("Received IOException while driving" + " IO on session " + sess, ioe);
sess.closeSession();
continue;
}
}
}
shutDownWorker(true);
}
private GearmanPacketMagic(String kind) {
this.name = new byte[1 + kind.length()];
this.name[0] = 0;
byte[] bytes = ByteUtils.toUTF8Bytes(kind);
System.arraycopy(bytes, 0, this.name, 1, kind.length());
}
