/**
* Get the protocol name. If the protocol class has a ProtocolAnnotation, then get the protocol
* name from the annotation; otherwise the class name is the protocol name.
*/
public static String getProtocolName(Class<?> protocol) {
if (protocol == null) {
return null;
}
ProtocolInfo anno = protocol.getAnnotation(ProtocolInfo.class);
return (anno == null) ? protocol.getName() : anno.protocolName();
}
/**
* Construct an RPC server.
*
* @param protocolClass - the protocol being registered can be null for compatibility with old
* usage (see below for details)
* @param protocolImpl the protocol impl that will be called
* @param conf the configuration to use
* @param bindAddress the address to bind on to listen for connection
* @param port the port to listen for connections on
* @param numHandlers the number of method handler threads to run
* @param verbose whether each call should be logged
*/
public Server(
Class<?> protocolClass,
Object protocolImpl,
Configuration conf,
String bindAddress,
int port,
int numHandlers,
int numReaders,
int queueSizePerHandler,
boolean verbose,
SecretManager<? extends TokenIdentifier> secretManager,
String portRangeConfig)
throws IOException {
super(
bindAddress,
port,
null,
numHandlers,
numReaders,
queueSizePerHandler,
conf,
classNameBase(protocolImpl.getClass().getName()),
secretManager,
portRangeConfig);
this.verbose = verbose;
Class<?>[] protocols;
if (protocolClass == null) { // derive protocol from impl
/*
* In order to remain compatible with the old usage where a single
* target protocolImpl is suppled for all protocol interfaces, and
* the protocolImpl is derived from the protocolClass(es)
* we register all interfaces extended by the protocolImpl
*/
protocols = RPC.getProtocolInterfaces(protocolImpl.getClass());
} else {
if (!protocolClass.isAssignableFrom(protocolImpl.getClass())) {
throw new IOException(
"protocolClass "
+ protocolClass
+ " is not implemented by protocolImpl which is of class "
+ protocolImpl.getClass());
}
// register protocol class and its super interfaces
registerProtocolAndImpl(RPC.RpcKind.RPC_WRITABLE, protocolClass, protocolImpl);
protocols = RPC.getProtocolInterfaces(protocolClass);
}
for (Class<?> p : protocols) {
if (!p.equals(VersionedProtocol.class)) {
registerProtocolAndImpl(RPC.RpcKind.RPC_WRITABLE, p, protocolImpl);
}
}
}
// Register protocol and its impl for rpc calls
void registerProtocolAndImpl(RpcKind rpcKind, Class<?> protocolClass, Object protocolImpl) {
String protocolName = RPC.getProtocolName(protocolClass);
long version;
try {
version = RPC.getProtocolVersion(protocolClass);
} catch (Exception ex) {
LOG.warn("Protocol " + protocolClass + " NOT registered as cannot get protocol version ");
return;
}
getProtocolImplMap(rpcKind)
.put(
new ProtoNameVer(protocolName, version),
new ProtoClassProtoImpl(protocolClass, protocolImpl));
LOG.debug(
"RpcKind = "
+ rpcKind
+ " Protocol Name = "
+ protocolName
+ " version="
+ version
+ " ProtocolImpl="
+ protocolImpl.getClass().getName()
+ " protocolClass="
+ protocolClass.getName());
}
/**
* Get all superInterfaces that extend VersionedProtocol
*
* @param childInterfaces
* @return the super interfaces that extend VersionedProtocol
*/
static Class<?>[] getSuperInterfaces(Class<?>[] childInterfaces) {
List<Class<?>> allInterfaces = new ArrayList<Class<?>>();
for (Class<?> childInterface : childInterfaces) {
if (VersionedProtocol.class.isAssignableFrom(childInterface)) {
allInterfaces.add(childInterface);
allInterfaces.addAll(Arrays.asList(getSuperInterfaces(childInterface.getInterfaces())));
} else {
LOG.warn(
"Interface "
+ childInterface
+ " ignored because it does not extend VersionedProtocol");
}
}
return allInterfaces.toArray(new Class[allInterfaces.size()]);
}
public Writable call(Class<?> protocol, Writable param, long receivedTime) throws IOException {
try {
Invocation call = (Invocation) param;
if (verbose) log("Call: " + call);
Method method = protocol.getMethod(call.getMethodName(), call.getParameterClasses());
method.setAccessible(true);
int qTime = (int) (System.currentTimeMillis() - receivedTime);
long startNanoTime = System.nanoTime();
Object value = method.invoke(instance, call.getParameters());
long processingMicroTime = (System.nanoTime() - startNanoTime) / 1000;
if (LOG.isDebugEnabled()) {
LOG.debug(
"Served: "
+ call.getMethodName()
+ " queueTime (millisec)= "
+ qTime
+ " procesingTime (microsec)= "
+ processingMicroTime);
}
rpcMetrics.rpcQueueTime.inc(qTime);
rpcMetrics.rpcProcessingTime.inc(processingMicroTime);
MetricsTimeVaryingRate m =
(MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName());
if (m == null) {
try {
m = new MetricsTimeVaryingRate(call.getMethodName(), rpcMetrics.registry);
} catch (IllegalArgumentException iae) {
// the metrics has been registered; re-fetch the handle
LOG.debug("Error register " + call.getMethodName(), iae);
m = (MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName());
}
}
// record call time in microseconds
m.inc(processingMicroTime);
if (verbose) log("Return: " + value);
return new ObjectWritable(method.getReturnType(), value);
} catch (InvocationTargetException e) {
Throwable target = e.getTargetException();
if (target instanceof IOException) {
throw (IOException) target;
} else {
IOException ioe = new IOException(target.toString());
ioe.setStackTrace(target.getStackTrace());
throw ioe;
}
} catch (Throwable e) {
if (!(e instanceof IOException)) {
LOG.error("Unexpected throwable object ", e);
}
IOException ioe = new IOException(e.toString());
ioe.setStackTrace(e.getStackTrace());
throw ioe;
}
}
/**
* Construct a client-side proxy object that implements the named protocol, talking to a server at
* the named address.
*
* @param <T>
*/
@Override
@SuppressWarnings("unchecked")
public <T> ProtocolProxy<T> getProxy(
Class<T> protocol,
long clientVersion,
InetSocketAddress addr,
UserGroupInformation ticket,
Configuration conf,
SocketFactory factory,
int rpcTimeout,
RetryPolicy connectionRetryPolicy)
throws IOException {
if (connectionRetryPolicy != null) {
throw new UnsupportedOperationException(
"Not supported: connectionRetryPolicy=" + connectionRetryPolicy);
}
T proxy =
(T)
Proxy.newProxyInstance(
protocol.getClassLoader(),
new Class[] {protocol},
new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
return new ProtocolProxy<T>(protocol, proxy, true);
}
// return the RpcEngine configured to handle a protocol
static synchronized RpcEngine getProtocolEngine(Class<?> protocol, Configuration conf) {
RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
if (engine == null) {
Class<?> impl =
conf.getClass(ENGINE_PROP + "." + protocol.getName(), WritableRpcEngine.class);
engine = (RpcEngine) ReflectionUtils.newInstance(impl, conf);
PROTOCOL_ENGINES.put(protocol, engine);
}
return engine;
}
/**
* Get the protocol version from protocol class. If the protocol class has a ProtocolAnnotation,
* then get the protocol name from the annotation; otherwise the class name is the protocol name.
*/
public static long getProtocolVersion(Class<?> protocol) {
if (protocol == null) {
throw new IllegalArgumentException("Null protocol");
}
long version;
ProtocolInfo anno = protocol.getAnnotation(ProtocolInfo.class);
if (anno != null) {
version = anno.protocolVersion();
if (version != -1) return version;
}
try {
Field versionField = protocol.getField("versionID");
versionField.setAccessible(true);
return versionField.getLong(protocol);
} catch (NoSuchFieldException ex) {
throw new RuntimeException(ex);
} catch (IllegalAccessException ex) {
throw new RuntimeException(ex);
}
}
private Method getEvaluateMethod(TajoDataTypes.DataType[] tajoParamTypes, Class<?> clazz) {
Constructor constructor = clazz.getConstructors()[0];
try {
instance = constructor.newInstance();
} catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
throw new TajoInternalError(e);
}
for (Method m : clazz.getMethods()) {
if (m.getName().equals("evaluate")) {
Class[] methodParamTypes = m.getParameterTypes();
if (checkParamTypes(methodParamTypes, tajoParamTypes)) {
return m;
}
}
}
throw new TajoInternalError(
new UndefinedFunctionException(String.format("Hive UDF (%s)", clazz.getSimpleName())));
}
/**
* Construct a client-side proxy that implements the named protocol, talking to a server at the
* named address.
*
* @param protocol protocol
* @param clientVersion client's version
* @param addr server address
* @param ticket security ticket
* @param conf configuration
* @param factory socket factory
* @param rpcTimeout max time for each rpc; 0 means no timeout
* @return the proxy
* @throws IOException if any error occurs
*/
@SuppressWarnings("unchecked")
public static <T extends VersionedProtocol> ProtocolProxy<T> getProtocolProxy(
Class<T> protocol,
long clientVersion,
InetSocketAddress addr,
UserGroupInformation ticket,
Configuration conf,
SocketFactory factory,
int rpcTimeout)
throws IOException {
T proxy =
(T)
Proxy.newProxyInstance(
protocol.getClassLoader(),
new Class[] {protocol},
new Invoker(addr, ticket, conf, factory, rpcTimeout, protocol));
String protocolName = protocol.getName();
try {
ProtocolSignature serverInfo =
proxy.getProtocolSignature(
protocolName, clientVersion, ProtocolSignature.getFingerprint(protocol.getMethods()));
return new ProtocolProxy<T>(protocol, proxy, serverInfo.getMethods());
} catch (RemoteException re) {
IOException ioe = re.unwrapRemoteException(IOException.class);
if (ioe.getMessage()
.startsWith(IOException.class.getName() + ": " + NoSuchMethodException.class.getName())) {
// Method getProtocolSignature not supported
long serverVersion = proxy.getProtocolVersion(protocol.getName(), clientVersion);
if (serverVersion == clientVersion) {
return new ProtocolProxy<T>(protocol, proxy, null);
}
throw new VersionMismatch(protocolName, clientVersion, serverVersion, proxy);
}
throw re;
}
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
id = in.readInt();
String clsName = in.readUTF();
if (clsName == null) innerSplit = in.readObject();
else {
// Split wrapper only used when classes available in our classpath, so Class.forName is ok
// here.
Class<Writable> cls = (Class<Writable>) Class.forName(clsName);
try {
innerSplit = U.newInstance(cls);
} catch (GridException e) {
throw new IOException(e);
}
((Writable) innerSplit).readFields(in);
}
}
/**
* Set a protocol to use a non-default RpcEngine.
*
* @param conf configuration to use
* @param protocol the protocol interface
* @param engine the RpcEngine impl
*/
public static void setProtocolEngine(Configuration conf, Class<?> protocol, Class<?> engine) {
conf.setClass(ENGINE_PROP + "." + protocol.getName(), engine, RpcEngine.class);
}
/**
* Get all interfaces that the given protocol implements or extends which are assignable from
* VersionedProtocol.
*/
static Class<?>[] getProtocolInterfaces(Class<?> protocol) {
Class<?>[] interfaces = protocol.getInterfaces();
return getSuperInterfaces(interfaces);
}
// Information needed to get a single file:
// BASE_PATH, FILE_ID, TIMESTAMP_START, TIMESTAMP_STOP, SOURCE, FILESYSTEM
private static Vector<Path> getFile(
FileSystem fs, Hashtable<String, String> config, dbutil db_util) throws Exception {
Long latestVersion = latestVersion(config, db_util);
try {
config.put("timestamp_start", config.get("timestamp_start"));
config.put("timestamp_real", latestVersion.toString());
config.put("timestamp_stop", latestVersion.toString());
} catch (Exception E) {
logger.error("Tryign to get file that is impossible to generate: " + getFullPath(config));
return null;
}
if (Integer.parseInt(config.get("timestamp_start"))
> Integer.parseInt(config.get("timestamp_stop"))) {
return null;
}
logger.debug(
"Getting DB for timestamp "
+ config.get("timestamp_start")
+ " to "
+ config.get("timestamp_stop"));
String final_result = getFullPath(config);
String temp_path_base =
config.get("local_temp_path")
+ "_"
+ config.get("task_id")
+ "_"
+ config.get("run_id")
+ "/";
Path newPath = new Path(final_result + "*");
Vector<Path> ret_path = new Vector<Path>();
String lockName = lock(final_result.replaceAll("/", "_"));
if (fs.globStatus(newPath).length != 0) {
ret_path.add(newPath);
unlock(lockName);
config.put("full_file_name", final_result);
return ret_path;
} else {
if (!config.get("source").equals("local")) {
config.put("temp_path_base", temp_path_base);
config.put("timestamp_start", config.get("timestamp_start"));
config.put("timestamp_real", latestVersion.toString());
config.put("timestamp_stop", latestVersion.toString());
Class<?> sourceClass =
Class.forName("org.gestore.plugin.source." + config.get("source") + "Source");
Method process_data = sourceClass.getMethod("process", Hashtable.class, FileSystem.class);
Object processor = sourceClass.newInstance();
Object retVal;
try {
retVal = process_data.invoke(processor, config, fs);
} catch (InvocationTargetException E) {
Throwable exception = E.getTargetException();
logger.error("Unable to call method in child class: " + exception.toString());
exception.printStackTrace(System.out);
unlock(lockName);
return null;
}
FileStatus[] files = (FileStatus[]) retVal;
if (files == null) {
logger.error("Error getting files, no files returned");
return null;
}
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path = new Path(temp_path_base + config.get("file_id"));
String suffix = getSuffix(config.get("file_id"), cur_file.getName());
cur_local_path = cur_local_path.suffix(suffix);
Path res_path = new Path(new String(final_result + suffix));
logger.debug("Moving file" + cur_file.toString() + " to " + res_path.toString());
if (config.get("copy").equals("true")) {
fs.moveFromLocalFile(cur_file, res_path);
} else {
fs.rename(cur_file, res_path);
}
}
config.put("full_file_name", final_result);
}
}
unlock(lockName);
return ret_path;
}