/**
* Get the list of codecs listed in the configuration
*
* @param conf the configuration to look in
* @return a list of the Configuration classes or null if the attribute was not set
*/
public static List<Class<? extends CompressionCodec>> getCodecClasses(Configuration conf) {
String codecsString = conf.get("io.compression.codecs");
if (codecsString != null) {
List<Class<? extends CompressionCodec>> result =
new ArrayList<Class<? extends CompressionCodec>>();
StringTokenizer codecSplit = new StringTokenizer(codecsString, ",");
while (codecSplit.hasMoreElements()) {
String codecSubstring = codecSplit.nextToken();
if (codecSubstring.length() != 0) {
try {
Class<?> cls = conf.getClassByName(codecSubstring);
if (!CompressionCodec.class.isAssignableFrom(cls)) {
throw new IllegalArgumentException(
"Class " + codecSubstring + " is not a CompressionCodec");
}
result.add(cls.asSubclass(CompressionCodec.class));
} catch (ClassNotFoundException ex) {
throw new IllegalArgumentException(
"Compression codec " + codecSubstring + " not found.", ex);
}
}
}
return result;
} else {
return null;
}
}
@SuppressWarnings("unchecked")
public void readFields(DataInput is) throws IOException {
disableCounter = is.readBoolean();
isMultiInputs = is.readBoolean();
totalSplits = is.readInt();
splitIndex = is.readInt();
inputIndex = is.readInt();
targetOps = (ArrayList<OperatorKey>) readObject(is);
int splitLen = is.readInt();
String splitClassName = is.readUTF();
try {
Class splitClass = conf.getClassByName(splitClassName);
SerializationFactory sf = new SerializationFactory(conf);
// The correct call sequence for Deserializer is, we shall open, then deserialize, but we
// shall not close
Deserializer d = sf.getDeserializer(splitClass);
d.open((InputStream) is);
wrappedSplits = new InputSplit[splitLen];
for (int i = 0; i < splitLen; i++) {
wrappedSplits[i] = (InputSplit) ReflectionUtils.newInstance(splitClass, conf);
d.deserialize(wrappedSplits[i]);
}
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
}
public void initialize(Configuration job, Properties tbl) throws SerDeException {
try {
// both the classname and the protocol name are Table properties
// the only hardwired assumption is that records are fixed on a
// per Table basis
String className =
tbl.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS);
Class<?> recordClass = job.getClassByName(className);
String protoName =
tbl.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT);
if (protoName == null) {
protoName = "TBinaryProtocol";
}
// For backward compatibility
protoName = protoName.replace("com.facebook.thrift.protocol", "org.apache.thrift.protocol");
TProtocolFactory tp = TReflectionUtils.getProtocolFactoryByName(protoName);
tsd = new ThriftByteStreamTypedSerDe(recordClass, tp, tp);
} catch (Exception e) {
throw new SerDeException(e);
}
}
/**
* Find and load the class with given name <tt>className</tt> by first finding it in the specified
* <tt>conf</tt>. If the specified <tt>conf</tt> is null, try load it directly.
*/
public static Class<?> loadClass(Configuration conf, String className) {
Class<?> declaredClass = null;
try {
if (conf != null) declaredClass = conf.getClassByName(className);
else declaredClass = Class.forName(className);
} catch (ClassNotFoundException e) {
throw new RuntimeException("readObject can't find class " + className, e);
}
return declaredClass;
}
/**
* Instantiates a Hadoop codec for compressing and decompressing Gzip files. This is the most
* common compression applied to WARC files.
*
* @param conf The Hadoop configuration.
*/
public static CompressionCodec getGzipCodec(Configuration conf) {
try {
return (CompressionCodec)
ReflectionUtils.newInstance(
conf.getClassByName("org.apache.hadoop.io.compress.GzipCodec")
.asSubclass(CompressionCodec.class),
conf);
} catch (ClassNotFoundException e) {
logger.warn("GzipCodec could not be instantiated", e);
return null;
}
}
public ErasureCode createErasureCode(Configuration conf) {
// Create the scheduler
Class<?> erasureCode = null;
try {
erasureCode =
conf.getClass(
ERASURE_CODE_KEY_PREFIX + this.id, conf.getClassByName(this.erasureCodeClass));
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
ErasureCode code = (ErasureCode) ReflectionUtils.newInstance(erasureCode, conf);
code.init(this);
return code;
}
private void setTypeRef(Configuration conf) {
String className = conf.get(CLASS_CONF_KEY);
if (className == null) {
throw new RuntimeException(CLASS_CONF_KEY + " is not set");
}
Class<?> clazz = null;
try {
clazz = conf.getClassByName(className);
} catch (ClassNotFoundException e) {
throw new RuntimeException("failed to instantiate class '" + className + "'", e);
}
typeRef = new TypeRef<M>(clazz) {};
}
public Writable readWritable(Writable writable) throws IOException {
ByteArrayInputStream bais = new ByteArrayInputStream(in.readBytes());
DataInputStream dis = new DataInputStream(bais);
String className = WritableUtils.readString(dis);
if (writable == null) {
try {
Class<? extends Writable> cls = conf.getClassByName(className).asSubclass(Writable.class);
writable = (Writable) ReflectionUtils.newInstance(cls, conf);
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
} else if (!writable.getClass().getName().equals(className)) {
throw new IOException("wrong Writable class given");
}
writable.readFields(dis);
return writable;
}
private static Class<? extends Message> getProtobufClass(
Configuration conf, String protoClassName) {
// Try both normal name and canonical name of the class.
Class<?> protoClass = null;
try {
if (conf == null) {
protoClass = Class.forName(protoClassName);
} else {
protoClass = conf.getClassByName(protoClassName);
}
} catch (ClassNotFoundException e) {
// the class name might be canonical name.
protoClass = getInnerProtobufClass(protoClassName);
}
return protoClass.asSubclass(Message.class);
}
/**
* Fetch a class by its name, rather than by a key name in the Configuration properties. The
* Configuration class is used for its internal cache of class names and to ensure that the same
* ClassLoader is used across all keys.
*
* @param conf the Configuration
* @param className the name of the class
* @param xface an interface or superclass of expected class
* @param <U> the type of xface
* @return the class or {@code null} if not found
*/
public static <U> Class<? extends U> getClassByName(
final Configuration conf, final String className, final Class<U> xface) {
if (className == null) {
return null;
}
try {
Class<?> theClass = conf.getClassByName(className);
if (theClass != null && !xface.isAssignableFrom(theClass)) {
throw new RuntimeException(theClass + " not " + xface.getName());
} else if (theClass != null) {
return theClass.asSubclass(xface);
} else {
return null;
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}