public static Vote getVote(int uid, int sid) {
ObjectCache cache = ObjectCache.getInstance();
Vote v = (Vote) cache.lookupObject(uid, sid, "votes");
if (v == null) {
v = new Vote(uid, sid);
}
return v;
}
public Vote(int uid, int sid) {
this.uid = uid;
this.sid = sid;
String query = "SELECT upvote FROM votes WHERE uid = ? AND sid = ?";
ArrayList<Object> args = new ArrayList<Object>();
args.add(new Integer(uid));
args.add(new Integer(sid));
vote = DataAccess.queryReturnInt(get_vote, query, args, "upvote");
ObjectCache cache = ObjectCache.getInstance();
cache.addObject(this, uid, sid, "votes");
}
@Override
public T get(int objectId) {
T object = _cache.get(objectId);
if (object != null && _outboundHandler != null && _outboundHandler.getEnabled()) {
_outboundHandler.process(object);
}
return object;
}
@Override
public boolean set(int objectId, T object, long scn) throws Exception {
if (object != null && _inboundHandler != null && _inboundHandler.getEnabled()) {
_inboundHandler.process(object);
}
synchronized (_cache) {
return _cache.set(objectId, object, scn);
}
}
@Override
public void closeOp(boolean abort) throws HiveException {
boolean spilled = false;
for (MapJoinTableContainer container : mapJoinTables) {
if (container != null) {
spilled = spilled || container.hasSpill();
container.dumpMetrics();
}
}
// For Hybrid Grace Hash Join, we need to see if there is any spilled data to be processed next
if (spilled) {
if (!abort) {
if (hashMapRowGetters == null) {
hashMapRowGetters = new ReusableGetAdaptor[mapJoinTables.length];
}
int numPartitions = 0;
// Find out number of partitions for each small table (should be same across tables)
for (byte pos = 0; pos < mapJoinTables.length; pos++) {
if (pos != conf.getPosBigTable()) {
firstSmallTable = (HybridHashTableContainer) mapJoinTables[pos];
numPartitions = firstSmallTable.getHashPartitions().length;
break;
}
}
assert numPartitions != 0 : "Number of partitions must be greater than 0!";
if (firstSmallTable.hasSpill()) {
spilledMapJoinTables = new MapJoinBytesTableContainer[mapJoinTables.length];
hybridMapJoinLeftover = true;
// Clear all in-memory partitions first
for (byte pos = 0; pos < mapJoinTables.length; pos++) {
MapJoinTableContainer tableContainer = mapJoinTables[pos];
if (tableContainer != null && tableContainer instanceof HybridHashTableContainer) {
HybridHashTableContainer hybridHtContainer =
(HybridHashTableContainer) tableContainer;
hybridHtContainer.dumpStats();
HashPartition[] hashPartitions = hybridHtContainer.getHashPartitions();
// Clear all in memory partitions first
for (int i = 0; i < hashPartitions.length; i++) {
if (!hashPartitions[i].isHashMapOnDisk()) {
hybridHtContainer.setTotalInMemRowCount(
hybridHtContainer.getTotalInMemRowCount()
- hashPartitions[i].getHashMapFromMemory().getNumValues());
hashPartitions[i].getHashMapFromMemory().clear();
}
}
assert hybridHtContainer.getTotalInMemRowCount() == 0;
}
}
// Reprocess the spilled data
for (int i = 0; i < numPartitions; i++) {
HashPartition[] hashPartitions = firstSmallTable.getHashPartitions();
if (hashPartitions[i].isHashMapOnDisk()) {
try {
continueProcess(i); // Re-process spilled data
} catch (Exception e) {
throw new HiveException(e);
}
for (byte pos = 0; pos < order.length; pos++) {
if (pos != conf.getPosBigTable()) spilledMapJoinTables[pos] = null;
}
}
}
}
}
if (isLogInfoEnabled) {
LOG.info("spilled: " + spilled + " abort: " + abort + ". Clearing spilled partitions.");
}
// spilled tables are loaded always (no sharing), so clear it
clearAllTableContainers();
cache.remove(cacheKey);
}
// in mapreduce case, we need to always clear up as mapreduce doesn't have object registry.
if ((this.getExecContext() != null)
&& (this.getExecContext().getLocalWork() != null)
&& (this.getExecContext().getLocalWork().getInputFileChangeSensitive())
&& !(HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")
&& SparkUtilities.isDedicatedCluster(hconf))) {
if (isLogInfoEnabled) {
LOG.info("MR: Clearing all map join table containers.");
}
clearAllTableContainers();
}
this.loader = null;
super.closeOp(abort);
}
@Override
protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
this.hconf = hconf;
unwrapContainer = new UnwrapRowContainer[conf.getTagLength()];
Collection<Future<?>> result = super.initializeOp(hconf);
if (result == null) {
result = new HashSet<Future<?>>();
}
int tagLen = conf.getTagLength();
// On Tez only: The hash map might already be cached in the container we run
// the task in. On MR: The cache is a no-op.
cacheKey =
HiveConf.getVar(hconf, HiveConf.ConfVars.HIVEQUERYID)
+ "__HASH_MAP_"
+ this.getOperatorId()
+ "_container";
cache = ObjectCacheFactory.getCache(hconf);
loader = getHashTableLoader(hconf);
hashMapRowGetters = null;
mapJoinTables = new MapJoinTableContainer[tagLen];
mapJoinTableSerdes = new MapJoinTableContainerSerDe[tagLen];
hashTblInitedOnce = false;
generateMapMetaData();
final ExecMapperContext mapContext = getExecContext();
final MapredContext mrContext = MapredContext.get();
if (!conf.isBucketMapJoin() && !conf.isDynamicPartitionHashJoin()) {
/*
* The issue with caching in case of bucket map join is that different tasks
* process different buckets and if the container is reused to join a different bucket,
* join results can be incorrect. The cache is keyed on operator id and for bucket map join
* the operator does not change but data needed is different. For a proper fix, this
* requires changes in the Tez API with regard to finding bucket id and
* also ability to schedule tasks to re-use containers that have cached the specific bucket.
*/
if (isLogInfoEnabled) {
LOG.info("This is not bucket map join, so cache");
}
Future<Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]>> future =
cache.retrieveAsync(
cacheKey,
new Callable<Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]>>() {
@Override
public Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> call()
throws HiveException {
return loadHashTable(mapContext, mrContext);
}
});
result.add(future);
} else if (!isInputFileChangeSensitive(mapContext)) {
loadHashTable(mapContext, mrContext);
hashTblInitedOnce = true;
}
return result;
}
@Override
public void persist() throws IOException {
synchronized (_cache) {
_cache.persist();
}
}
@Override
public void sync() throws IOException {
synchronized (_cache) {
_cache.sync();
}
}
@Override
public boolean delete(int objectId, long scn) throws Exception {
synchronized (_cache) {
return _cache.delete(objectId, scn);
}
}
@Override
public int getObjectIdStart() {
return _cache.getObjectIdStart();
}
@Override
public int getObjectIdCount() {
return _cache.getObjectIdCount();
}
@Override
public void saveHWMark(long endOfPeriod) throws Exception {
_cache.saveHWMark(endOfPeriod);
}
@Override
public long getLWMark() {
return _cache.getLWMark();
}
/**
* Determines what indices in the given sequence are active. An active index i means that the i-th
* method call creates an interesting/useful value that can be used as an input to a larger
* sequence; inactive indices are never used as inputs. The effect of setting active/inactive
* indices is that the SequenceCollection to which the given sequences is added only considers the
* active indices when deciding whether the sequence creates values of a given type.
*
* <p>In addition to determining active indices, this method determines if any primitive values
* created during execution of the sequence are new values not encountered before. Such values are
* added to the component manager so they can be used during subsequent generation attempts.
*/
public void processSequence(ExecutableSequence seq) {
Tracer.trace("processSequence");
if (GenInputsAbstract.offline) {
Tracer.trace("offline@processSequence");
if (Log.isLoggingOn()) {
Log.logLine(
"Making all indices active (offline generation specified; sequences are not executed).");
}
seq.sequence.setAllActiveFlags();
return;
}
Tracer.trace("NOT offline@processSequence");
if (seq.hasNonExecutedStatements()) {
if (Log.isLoggingOn()) {
Log.logLine(
"Making all indices inactive (sequence has non-executed statements, so judging it inadequate for further extension).");
}
seq.sequence.clearAllActiveFlags();
return;
}
if (seq.hasFailure()) {
if (Log.isLoggingOn()) {
Log.logLine(
"Making all indices inactive (sequence reveals a failure, so judging it inadequate for further extension)");
}
seq.sequence.clearAllActiveFlags();
return;
}
if (!seq.isNormalExecution()) {
if (Log.isLoggingOn()) {
Log.logLine(
"Making all indices inactive (exception thrown, or failure revealed during execution).");
}
seq.sequence.clearAllActiveFlags();
return;
}
// If runtime value is a primitive value, clear active flag, and
// if the value is new, add a sequence corresponding to that value.
for (int i = 0; i < seq.sequence.size(); i++) {
// type ensured by isNormalExecution clause ealier in this method.
NormalExecution e = (NormalExecution) seq.getResult(i);
Object runtimeValue = e.getRuntimeValue();
if (runtimeValue == null) {
if (Log.isLoggingOn()) {
Log.logLine("Making index " + i + " inactive (value is null)");
}
seq.sequence.clearActiveFlag(i);
continue;
}
Class<?> objectClass = runtimeValue.getClass();
Tracer.trace("processSequence-useobjcache");
if (PrimitiveTypes.isBoxedOrPrimitiveOrStringType(objectClass)) {
if (Log.isLoggingOn()) {
Log.logLine("Making index " + i + " inactive (value is a primitive)");
}
seq.sequence.clearActiveFlag(i);
boolean looksLikeObjToString =
(runtimeValue instanceof String)
&& PrimitiveTypes.looksLikeObjectToString((String) runtimeValue);
boolean tooLongString =
(runtimeValue instanceof String)
&& !PrimitiveTypes.stringLengthOK((String) runtimeValue);
if (!looksLikeObjToString && !tooLongString && runtimePrimitivesSeen.add(runtimeValue)) {
// Have not seen this value before; add it to the component set.
componentManager.addGeneratedSequence(
PrimitiveOrStringOrNullDecl.sequenceForPrimitive(runtimeValue));
}
} else if (GenInputsAbstract.use_object_cache) {
Tracer.trace("use_object_cache@processSequence-useobjcache");
objectCache.setActiveFlags(seq, i);
} else {
if (Log.isLoggingOn()) {
Log.logLine("Making index " + i + " active.");
}
}
}
}