コード例 #1
0
ファイル: Vote.java プロジェクト: jwegan/parseexception
  public static Vote getVote(int uid, int sid) {
    ObjectCache cache = ObjectCache.getInstance();
    Vote v = (Vote) cache.lookupObject(uid, sid, "votes");
    if (v == null) {
      v = new Vote(uid, sid);
    }

    return v;
  }
コード例 #2
0
ファイル: Vote.java プロジェクト: jwegan/parseexception
  public Vote(int uid, int sid) {
    this.uid = uid;
    this.sid = sid;

    String query = "SELECT upvote FROM votes WHERE uid = ? AND sid = ?";
    ArrayList<Object> args = new ArrayList<Object>();
    args.add(new Integer(uid));
    args.add(new Integer(sid));
    vote = DataAccess.queryReturnInt(get_vote, query, args, "upvote");

    ObjectCache cache = ObjectCache.getInstance();
    cache.addObject(this, uid, sid, "votes");
  }
コード例 #3
0
ファイル: ObjectCacheAgent.java プロジェクト: rsumbaly/krati
 @Override
 public T get(int objectId) {
   T object = _cache.get(objectId);
   if (object != null && _outboundHandler != null && _outboundHandler.getEnabled()) {
     _outboundHandler.process(object);
   }
   return object;
 }
コード例 #4
0
ファイル: ObjectCacheAgent.java プロジェクト: rsumbaly/krati
  @Override
  public boolean set(int objectId, T object, long scn) throws Exception {
    if (object != null && _inboundHandler != null && _inboundHandler.getEnabled()) {
      _inboundHandler.process(object);
    }

    synchronized (_cache) {
      return _cache.set(objectId, object, scn);
    }
  }
コード例 #5
0
  @Override
  public void closeOp(boolean abort) throws HiveException {
    boolean spilled = false;
    for (MapJoinTableContainer container : mapJoinTables) {
      if (container != null) {
        spilled = spilled || container.hasSpill();
        container.dumpMetrics();
      }
    }

    // For Hybrid Grace Hash Join, we need to see if there is any spilled data to be processed next
    if (spilled) {
      if (!abort) {
        if (hashMapRowGetters == null) {
          hashMapRowGetters = new ReusableGetAdaptor[mapJoinTables.length];
        }
        int numPartitions = 0;
        // Find out number of partitions for each small table (should be same across tables)
        for (byte pos = 0; pos < mapJoinTables.length; pos++) {
          if (pos != conf.getPosBigTable()) {
            firstSmallTable = (HybridHashTableContainer) mapJoinTables[pos];
            numPartitions = firstSmallTable.getHashPartitions().length;
            break;
          }
        }
        assert numPartitions != 0 : "Number of partitions must be greater than 0!";

        if (firstSmallTable.hasSpill()) {
          spilledMapJoinTables = new MapJoinBytesTableContainer[mapJoinTables.length];
          hybridMapJoinLeftover = true;

          // Clear all in-memory partitions first
          for (byte pos = 0; pos < mapJoinTables.length; pos++) {
            MapJoinTableContainer tableContainer = mapJoinTables[pos];
            if (tableContainer != null && tableContainer instanceof HybridHashTableContainer) {
              HybridHashTableContainer hybridHtContainer =
                  (HybridHashTableContainer) tableContainer;
              hybridHtContainer.dumpStats();

              HashPartition[] hashPartitions = hybridHtContainer.getHashPartitions();
              // Clear all in memory partitions first
              for (int i = 0; i < hashPartitions.length; i++) {
                if (!hashPartitions[i].isHashMapOnDisk()) {
                  hybridHtContainer.setTotalInMemRowCount(
                      hybridHtContainer.getTotalInMemRowCount()
                          - hashPartitions[i].getHashMapFromMemory().getNumValues());
                  hashPartitions[i].getHashMapFromMemory().clear();
                }
              }
              assert hybridHtContainer.getTotalInMemRowCount() == 0;
            }
          }

          // Reprocess the spilled data
          for (int i = 0; i < numPartitions; i++) {
            HashPartition[] hashPartitions = firstSmallTable.getHashPartitions();
            if (hashPartitions[i].isHashMapOnDisk()) {
              try {
                continueProcess(i); // Re-process spilled data
              } catch (Exception e) {
                throw new HiveException(e);
              }
              for (byte pos = 0; pos < order.length; pos++) {
                if (pos != conf.getPosBigTable()) spilledMapJoinTables[pos] = null;
              }
            }
          }
        }
      }

      if (isLogInfoEnabled) {
        LOG.info("spilled: " + spilled + " abort: " + abort + ". Clearing spilled partitions.");
      }

      // spilled tables are loaded always (no sharing), so clear it
      clearAllTableContainers();
      cache.remove(cacheKey);
    }

    // in mapreduce case, we need to always clear up as mapreduce doesn't have object registry.
    if ((this.getExecContext() != null)
        && (this.getExecContext().getLocalWork() != null)
        && (this.getExecContext().getLocalWork().getInputFileChangeSensitive())
        && !(HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")
            && SparkUtilities.isDedicatedCluster(hconf))) {
      if (isLogInfoEnabled) {
        LOG.info("MR: Clearing all map join table containers.");
      }
      clearAllTableContainers();
    }

    this.loader = null;
    super.closeOp(abort);
  }
コード例 #6
0
  @Override
  protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
    this.hconf = hconf;
    unwrapContainer = new UnwrapRowContainer[conf.getTagLength()];

    Collection<Future<?>> result = super.initializeOp(hconf);
    if (result == null) {
      result = new HashSet<Future<?>>();
    }

    int tagLen = conf.getTagLength();

    // On Tez only: The hash map might already be cached in the container we run
    // the task in. On MR: The cache is a no-op.
    cacheKey =
        HiveConf.getVar(hconf, HiveConf.ConfVars.HIVEQUERYID)
            + "__HASH_MAP_"
            + this.getOperatorId()
            + "_container";

    cache = ObjectCacheFactory.getCache(hconf);
    loader = getHashTableLoader(hconf);

    hashMapRowGetters = null;

    mapJoinTables = new MapJoinTableContainer[tagLen];
    mapJoinTableSerdes = new MapJoinTableContainerSerDe[tagLen];
    hashTblInitedOnce = false;

    generateMapMetaData();

    final ExecMapperContext mapContext = getExecContext();
    final MapredContext mrContext = MapredContext.get();

    if (!conf.isBucketMapJoin() && !conf.isDynamicPartitionHashJoin()) {
      /*
       * The issue with caching in case of bucket map join is that different tasks
       * process different buckets and if the container is reused to join a different bucket,
       * join results can be incorrect. The cache is keyed on operator id and for bucket map join
       * the operator does not change but data needed is different. For a proper fix, this
       * requires changes in the Tez API with regard to finding bucket id and
       * also ability to schedule tasks to re-use containers that have cached the specific bucket.
       */
      if (isLogInfoEnabled) {
        LOG.info("This is not bucket map join, so cache");
      }

      Future<Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]>> future =
          cache.retrieveAsync(
              cacheKey,
              new Callable<Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]>>() {
                @Override
                public Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> call()
                    throws HiveException {
                  return loadHashTable(mapContext, mrContext);
                }
              });
      result.add(future);
    } else if (!isInputFileChangeSensitive(mapContext)) {
      loadHashTable(mapContext, mrContext);
      hashTblInitedOnce = true;
    }
    return result;
  }
コード例 #7
0
ファイル: ObjectCacheAgent.java プロジェクト: rsumbaly/krati
 @Override
 public void persist() throws IOException {
   synchronized (_cache) {
     _cache.persist();
   }
 }
コード例 #8
0
ファイル: ObjectCacheAgent.java プロジェクト: rsumbaly/krati
 @Override
 public void sync() throws IOException {
   synchronized (_cache) {
     _cache.sync();
   }
 }
コード例 #9
0
ファイル: ObjectCacheAgent.java プロジェクト: rsumbaly/krati
 @Override
 public boolean delete(int objectId, long scn) throws Exception {
   synchronized (_cache) {
     return _cache.delete(objectId, scn);
   }
 }
コード例 #10
0
ファイル: ObjectCacheAgent.java プロジェクト: rsumbaly/krati
 @Override
 public int getObjectIdStart() {
   return _cache.getObjectIdStart();
 }
コード例 #11
0
ファイル: ObjectCacheAgent.java プロジェクト: rsumbaly/krati
 @Override
 public int getObjectIdCount() {
   return _cache.getObjectIdCount();
 }
コード例 #12
0
ファイル: ObjectCacheAgent.java プロジェクト: rsumbaly/krati
 @Override
 public void saveHWMark(long endOfPeriod) throws Exception {
   _cache.saveHWMark(endOfPeriod);
 }
コード例 #13
0
ファイル: ObjectCacheAgent.java プロジェクト: rsumbaly/krati
 @Override
 public long getLWMark() {
   return _cache.getLWMark();
 }
コード例 #14
0
  /**
   * Determines what indices in the given sequence are active. An active index i means that the i-th
   * method call creates an interesting/useful value that can be used as an input to a larger
   * sequence; inactive indices are never used as inputs. The effect of setting active/inactive
   * indices is that the SequenceCollection to which the given sequences is added only considers the
   * active indices when deciding whether the sequence creates values of a given type.
   *
   * <p>In addition to determining active indices, this method determines if any primitive values
   * created during execution of the sequence are new values not encountered before. Such values are
   * added to the component manager so they can be used during subsequent generation attempts.
   */
  public void processSequence(ExecutableSequence seq) {
    Tracer.trace("processSequence");
    if (GenInputsAbstract.offline) {
      Tracer.trace("offline@processSequence");
      if (Log.isLoggingOn()) {
        Log.logLine(
            "Making all indices active (offline generation specified; sequences are not executed).");
      }
      seq.sequence.setAllActiveFlags();
      return;
    }
    Tracer.trace("NOT offline@processSequence");

    if (seq.hasNonExecutedStatements()) {
      if (Log.isLoggingOn()) {
        Log.logLine(
            "Making all indices inactive (sequence has non-executed statements, so judging it inadequate for further extension).");
      }
      seq.sequence.clearAllActiveFlags();
      return;
    }

    if (seq.hasFailure()) {
      if (Log.isLoggingOn()) {
        Log.logLine(
            "Making all indices inactive (sequence reveals a failure, so judging it inadequate for further extension)");
      }
      seq.sequence.clearAllActiveFlags();
      return;
    }

    if (!seq.isNormalExecution()) {
      if (Log.isLoggingOn()) {
        Log.logLine(
            "Making all indices inactive (exception thrown, or failure revealed during execution).");
      }
      seq.sequence.clearAllActiveFlags();
      return;
    }

    // If runtime value is a primitive value, clear active flag, and
    // if the value is new, add a sequence corresponding to that value.
    for (int i = 0; i < seq.sequence.size(); i++) {

      // type ensured by isNormalExecution clause ealier in this method.
      NormalExecution e = (NormalExecution) seq.getResult(i);
      Object runtimeValue = e.getRuntimeValue();
      if (runtimeValue == null) {
        if (Log.isLoggingOn()) {
          Log.logLine("Making index " + i + " inactive (value is null)");
        }
        seq.sequence.clearActiveFlag(i);
        continue;
      }

      Class<?> objectClass = runtimeValue.getClass();

      Tracer.trace("processSequence-useobjcache");
      if (PrimitiveTypes.isBoxedOrPrimitiveOrStringType(objectClass)) {
        if (Log.isLoggingOn()) {
          Log.logLine("Making index " + i + " inactive (value is a primitive)");
        }
        seq.sequence.clearActiveFlag(i);

        boolean looksLikeObjToString =
            (runtimeValue instanceof String)
                && PrimitiveTypes.looksLikeObjectToString((String) runtimeValue);
        boolean tooLongString =
            (runtimeValue instanceof String)
                && !PrimitiveTypes.stringLengthOK((String) runtimeValue);
        if (!looksLikeObjToString && !tooLongString && runtimePrimitivesSeen.add(runtimeValue)) {
          // Have not seen this value before; add it to the component set.
          componentManager.addGeneratedSequence(
              PrimitiveOrStringOrNullDecl.sequenceForPrimitive(runtimeValue));
        }
      } else if (GenInputsAbstract.use_object_cache) {
        Tracer.trace("use_object_cache@processSequence-useobjcache");
        objectCache.setActiveFlags(seq, i);
      } else {
        if (Log.isLoggingOn()) {
          Log.logLine("Making index " + i + " active.");
        }
      }
    }
  }