예제 #1
0
파일: ExecMapper.java 프로젝트: cnhans/tdw
  public void close() {
    if (oc == null) {
      l4j.trace("Close called. no row processed by map.");
    }

    try {
      mo.close(abort);
      if (fetchOperators != null) {
        mapredLocalWork localWork = mo.getConf().getMapLocalWork();
        for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
          Operator<? extends Serializable> forwardOp =
              localWork.getAliasToWork().get(entry.getKey());
          forwardOp.close(abort);
        }
      }

      if (l4j.isInfoEnabled()) {
        long used_memory = memoryMXBean.getHeapMemoryUsage().getUsed();
        l4j.info("ExecMapper: processed " + numRows + " rows: used memory = " + used_memory);
      }

      reportStats rps = new reportStats(rp);
      mo.preorderMap(rps);
      oc = null;
      mo.setOutputCollector(oc);
      return;
    } catch (Exception e) {
      if (!abort) {
        l4j.error("Hit error while closing operators - failing tree");
        throw new RuntimeException("Error while closing operators", e);
      }
    }
  }
예제 #2
0
파일: ExecMapper.java 프로젝트: cnhans/tdw
  private void processOldMapLocalWork() {
    if (fetchOperators != null) {
      try {
        mapredLocalWork localWork = mo.getConf().getMapLocalWork();
        int fetchOpNum = 0;
        for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
          int fetchOpRows = 0;
          String alias = entry.getKey();
          FetchOperator fetchOp = entry.getValue();
          Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);

          while (true) {
            InspectableObject row = fetchOp.getNextRow();
            if (row == null) {
              break;
            }
            fetchOpRows++;
            forwardOp.process(row.o, 0);
          }

          if (l4j.isInfoEnabled()) {
            l4j.info(
                "fetch "
                    + fetchOpNum++
                    + " processed "
                    + fetchOpRows
                    + " used mem: "
                    + memoryMXBean.getHeapMemoryUsage().getUsed());
          }
        }
      } catch (Throwable e) {
        abort = true;
        if (e instanceof OutOfMemoryError) {
          throw (OutOfMemoryError) e;
        } else {
          throw new RuntimeException("Map local work failed", e);
        }
      }
    }
  }
예제 #3
0
파일: ExecMapper.java 프로젝트: cnhans/tdw
  private void processMapLocalWork() {
    l4j.info("Begin to process map side computing!");
    if (fetchOperators != null) {
      try {
        int fetchOpNum = 0;
        for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
          int fetchOpRows = 0;
          String alias = entry.getKey();
          FetchOperator fetchOp = entry.getValue();
          l4j.info("The table processed in the fetch operator: " + alias);

          fetchOp.clearFetchContext();
          setUpFetchOpContext(fetchOp, alias);

          Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);

          while (true) {
            InspectableObject row = fetchOp.getNextRow();
            if (row == null) {
              forwardOp.close(false);
              break;
            }
            fetchOpRows++;
            forwardOp.process(row.o, 0);
            if (forwardOp.getDone()) {
              done = true;
              break;
            }
          }

          if (l4j.isInfoEnabled()) {
            l4j.info(
                "fetch "
                    + fetchOpNum++
                    + " processed "
                    + fetchOpRows
                    + " used mem: "
                    + memoryMXBean.getHeapMemoryUsage().getUsed());
          }
        }
      } catch (Throwable e) {
        abort = true;
        if (e instanceof OutOfMemoryError) {
          throw (OutOfMemoryError) e;
        } else {
          throw new RuntimeException("Map local work failed", e);
        }
      }
    }
  }
예제 #4
0
파일: ExecMapper.java 프로젝트: cnhans/tdw
  public void configure(JobConf job) {
    memoryMXBean = ManagementFactory.getMemoryMXBean();
    l4j.info("maximum memory = " + memoryMXBean.getHeapMemoryUsage().getMax());

    try {
      l4j.info(
          "conf classpath = " + Arrays.asList(((URLClassLoader) job.getClassLoader()).getURLs()));
      l4j.info(
          "thread classpath = "
              + Arrays.asList(
                  ((URLClassLoader) Thread.currentThread().getContextClassLoader()).getURLs()));
    } catch (Exception e) {
      l4j.info("cannot get classpath: " + e.getMessage());
    }
    try {
      jc = job;
      int estCountBucketSize = jc.getInt("hive.exec.estdistinct.bucketsize.log", 15);
      int estCountBufferSize = jc.getInt("hive.exec.estdistinct.buffsize.log", 8);
      GenericUDAFCardinalityEstimation.initParams(estCountBucketSize, estCountBufferSize);

      boolean isChangSizeZero2Null =
          jc.getBoolean(
              HiveConf.ConfVars.HIVE_UDTF_EXPLODE_CHANGE_ZERO_SIZE_2_NULL.varname,
              HiveConf.ConfVars.HIVE_UDTF_EXPLODE_CHANGE_ZERO_SIZE_2_NULL.defaultBoolVal);
      boolean isChangeNull2Null =
          jc.getBoolean(
              HiveConf.ConfVars.HIVE_UDTF_EXPLODE_CHANGE_NULL_2_NULL.varname,
              HiveConf.ConfVars.HIVE_UDTF_EXPLODE_CHANGE_NULL_2_NULL.defaultBoolVal);
      GenericUDTFExplode.isChangSizeZero2Null = isChangSizeZero2Null;
      GenericUDTFExplode.isChangNull2Null = isChangeNull2Null;
      GenericUDTFPosExplode.isChangSizeZero2Null = isChangSizeZero2Null;
      GenericUDTFPosExplode.isChangNull2Null = isChangeNull2Null;

      mapredWork mrwork = Utilities.getMapRedWork(job);
      mo = new MapOperator();
      mo.setConf(mrwork);
      mo.setChildren(job);
      l4j.info(mo.dump(0));
      mo.initialize(jc, null);

      localWork = mrwork.getMapLocalWork();
      if (localWork == null) {
        return;
      }
      fetchOperators = new HashMap<String, FetchOperator>();
      for (Map.Entry<String, fetchWork> entry : localWork.getAliasToFetchWork().entrySet()) {
        fetchOperators.put(entry.getKey(), new FetchOperator(entry.getValue(), job));
        l4j.info("fetchoperator for " + entry.getKey() + " created");
      }
      for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
        Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
        forwardOp.initialize(
            jc, new ObjectInspector[] {entry.getValue().getOutputObjectInspector()});
        l4j.info("fetchoperator for " + entry.getKey() + " initialized");
      }
    } catch (Throwable e) {
      abort = true;
      if (e instanceof OutOfMemoryError) {
        throw (OutOfMemoryError) e;
      } else {
        throw new RuntimeException("Map operator initialization failed", e);
      }
    }
  }