Пример #1
0
 @SuppressWarnings("unchecked")
 private <X extends StateHandle<?>> X serializeDeserialize(X handle)
     throws IOException, ClassNotFoundException {
   byte[] serialized = InstantiationUtil.serializeObject(handle);
   return (X)
       InstantiationUtil.deserializeObject(
           serialized, Thread.currentThread().getContextClassLoader());
 }
Пример #2
0
 private <T> T copyObject(T object) {
   try {
     return InstantiationUtil.deserializeObject(
         InstantiationUtil.serializeObject(object), getClass().getClassLoader());
   } catch (IOException | ClassNotFoundException e) {
     throw new RuntimeException("Failed to copy object.");
   }
 }
 private org.apache.hadoop.fs.FileSystem instantiateFileSystem(
     Class<? extends org.apache.hadoop.fs.FileSystem> fsClass) throws IOException {
   try {
     return fsClass.newInstance();
   } catch (ExceptionInInitializerError e) {
     throw new IOException(
         "The filesystem class '"
             + fsClass.getName()
             + "' throw an exception upon initialization.",
         e.getException());
   } catch (Throwable t) {
     String errorMessage = InstantiationUtil.checkForInstantiationError(fsClass);
     if (errorMessage != null) {
       throw new IOException(
           "The filesystem class '"
               + fsClass.getName()
               + "' cannot be instantiated: "
               + errorMessage);
     } else {
       throw new IOException(
           "An error occurred while instantiating the filesystem class '"
               + fsClass.getName()
               + "'.",
           t);
     }
   }
 }
Пример #4
0
    /**
     * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current
     * state handle
     *
     * @throws IOException
     * @throws ClassNotFoundException
     * @throws RocksDBException
     */
    private void restoreKVStateMetaData()
        throws IOException, ClassNotFoundException, RocksDBException {
      // read number of k/v states
      int numColumns = currentStateHandleInView.readInt();

      // those two lists are aligned and should later have the same size!
      currentStateHandleKVStateColumnFamilies = new ArrayList<>(numColumns);

      // restore the empty columns for the k/v states through the metadata
      for (int i = 0; i < numColumns; i++) {

        StateDescriptor<?, ?> stateDescriptor =
            InstantiationUtil.deserializeObject(
                currentStateHandleInStream, rocksDBKeyedStateBackend.userCodeClassLoader);

        Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>> columnFamily =
            rocksDBKeyedStateBackend.kvStateInformation.get(stateDescriptor.getName());

        if (null == columnFamily) {
          ColumnFamilyDescriptor columnFamilyDescriptor =
              new ColumnFamilyDescriptor(
                  stateDescriptor.getName().getBytes(), rocksDBKeyedStateBackend.columnOptions);

          columnFamily =
              new Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>>(
                  rocksDBKeyedStateBackend.db.createColumnFamily(columnFamilyDescriptor),
                  stateDescriptor);

          rocksDBKeyedStateBackend.kvStateInformation.put(stateDescriptor.getName(), columnFamily);
        }

        currentStateHandleKVStateColumnFamilies.add(columnFamily.f0);
      }
    }
Пример #5
0
    /** Writes the {@code Context} to the given state checkpoint output. */
    protected void writeToState(StateBackend.CheckpointStateOutputView out) throws IOException {
      keySerializer.serialize(key, out);
      windowSerializer.serialize(window, out);
      out.writeLong(watermarkTimer);
      out.writeLong(processingTimeTimer);

      byte[] serializedState = InstantiationUtil.serializeObject(state);
      out.writeInt(serializedState.length);
      out.write(serializedState, 0, serializedState.length);

      MultiplexingStreamRecordSerializer<IN> recordSerializer =
          new MultiplexingStreamRecordSerializer<>(inputSerializer);
      out.writeInt(windowBuffer.size());
      for (StreamRecord<IN> element : windowBuffer.getElements()) {
        recordSerializer.serialize(element, out);
      }
    }
Пример #6
0
    /**
     * Constructs a new {@code Context} by reading from a {@link DataInputView} that contains a
     * serialized context that we wrote in {@link
     * #writeToState(StateBackend.CheckpointStateOutputView)}
     */
    @SuppressWarnings("unchecked")
    protected Context(DataInputView in, ClassLoader userClassloader) throws Exception {
      this.key = keySerializer.deserialize(in);
      this.window = windowSerializer.deserialize(in);
      this.watermarkTimer = in.readLong();
      this.processingTimeTimer = in.readLong();

      int stateSize = in.readInt();
      byte[] stateData = new byte[stateSize];
      in.read(stateData);
      state = InstantiationUtil.deserializeObject(stateData, userClassloader);

      this.windowBuffer = windowBufferFactory.create();
      int numElements = in.readInt();
      MultiplexingStreamRecordSerializer<IN> recordSerializer =
          new MultiplexingStreamRecordSerializer<>(inputSerializer);
      for (int i = 0; i < numElements; i++) {
        windowBuffer.storeElement(recordSerializer.deserialize(in).<IN>asRecord());
      }
    }
Пример #7
0
    private void writeKVStateMetaData() throws IOException, InterruptedException {
      // write number of k/v states
      outputView.writeInt(stateBackend.kvStateInformation.size());

      int kvStateId = 0;
      // iterate all column families, where each column family holds one k/v state, to write the
      // metadata
      for (Map.Entry<String, Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>>> column :
          stateBackend.kvStateInformation.entrySet()) {

        // be cooperative and check for interruption from time to time in the hot loop
        checkInterrupted();

        // write StateDescriptor for this k/v state
        InstantiationUtil.serializeObject(outStream, column.getValue().f1);

        // retrieve iterator for this k/v states
        readOptions = new ReadOptions();
        readOptions.setSnapshot(snapshot);
        RocksIterator iterator = stateBackend.db.newIterator(column.getValue().f0, readOptions);
        kvStateIterators.add(new Tuple2<>(iterator, kvStateId));
        ++kvStateId;
      }
    }
Пример #8
0
  @Override
  public void invoke() throws Exception {
    if (LOG.isDebugEnabled()) {
      LOG.debug(getLogString("Starting data sink operator"));
    }

    ExecutionConfig executionConfig;
    try {
      ExecutionConfig c =
          (ExecutionConfig)
              InstantiationUtil.readObjectFromConfig(
                  getJobConfiguration(), ExecutionConfig.CONFIG_KEY, getUserCodeClassLoader());
      if (c != null) {
        executionConfig = c;
      } else {
        LOG.warn("The execution config returned by the configuration was null");
        executionConfig = new ExecutionConfig();
      }
    } catch (IOException e) {
      throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: " + e);
    } catch (ClassNotFoundException e) {
      throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: " + e);
    }
    boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();

    try {

      // initialize local strategies
      MutableObjectIterator<IT> input1;
      switch (this.config.getInputLocalStrategy(0)) {
        case NONE:
          // nothing to do
          localStrategy = null;
          input1 = reader;
          break;
        case SORT:
          // initialize sort local strategy
          try {
            // get type comparator
            TypeComparatorFactory<IT> compFact =
                this.config.getInputComparator(0, getUserCodeClassLoader());
            if (compFact == null) {
              throw new Exception("Missing comparator factory for local strategy on input " + 0);
            }

            // initialize sorter
            UnilateralSortMerger<IT> sorter =
                new UnilateralSortMerger<IT>(
                    getEnvironment().getMemoryManager(),
                    getEnvironment().getIOManager(),
                    this.reader,
                    this,
                    this.inputTypeSerializerFactory,
                    compFact.createComparator(),
                    this.config.getRelativeMemoryInput(0),
                    this.config.getFilehandlesInput(0),
                    this.config.getSpillingThresholdInput(0));

            this.localStrategy = sorter;
            input1 = sorter.getIterator();
          } catch (Exception e) {
            throw new RuntimeException(
                "Initializing the input processing failed"
                    + (e.getMessage() == null ? "." : ": " + e.getMessage()),
                e);
          }
          break;
        default:
          throw new RuntimeException("Invalid local strategy for DataSinkTask");
      }

      // read the reader and write it to the output

      final TypeSerializer<IT> serializer = this.inputTypeSerializerFactory.getSerializer();
      final MutableObjectIterator<IT> input = input1;
      final OutputFormat<IT> format = this.format;

      // check if task has been canceled
      if (this.taskCanceled) {
        return;
      }

      if (LOG.isDebugEnabled()) {
        LOG.debug(getLogString("Starting to produce output"));
      }

      // open
      format.open(
          this.getEnvironment().getIndexInSubtaskGroup(),
          this.getEnvironment().getNumberOfSubtasks());

      if (objectReuseEnabled) {
        IT record = serializer.createInstance();

        // work!
        while (!this.taskCanceled && ((record = input.next(record)) != null)) {
          format.writeRecord(record);
        }
      } else {
        IT record;

        // work!
        while (!this.taskCanceled && ((record = input.next()) != null)) {
          format.writeRecord(record);
        }
      }

      // close. We close here such that a regular close throwing an exception marks a task as
      // failed.
      if (!this.taskCanceled) {
        this.format.close();
        this.format = null;
      }
    } catch (Exception ex) {

      // make a best effort to clean up
      try {
        if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) {
          cleanupCalled = true;
          ((CleanupWhenUnsuccessful) format).tryCleanupOnError();
        }
      } catch (Throwable t) {
        LOG.error("Cleanup on error failed.", t);
      }

      ex = ExceptionInChainedStubException.exceptionUnwrap(ex);

      if (ex instanceof CancelTaskException) {
        // forward canceling exception
        throw ex;
      }
      // drop, if the task was canceled
      else if (!this.taskCanceled) {
        if (LOG.isErrorEnabled()) {
          LOG.error(getLogString("Error in user code: " + ex.getMessage()), ex);
        }
        throw ex;
      }
    } finally {
      if (this.format != null) {
        // close format, if it has not been closed, yet.
        // This should only be the case if we had a previous error, or were canceled.
        try {
          this.format.close();
        } catch (Throwable t) {
          if (LOG.isWarnEnabled()) {
            LOG.warn(getLogString("Error closing the output format"), t);
          }
        }
      }
      // close local strategy if necessary
      if (localStrategy != null) {
        try {
          this.localStrategy.close();
        } catch (Throwable t) {
          LOG.error("Error closing local strategy", t);
        }
      }

      RegularPactTask.clearReaders(new MutableReader[] {inputReader});
    }

    if (!this.taskCanceled) {
      if (LOG.isDebugEnabled()) {
        LOG.debug(getLogString("Finished data sink operator"));
      }
    } else {
      if (LOG.isDebugEnabled()) {
        LOG.debug(getLogString("Data sink operator cancelled"));
      }
    }
  }
Пример #9
0
  /** For backwards compatibility, remove again later! */
  @Deprecated
  private void restoreOldSavepointKeyedState(Collection<KeyGroupsStateHandle> restoreState)
      throws Exception {

    if (restoreState.isEmpty()) {
      return;
    }

    Preconditions.checkState(1 == restoreState.size(), "Only one element expected here.");
    HashMap<String, RocksDBStateBackend.FinalFullyAsyncSnapshot> namedStates =
        InstantiationUtil.deserializeObject(
            restoreState.iterator().next().openInputStream(), userCodeClassLoader);

    Preconditions.checkState(1 == namedStates.size(), "Only one element expected here.");
    DataInputView inputView =
        namedStates.values().iterator().next().stateHandle.getState(userCodeClassLoader);

    // clear k/v state information before filling it
    kvStateInformation.clear();

    // first get the column family mapping
    int numColumns = inputView.readInt();
    Map<Byte, StateDescriptor> columnFamilyMapping = new HashMap<>(numColumns);
    for (int i = 0; i < numColumns; i++) {
      byte mappingByte = inputView.readByte();

      ObjectInputStream ooIn =
          new InstantiationUtil.ClassLoaderObjectInputStream(
              new DataInputViewStream(inputView), userCodeClassLoader);

      StateDescriptor stateDescriptor = (StateDescriptor) ooIn.readObject();

      columnFamilyMapping.put(mappingByte, stateDescriptor);

      // this will fill in the k/v state information
      getColumnFamily(stateDescriptor);
    }

    // try and read until EOF
    try {
      // the EOFException will get us out of this...
      while (true) {
        byte mappingByte = inputView.readByte();
        ColumnFamilyHandle handle = getColumnFamily(columnFamilyMapping.get(mappingByte));
        byte[] keyAndNamespace = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);

        ByteArrayInputStreamWithPos bis = new ByteArrayInputStreamWithPos(keyAndNamespace);

        K reconstructedKey = keySerializer.deserialize(new DataInputViewStreamWrapper(bis));
        int len = bis.getPosition();

        int keyGroup =
            (byte) KeyGroupRangeAssignment.assignToKeyGroup(reconstructedKey, numberOfKeyGroups);

        if (keyGroupPrefixBytes == 1) {
          // copy and override one byte (42) between key and namespace
          System.arraycopy(keyAndNamespace, 0, keyAndNamespace, 1, len);
          keyAndNamespace[0] = (byte) keyGroup;
        } else {
          byte[] largerKey = new byte[1 + keyAndNamespace.length];

          // write key-group
          largerKey[0] = (byte) ((keyGroup >> 8) & 0xFF);
          largerKey[1] = (byte) (keyGroup & 0xFF);

          // write key
          System.arraycopy(keyAndNamespace, 0, largerKey, 2, len);

          // skip one byte (42), write namespace
          System.arraycopy(
              keyAndNamespace, 1 + len, largerKey, 2 + len, keyAndNamespace.length - len - 1);
          keyAndNamespace = largerKey;
        }

        byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);
        db.put(handle, keyAndNamespace, value);
      }
    } catch (EOFException e) {
      // expected
    }
  }
 /**
  * Writes to the output stream the error return code, and the given exception in serialized form.
  *
  * @param out Thr output stream to write to.
  * @param t The exception to send.
  * @throws IOException Thrown, if the output stream could not be written to.
  */
 private static void writeErrorToStream(OutputStream out, Throwable t) throws IOException {
   byte[] bytes = InstantiationUtil.serializeObject(t);
   out.write(RETURN_ERROR);
   writeLength(bytes.length, out);
   out.write(bytes);
 }