private void flushBufferedData() throws IOException {
    if (o == 0) {
      return;
    }
    checksum.reset();
    checksum.update(buffer, 0, o);
    final int check = (int) checksum.getValue();
    int compressedLength = compressor.compress(buffer, 0, o, compressedBuffer, HEADER_LENGTH);
    final int compressMethod;
    if (compressedLength >= o) {
      compressMethod = COMPRESSION_METHOD_RAW;
      compressedLength = o;
      System.arraycopy(buffer, 0, compressedBuffer, HEADER_LENGTH, o);
    } else {
      compressMethod = COMPRESSION_METHOD_LZ4;
    }

    compressedBuffer[MAGIC_LENGTH] = (byte) (compressMethod | compressionLevel);
    writeIntLE(compressedLength, compressedBuffer, MAGIC_LENGTH + 1);
    writeIntLE(o, compressedBuffer, MAGIC_LENGTH + 5);
    writeIntLE(check, compressedBuffer, MAGIC_LENGTH + 9);
    assert MAGIC_LENGTH + 13 == HEADER_LENGTH;
    out.write(compressedBuffer, 0, HEADER_LENGTH + compressedLength);
    o = 0;
  }
  private void decompress(byte[] compressed) throws IOException {
    // uncompress
    validBufferBytes =
        info.parameters.sstableCompressor.uncompress(
            compressed, 0, compressed.length - checksumBytes.length, buffer, 0);
    uncompressedBytes += validBufferBytes;

    // validate crc randomly
    if (info.parameters.getCrcCheckChance() > FBUtilities.threadLocalRandom().nextDouble()) {
      checksum.update(buffer, 0, validBufferBytes);

      System.arraycopy(
          compressed,
          compressed.length - checksumBytes.length,
          checksumBytes,
          0,
          checksumBytes.length);
      if (Ints.fromByteArray(checksumBytes) != (int) checksum.getValue())
        throw new IOException("CRC unmatched");

      // reset checksum object back to the original (blank) state
      checksum.reset();
    }

    // buffer offset is always aligned
    bufferOffset = current & ~(buffer.length - 1);
  }
Exemple #3
0
  public SpillRecord(Path indexFileName, JobConf job, Checksum crc, String expectedIndexOwner)
      throws IOException {

    final FileSystem rfs = FileSystem.getLocal(job).getRaw();
    final DataInputStream in =
        new DataInputStream(
            SecureIOUtils.openForRead(
                new File(indexFileName.toUri().getPath()), expectedIndexOwner, null));
    try {
      final long length = rfs.getFileStatus(indexFileName).getLen();
      final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;
      final int size = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH;

      buf = ByteBuffer.allocate(size);
      if (crc != null) {
        crc.reset();
        CheckedInputStream chk = new CheckedInputStream(in, crc);
        IOUtils.readFully(chk, buf.array(), 0, size);
        if (chk.getChecksum().getValue() != in.readLong()) {
          throw new ChecksumException("Checksum error reading spill index: " + indexFileName, -1);
        }
      } else {
        IOUtils.readFully(in, buf.array(), 0, size);
      }
      entries = buf.asLongBuffer();
    } finally {
      in.close();
    }
  }
Exemple #4
0
 public void writeToFile(Path loc, JobConf job, Checksum crc) throws IOException {
   final FileSystem rfs = FileSystem.getLocal(job).getRaw();
   CheckedOutputStream chk = null;
   final FSDataOutputStream out = rfs.create(loc);
   try {
     if (crc != null) {
       crc.reset();
       chk = new CheckedOutputStream(out, crc);
       chk.write(buf.array());
       out.writeLong(chk.getChecksum().getValue());
     } else {
       out.write(buf.array());
     }
   } finally {
     if (chk != null) {
       chk.close();
     } else {
       out.close();
     }
   }
 }
Exemple #5
0
  public void recover(File file) throws IOException {
    logger.info("Replaying " + file.getPath());
    final long segment = CommitLogSegment.idFromFilename(file.getName());
    RandomAccessReader reader = RandomAccessReader.open(new File(file.getAbsolutePath()), true);
    try {
      assert reader.length() <= Integer.MAX_VALUE;
      int replayPosition;
      if (globalPosition.segment < segment) replayPosition = 0;
      else if (globalPosition.segment == segment) replayPosition = globalPosition.position;
      else replayPosition = (int) reader.length();

      if (replayPosition < 0 || replayPosition >= reader.length()) {
        // replayPosition > reader.length() can happen if some data gets flushed before it is
        // written to the commitlog
        // (see https://issues.apache.org/jira/browse/CASSANDRA-2285)
        logger.debug("skipping replay of fully-flushed {}", file);
        return;
      }

      reader.seek(replayPosition);

      if (logger.isDebugEnabled())
        logger.debug("Replaying " + file + " starting at " + reader.getFilePointer());

      /* read the logs populate RowMutation and apply */
      while (!reader.isEOF()) {
        if (logger.isDebugEnabled()) logger.debug("Reading mutation at " + reader.getFilePointer());

        long claimedCRC32;
        int serializedSize;
        try {
          // any of the reads may hit EOF
          serializedSize = reader.readInt();
          if (serializedSize == CommitLog.END_OF_SEGMENT_MARKER) {
            logger.debug("Encountered end of segment marker at " + reader.getFilePointer());
            break;
          }

          // RowMutation must be at LEAST 10 bytes:
          // 3 each for a non-empty Table and Key (including the
          // 2-byte length from writeUTF/writeWithShortLength) and 4 bytes for column count.
          // This prevents CRC by being fooled by special-case garbage in the file; see
          // CASSANDRA-2128
          if (serializedSize < 10) break;
          long claimedSizeChecksum = reader.readLong();
          checksum.reset();
          checksum.update(serializedSize);
          if (checksum.getValue() != claimedSizeChecksum)
            break; // entry wasn't synced correctly/fully. that's
          // ok.

          if (serializedSize > buffer.length) buffer = new byte[(int) (1.2 * serializedSize)];
          reader.readFully(buffer, 0, serializedSize);
          claimedCRC32 = reader.readLong();
        } catch (EOFException eof) {
          break; // last CL entry didn't get completely written. that's ok.
        }

        checksum.update(buffer, 0, serializedSize);
        if (claimedCRC32 != checksum.getValue()) {
          // this entry must not have been fsynced. probably the rest is bad too,
          // but just in case there is no harm in trying them (since we still read on an entry
          // boundary)
          continue;
        }

        /* deserialize the commit log entry */
        FastByteArrayInputStream bufIn = new FastByteArrayInputStream(buffer, 0, serializedSize);
        RowMutation rm;
        try {
          // assuming version here. We've gone to lengths to make sure what gets written to the CL
          // is in
          // the current version. so do make sure the CL is drained prior to upgrading a node.
          rm =
              RowMutation.serializer()
                  .deserialize(
                      new DataInputStream(bufIn),
                      MessagingService.version_,
                      IColumnSerializer.Flag.LOCAL);
        } catch (UnknownColumnFamilyException ex) {
          AtomicInteger i = invalidMutations.get(ex.cfId);
          if (i == null) {
            i = new AtomicInteger(1);
            invalidMutations.put(ex.cfId, i);
          } else i.incrementAndGet();
          continue;
        }

        if (logger.isDebugEnabled())
          logger.debug(
              String.format(
                  "replaying mutation for %s.%s: %s",
                  rm.getTable(),
                  ByteBufferUtil.bytesToHex(rm.key()),
                  "{" + StringUtils.join(rm.getColumnFamilies().iterator(), ", ") + "}"));

        final long entryLocation = reader.getFilePointer();
        final RowMutation frm = rm;
        Runnable runnable =
            new WrappedRunnable() {
              public void runMayThrow() throws IOException {
                if (Schema.instance.getKSMetaData(frm.getTable()) == null) return;
                if (pointInTimeExceeded(frm)) return;

                final Table table = Table.open(frm.getTable());
                RowMutation newRm = new RowMutation(frm.getTable(), frm.key());

                // Rebuild the row mutation, omitting column families that
                // a) have already been flushed,
                // b) are part of a cf that was dropped. Keep in mind that the cf.name() is suspect.
                // do every thing based on the cfid instead.
                for (ColumnFamily columnFamily : frm.getColumnFamilies()) {
                  if (Schema.instance.getCF(columnFamily.id()) == null)
                    // null means the cf has been dropped
                    continue;

                  ReplayPosition rp = cfPositions.get(columnFamily.id());

                  // replay if current segment is newer than last flushed one or,
                  // if it is the last known segment, if we are after the replay position
                  if (segment > rp.segment
                      || (segment == rp.segment && entryLocation > rp.position)) {
                    newRm.add(columnFamily);
                    replayedCount.incrementAndGet();
                  }
                }
                if (!newRm.isEmpty()) {
                  Table.open(newRm.getTable()).apply(newRm, false);
                  tablesRecovered.add(table);
                }
              }
            };
        futures.add(StageManager.getStage(Stage.MUTATION).submit(runnable));
        if (futures.size() > MAX_OUTSTANDING_REPLAY_COUNT) {
          FBUtilities.waitOnFutures(futures);
          futures.clear();
        }
      }
    } finally {
      FileUtils.closeQuietly(reader);
      logger.info("Finished reading " + file);
    }
  }
Exemple #6
0
 /**
  * Calculates a CRC32 checksum of the specified data.
  *
  * @param data - data to be hashed.
  * @return - a CRC32 checksum (4 bytes).
  */
 private int crc(byte[] data) {
   checksum.reset();
   checksum.update(data, 0, data.length);
   return (int) checksum.getValue();
 }
Exemple #7
0
 public void reset() {
   summer.reset();
   inSum = 0;
 }