@Override public String toString() { return String.format( "CounterCacheKey(%s, %s, %s)", cfId, ByteBufferUtil.bytesToHex(ByteBuffer.wrap(partitionKey)), ByteBufferUtil.bytesToHex(ByteBuffer.wrap(cellName))); }
@Override public void handleUser(Message<JsonObject> event) { Integer id = event.body.getInteger("id"); JsonObject params = event.body.getObject("op"); JsonObject state = event.body.getObject("state"); JsonObject meta = state.getObject("meta"); JsonObject metaColumn = state.getObject("metaColumn"); JsonObject metaRanged = state.getObject("metaRanged"); if (meta == null) { meta = new JsonObject(); } if (metaColumn == null) { metaColumn = new JsonObject(); } if (metaRanged == null) { metaRanged = new JsonObject(); } if (params.getString("type") != null) { StringBuilder key = new StringBuilder(); key.append(HandlerUtils.instance.determineKs(params, state, null)); key.append(' '); key.append(HandlerUtils.instance.determineCf(params, state, null)); key.append(' '); key.append(params.getString("type")); meta.putObject( key.toString(), new JsonObject().putString("clazz", params.getString("clazz"))); state.putObject("meta", meta); event.reply(new JsonObject().putString(id.toString(), "OK").putObject("state", state)); } else if (params.getField("name") != null) { Object o = params.getField("name"); ByteBuffer bb = HandlerUtils.instance.byteBufferForObject(o); StringBuilder key = new StringBuilder(); key.append(HandlerUtils.instance.determineKs(params, state, null)); key.append(' '); key.append(HandlerUtils.instance.determineCf(params, state, null)); key.append(' '); key.append(ByteBufferUtil.bytesToHex(bb)); metaColumn.putObject( key.toString(), new JsonObject().putString("clazz", params.getString("clazz"))); state.putObject("metaColumn", metaColumn); event.reply(new JsonObject().putString(id.toString(), "OK").putObject("state", state)); } else if (params.getField(Operations.RANGE_START) != null) { Object start = params.getField(Operations.RANGE_START); Object end = params.getField(Operations.RANGE_END); ByteBuffer sbuf = HandlerUtils.instance.byteBufferForObject(start); ByteBuffer ebuf = HandlerUtils.instance.byteBufferForObject(end); StringBuilder key = new StringBuilder(); key.append(HandlerUtils.instance.determineKs(params, state, null)); key.append(' '); key.append(HandlerUtils.instance.determineCf(params, state, null)); key.append(' '); key.append(ByteBufferUtil.bytesToHex(sbuf)); JsonObject value = new JsonObject(); value.putString("clazz", params.getString("clazz")); value.putString("end", ByteBufferUtil.bytesToHex(ebuf)); metaRanged.putObject(key.toString(), value); state.putObject("metaRanged", metaRanged); event.reply(new JsonObject().putString(id.toString(), "OK").putObject("state", state)); } else { throw new RuntimeException("hit bottom this is bad ok"); } }
public static void recover(File[] clogs) throws IOException { Set<Table> tablesRecovered = new HashSet<Table>(); List<Future<?>> futures = new ArrayList<Future<?>>(); byte[] bytes = new byte[4096]; Map<Integer, AtomicInteger> invalidMutations = new HashMap<Integer, AtomicInteger>(); for (File file : clogs) { int bufferSize = (int) Math.min(Math.max(file.length(), 1), 32 * 1024 * 1024); BufferedRandomAccessFile reader = new BufferedRandomAccessFile(new File(file.getAbsolutePath()), "r", bufferSize, true); try { CommitLogHeader clHeader = null; int replayPosition = 0; String headerPath = CommitLogHeader.getHeaderPathFromSegmentPath(file.getAbsolutePath()); try { clHeader = CommitLogHeader.readCommitLogHeader(headerPath); replayPosition = clHeader.getReplayPosition(); } catch (IOException ioe) { logger.info( headerPath + " incomplete, missing or corrupt. Everything is ok, don't panic. CommitLog will be replayed from the beginning"); logger.debug("exception was", ioe); } if (replayPosition < 0) { logger.debug("skipping replay of fully-flushed {}", file); continue; } reader.seek(replayPosition); if (logger.isDebugEnabled()) logger.debug("Replaying " + file + " starting at " + reader.getFilePointer()); /* read the logs populate RowMutation and apply */ while (!reader.isEOF()) { if (logger.isDebugEnabled()) logger.debug("Reading mutation at " + reader.getFilePointer()); long claimedCRC32; Checksum checksum = new CRC32(); int serializedSize; try { // any of the reads may hit EOF serializedSize = reader.readInt(); // RowMutation must be at LEAST 10 bytes: // 3 each for a non-empty Table and Key (including the 2-byte length from // writeUTF/writeWithShortLength) and 4 bytes for column count. // This prevents CRC by being fooled by special-case garbage in the file; see // CASSANDRA-2128 if (serializedSize < 10) break; long claimedSizeChecksum = reader.readLong(); checksum.update(serializedSize); if (checksum.getValue() != claimedSizeChecksum) break; // entry wasn't synced correctly/fully. that's ok. if (serializedSize > bytes.length) bytes = new byte[(int) (1.2 * serializedSize)]; reader.readFully(bytes, 0, serializedSize); claimedCRC32 = reader.readLong(); } catch (EOFException eof) { break; // last CL entry didn't get completely written. that's ok. } checksum.update(bytes, 0, serializedSize); if (claimedCRC32 != checksum.getValue()) { // this entry must not have been fsynced. probably the rest is bad too, // but just in case there is no harm in trying them (since we still read on an entry // boundary) continue; } /* deserialize the commit log entry */ ByteArrayInputStream bufIn = new ByteArrayInputStream(bytes, 0, serializedSize); RowMutation rm = null; try { rm = RowMutation.serializer().deserialize(new DataInputStream(bufIn)); } catch (UnserializableColumnFamilyException ex) { AtomicInteger i = invalidMutations.get(ex.cfId); if (i == null) { i = new AtomicInteger(1); invalidMutations.put(ex.cfId, i); } else i.incrementAndGet(); continue; } if (logger.isDebugEnabled()) logger.debug( String.format( "replaying mutation for %s.%s: %s", rm.getTable(), ByteBufferUtil.bytesToHex(rm.key()), "{" + StringUtils.join(rm.getColumnFamilies(), ", ") + "}")); final Table table = Table.open(rm.getTable()); tablesRecovered.add(table); final Collection<ColumnFamily> columnFamilies = new ArrayList<ColumnFamily>(rm.getColumnFamilies()); final long entryLocation = reader.getFilePointer(); final CommitLogHeader finalHeader = clHeader; final RowMutation frm = rm; Runnable runnable = new WrappedRunnable() { public void runMayThrow() throws IOException { RowMutation newRm = new RowMutation(frm.getTable(), frm.key()); // Rebuild the row mutation, omitting column families that a) have already been // flushed, // b) are part of a cf that was dropped. Keep in mind that the cf.name() is // suspect. do every // thing based on the cfid instead. for (ColumnFamily columnFamily : columnFamilies) { if (CFMetaData.getCF(columnFamily.id()) == null) // null means the cf has been dropped continue; if (finalHeader == null || (finalHeader.isDirty(columnFamily.id()) && entryLocation > finalHeader.getPosition(columnFamily.id()))) newRm.add(columnFamily); } if (!newRm.isEmpty()) { Table.open(newRm.getTable()).apply(newRm, null, false); } } }; futures.add(StageManager.getStage(Stage.MUTATION).submit(runnable)); if (futures.size() > MAX_OUTSTANDING_REPLAY_COUNT) { FBUtilities.waitOnFutures(futures); futures.clear(); } } } finally { FileUtils.closeQuietly(reader); logger.info("Finished reading " + file); } } for (Map.Entry<Integer, AtomicInteger> entry : invalidMutations.entrySet()) logger.info( String.format( "Skipped %d mutations from unknown (probably removed) CF with id %d", entry.getValue().intValue(), entry.getKey())); // wait for all the writes to finish on the mutation stage FBUtilities.waitOnFutures(futures); logger.debug("Finished waiting on mutations from recovery"); // flush replayed tables futures.clear(); for (Table table : tablesRecovered) futures.addAll(table.flush()); FBUtilities.waitOnFutures(futures); }
/** * Returns the hexadecimal {@code String} representation of the specified {@link ByteBuffer}. * * @param byteBuffer A {@link ByteBuffer}. * @return The hexadecimal {@code String} representation of the specified {@link ByteBuffer}. */ public static String toHex(ByteBuffer byteBuffer) { return ByteBufferUtil.bytesToHex(byteBuffer); }
public void recover(File file) throws IOException { logger.info("Replaying " + file.getPath()); final long segment = CommitLogSegment.idFromFilename(file.getName()); RandomAccessReader reader = RandomAccessReader.open(new File(file.getAbsolutePath()), true); try { assert reader.length() <= Integer.MAX_VALUE; int replayPosition; if (globalPosition.segment < segment) replayPosition = 0; else if (globalPosition.segment == segment) replayPosition = globalPosition.position; else replayPosition = (int) reader.length(); if (replayPosition < 0 || replayPosition >= reader.length()) { // replayPosition > reader.length() can happen if some data gets flushed before it is // written to the commitlog // (see https://issues.apache.org/jira/browse/CASSANDRA-2285) logger.debug("skipping replay of fully-flushed {}", file); return; } reader.seek(replayPosition); if (logger.isDebugEnabled()) logger.debug("Replaying " + file + " starting at " + reader.getFilePointer()); /* read the logs populate RowMutation and apply */ while (!reader.isEOF()) { if (logger.isDebugEnabled()) logger.debug("Reading mutation at " + reader.getFilePointer()); long claimedCRC32; int serializedSize; try { // any of the reads may hit EOF serializedSize = reader.readInt(); if (serializedSize == CommitLog.END_OF_SEGMENT_MARKER) { logger.debug("Encountered end of segment marker at " + reader.getFilePointer()); break; } // RowMutation must be at LEAST 10 bytes: // 3 each for a non-empty Table and Key (including the // 2-byte length from writeUTF/writeWithShortLength) and 4 bytes for column count. // This prevents CRC by being fooled by special-case garbage in the file; see // CASSANDRA-2128 if (serializedSize < 10) break; long claimedSizeChecksum = reader.readLong(); checksum.reset(); checksum.update(serializedSize); if (checksum.getValue() != claimedSizeChecksum) break; // entry wasn't synced correctly/fully. that's // ok. if (serializedSize > buffer.length) buffer = new byte[(int) (1.2 * serializedSize)]; reader.readFully(buffer, 0, serializedSize); claimedCRC32 = reader.readLong(); } catch (EOFException eof) { break; // last CL entry didn't get completely written. that's ok. } checksum.update(buffer, 0, serializedSize); if (claimedCRC32 != checksum.getValue()) { // this entry must not have been fsynced. probably the rest is bad too, // but just in case there is no harm in trying them (since we still read on an entry // boundary) continue; } /* deserialize the commit log entry */ FastByteArrayInputStream bufIn = new FastByteArrayInputStream(buffer, 0, serializedSize); RowMutation rm; try { // assuming version here. We've gone to lengths to make sure what gets written to the CL // is in // the current version. so do make sure the CL is drained prior to upgrading a node. rm = RowMutation.serializer() .deserialize( new DataInputStream(bufIn), MessagingService.version_, IColumnSerializer.Flag.LOCAL); } catch (UnknownColumnFamilyException ex) { AtomicInteger i = invalidMutations.get(ex.cfId); if (i == null) { i = new AtomicInteger(1); invalidMutations.put(ex.cfId, i); } else i.incrementAndGet(); continue; } if (logger.isDebugEnabled()) logger.debug( String.format( "replaying mutation for %s.%s: %s", rm.getTable(), ByteBufferUtil.bytesToHex(rm.key()), "{" + StringUtils.join(rm.getColumnFamilies().iterator(), ", ") + "}")); final long entryLocation = reader.getFilePointer(); final RowMutation frm = rm; Runnable runnable = new WrappedRunnable() { public void runMayThrow() throws IOException { if (Schema.instance.getKSMetaData(frm.getTable()) == null) return; if (pointInTimeExceeded(frm)) return; final Table table = Table.open(frm.getTable()); RowMutation newRm = new RowMutation(frm.getTable(), frm.key()); // Rebuild the row mutation, omitting column families that // a) have already been flushed, // b) are part of a cf that was dropped. Keep in mind that the cf.name() is suspect. // do every thing based on the cfid instead. for (ColumnFamily columnFamily : frm.getColumnFamilies()) { if (Schema.instance.getCF(columnFamily.id()) == null) // null means the cf has been dropped continue; ReplayPosition rp = cfPositions.get(columnFamily.id()); // replay if current segment is newer than last flushed one or, // if it is the last known segment, if we are after the replay position if (segment > rp.segment || (segment == rp.segment && entryLocation > rp.position)) { newRm.add(columnFamily); replayedCount.incrementAndGet(); } } if (!newRm.isEmpty()) { Table.open(newRm.getTable()).apply(newRm, false); tablesRecovered.add(table); } } }; futures.add(StageManager.getStage(Stage.MUTATION).submit(runnable)); if (futures.size() > MAX_OUTSTANDING_REPLAY_COUNT) { FBUtilities.waitOnFutures(futures); futures.clear(); } } } finally { FileUtils.closeQuietly(reader); logger.info("Finished reading " + file); } }
public String getString(ByteBuffer bytes) { return ByteBufferUtil.bytesToHex(bytes); }
private static String getUnQuotedCqlBlob(ByteBuffer term, boolean isCQL3) { return isCQL3 ? "0x" + ByteBufferUtil.bytesToHex(term) : ByteBufferUtil.bytesToHex(term); }