Beispiel #1
0
  public long writeCommit(CommitUpdateCommand cmd, int flags) {
    LogCodec codec = new LogCodec(resolver);
    synchronized (this) {
      try {
        long pos = fos.size(); // if we had flushed, this should be equal to channel.position()

        if (pos == 0) {
          writeLogHeader(codec);
          pos = fos.size();
        }
        codec.init(fos);
        codec.writeTag(JavaBinCodec.ARR, 3);
        codec.writeInt(UpdateLog.COMMIT | flags); // should just take one byte
        codec.writeLong(cmd.getVersion());
        codec.writeStr(END_MESSAGE); // ensure these bytes are (almost) last in the file

        endRecord(pos);

        fos.flush(); // flush since this will be the last record in a log fill
        assert fos.size() == channel.size();

        return pos;
      } catch (IOException e) {
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
      }
    }
  }
Beispiel #2
0
 public static void writeVLong(long i, FastOutputStream out) throws IOException {
   while ((i & ~0x7F) != 0) {
     out.writeByte((byte) ((i & 0x7f) | 0x80));
     i >>>= 7;
   }
   out.writeByte((byte) i);
 }
Beispiel #3
0
  protected void close() {
    try {
      if (debug) {
        log.debug("Closing tlog" + this);
      }

      synchronized (this) {
        fos.flush();
        fos.close();
      }

      if (deleteOnClose) {
        try {
          Files.deleteIfExists(tlogFile.toPath());
        } catch (IOException e) {
          // TODO: should this class care if a file couldnt be deleted?
          // this just emulates previous behavior, where only SecurityException would be handled.
        }
      }
    } catch (IOException e) {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
    } finally {
      assert ObjectReleaseTracker.release(this);
    }
  }
Beispiel #4
0
 // This could mess with any readers or reverse readers that are open, or anything that might try
 // to do a log lookup.
 // This should only be used to roll back buffered updates, not actually applied updates.
 public void rollback(long pos) throws IOException {
   synchronized (this) {
     assert snapshot_size == pos;
     fos.flush();
     raf.setLength(pos);
     fos.setWritten(pos);
     assert fos.size() == pos;
     numRecords = snapshot_numRecords;
   }
 }
Beispiel #5
0
 public void marshal(Object nl, OutputStream os) throws IOException {
   assert !alreadyMarshalled;
   init(FastOutputStream.wrap(os));
   try {
     daos.writeByte(VERSION);
     writeVal(nl);
   } finally {
     daos.flushBuffer();
     alreadyMarshalled = true;
   }
 }
Beispiel #6
0
 public void writeTag(byte tag, int size) throws IOException {
   if ((tag & 0xe0) != 0) {
     if (size < 0x1f) {
       daos.writeByte(tag | size);
     } else {
       daos.writeByte(tag | 0x1f);
       writeVInt(size - 0x1f, daos);
     }
   } else {
     daos.writeByte(tag);
     writeVInt(size, daos);
   }
 }
Beispiel #7
0
  private void checkWriteHeader(LogCodec codec, SolrInputDocument optional) throws IOException {

    // Unsynchronized access.  We can get away with an unsynchronized access here
    // since we will never get a false non-zero when the position is in fact 0.
    // rollback() is the only function that can reset to zero, and it blocks updates.
    if (fos.size() != 0) return;

    synchronized (this) {
      if (fos.size() != 0) return; // check again while synchronized
      if (optional != null) {
        addGlobalStrings(optional.getFieldNames());
      }
      writeLogHeader(codec);
    }
  }
Beispiel #8
0
 public void writeLong(long val) throws IOException {
   if ((val & 0xff00000000000000L) == 0) {
     int b = SLONG | ((int) val & 0x0f);
     if (val >= 0x0f) {
       b |= 0x10;
       daos.writeByte(b);
       writeVLong(val >>> 4, daos);
     } else {
       daos.writeByte(b);
     }
   } else {
     daos.writeByte(LONG);
     daos.writeLong(val);
   }
 }
Beispiel #9
0
  public long writeDelete(DeleteUpdateCommand cmd, int flags) {
    LogCodec codec = new LogCodec(resolver);

    try {
      checkWriteHeader(codec, null);

      BytesRef br = cmd.getIndexedId();

      MemOutputStream out = new MemOutputStream(new byte[20 + br.length]);
      codec.init(out);
      codec.writeTag(JavaBinCodec.ARR, 3);
      codec.writeInt(UpdateLog.DELETE | flags); // should just take one byte
      codec.writeLong(cmd.getVersion());
      codec.writeByteArray(br.bytes, br.offset, br.length);

      synchronized (this) {
        long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
        assert pos != 0;
        out.writeAll(fos);
        endRecord(pos);
        // fos.flushBuffer();  // flush later
        return pos;
      }

    } catch (IOException e) {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
    }
  }
Beispiel #10
0
 /**
  * takes a snapshot of the current position and number of records for later possible rollback, and
  * returns the position
  */
 public long snapshot() {
   synchronized (this) {
     snapshot_size = fos.size();
     snapshot_numRecords = numRecords;
     return snapshot_size;
   }
 }
Beispiel #11
0
  /* This method is thread safe */
  public Object lookup(long pos) {
    // A negative position can result from a log replay (which does not re-log, but does
    // update the version map.  This is OK since the node won't be ACTIVE when this happens.
    if (pos < 0) return null;

    try {
      // make sure any unflushed buffer has been flushed
      synchronized (this) {
        // TODO: optimize this by keeping track of what we have flushed up to
        fos.flushBuffer();
        /**
         * * System.out.println("###flushBuffer to " + fos.size() + " raf.length()=" + raf.length()
         * + " pos="+pos); if (fos.size() != raf.length() || pos >= fos.size() ) { throw new
         * RuntimeException("ERROR" + "###flushBuffer to " + fos.size() + " raf.length()=" +
         * raf.length() + " pos="+pos); } *
         */
      }

      ChannelFastInputStream fis = new ChannelFastInputStream(channel, pos);
      LogCodec codec = new LogCodec(resolver);
      return codec.readVal(fis);
    } catch (IOException e) {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
    }
  }
Beispiel #12
0
  public void writeInt(int val) throws IOException {
    if (val > 0) {
      int b = SINT | (val & 0x0f);

      if (val >= 0x0f) {
        b |= 0x10;
        daos.writeByte(b);
        writeVInt(val >>> 4, daos);
      } else {
        daos.writeByte(b);
      }

    } else {
      daos.writeByte(INT);
      daos.writeInt(val);
    }
  }
Beispiel #13
0
  public boolean endsWithCommit() throws IOException {
    long size;
    synchronized (this) {
      fos.flush();
      size = fos.size();
    }

    // the end of the file should have the end message (added during a commit) plus a 4 byte size
    byte[] buf = new byte[END_MESSAGE.length()];
    long pos = size - END_MESSAGE.length() - 4;
    if (pos < 0) return false;
    ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
    is.read(buf);
    for (int i = 0; i < buf.length; i++) {
      if (buf[i] != END_MESSAGE.charAt(i)) return false;
    }
    return true;
  }
Beispiel #14
0
  protected void writeLogHeader(LogCodec codec) throws IOException {
    long pos = fos.size();
    assert pos == 0;

    Map header = new LinkedHashMap<String, Object>();
    header.put("SOLR_TLOG", 1); // a magic string + version number
    header.put("strings", globalStringList);
    codec.marshal(header, fos);

    endRecord(pos);
  }
Beispiel #15
0
 public long writeData(Object o) {
   LogCodec codec = new LogCodec(resolver);
   try {
     long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
     codec.init(fos);
     codec.writeVal(o);
     return pos;
   } catch (IOException e) {
     throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
   }
 }
  @Test
  public void testgzip() throws Exception {
    ByteArrayOutputStream b = new ByteArrayOutputStream();
    FastOutputStream fos = new FastOutputStream(b);
    GZIPOutputStream gzos = new GZIPOutputStream(fos);
    String ss = "Helloooooooooooooooooooo";
    writeChars(gzos, ss, 0, ss.length());
    gzos.close();
    JavaBinCodec.writeVInt(10, fos);
    fos.flushBuffer();
    GZIPInputStream gzis =
        new GZIPInputStream(new ByteArrayInputStream(b.toByteArray(), 0, b.size()));
    char[] cbuf = new char[ss.length()];
    readChars(gzis, cbuf, 0, ss.length());
    assertEquals(new String(cbuf), ss);
    // System.out.println("passes w/o FastInputStream");

    ByteArrayInputStream bis = new ByteArrayInputStream(b.toByteArray(), 0, b.size());
    gzis = new GZIPInputStream(new FastInputStream(bis));
    cbuf = new char[ss.length()];
    readChars(gzis, cbuf, 0, ss.length());
    assertEquals(new String(cbuf), ss);
    // System.out.println("passes w FastInputStream");
  }
Beispiel #17
0
  public void finish(UpdateLog.SyncLevel syncLevel) {
    if (syncLevel == UpdateLog.SyncLevel.NONE) return;
    try {
      synchronized (this) {
        fos.flushBuffer();
      }

      if (syncLevel == UpdateLog.SyncLevel.FSYNC) {
        // Since fsync is outside of synchronized block, we can end up with a partial
        // last record on power failure (which is OK, and does not represent an error...
        // we just need to be aware of it when reading).
        raf.getFD().sync();
      }

    } catch (IOException e) {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
    }
  }
Beispiel #18
0
  /** write the string as tag+length, with length being the number of UTF-8 bytes */
  public void writeStr(String s) throws IOException {
    if (s == null) {
      writeTag(NULL);
      return;
    }
    int end = s.length();
    int maxSize = end * ByteUtils.MAX_UTF8_BYTES_PER_CHAR;

    if (maxSize <= MAX_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY) {
      if (bytes == null || bytes.length < maxSize) bytes = new byte[maxSize];
      int sz = ByteUtils.UTF16toUTF8(s, 0, end, bytes, 0);
      writeTag(STR, sz);
      daos.write(bytes, 0, sz);
    } else {
      // double pass logic for large strings, see SOLR-7971
      int sz = ByteUtils.calcUTF16toUTF8Length(s, 0, end);
      writeTag(STR, sz);
      if (bytes == null || bytes.length < 8192) bytes = new byte[8192];
      ByteUtils.writeUTF16toUTF8(s, 0, end, daos, bytes);
    }
  }
Beispiel #19
0
  public long write(AddUpdateCommand cmd, int flags) {
    LogCodec codec = new LogCodec(resolver);
    SolrInputDocument sdoc = cmd.getSolrInputDocument();

    try {
      checkWriteHeader(codec, sdoc);

      // adaptive buffer sizing
      int bufSize = lastAddSize; // unsynchronized access of lastAddSize should be fine
      bufSize = Math.min(1024 * 1024, bufSize + (bufSize >> 3) + 256);

      MemOutputStream out = new MemOutputStream(new byte[bufSize]);
      codec.init(out);
      codec.writeTag(JavaBinCodec.ARR, 3);
      codec.writeInt(UpdateLog.ADD | flags); // should just take one byte
      codec.writeLong(cmd.getVersion());
      codec.writeSolrInputDocument(cmd.getSolrInputDocument());
      lastAddSize = (int) out.size();

      synchronized (this) {
        long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
        assert pos != 0;

        /**
         * * System.out.println("###writing at " + pos + " fos.size()=" + fos.size() + "
         * raf.length()=" + raf.length()); if (pos != fos.size()) { throw new
         * RuntimeException("ERROR" + "###writing at " + pos + " fos.size()=" + fos.size() + "
         * raf.length()=" + raf.length()); } *
         */
        out.writeAll(fos);
        endRecord(pos);
        // fos.flushBuffer();  // flush later
        return pos;
      }

    } catch (IOException e) {
      // TODO: reset our file pointer back to "pos", the start of this record.
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error logging add", e);
    }
  }
Beispiel #20
0
  public long writeDeleteByQuery(DeleteUpdateCommand cmd, int flags) {
    LogCodec codec = new LogCodec(resolver);
    try {
      checkWriteHeader(codec, null);

      MemOutputStream out = new MemOutputStream(new byte[20 + (cmd.query.length())]);
      codec.init(out);
      codec.writeTag(JavaBinCodec.ARR, 3);
      codec.writeInt(UpdateLog.DELETE_BY_QUERY | flags); // should just take one byte
      codec.writeLong(cmd.getVersion());
      codec.writeStr(cmd.query);

      synchronized (this) {
        long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
        out.writeAll(fos);
        endRecord(pos);
        // fos.flushBuffer();  // flush later
        return pos;
      }
    } catch (IOException e) {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
    }
  }
Beispiel #21
0
  public boolean writePrimitive(Object val) throws IOException {
    if (val == null) {
      daos.writeByte(NULL);
      return true;
    } else if (val instanceof String) {
      writeStr((String) val);
      return true;
    } else if (val instanceof Number) {

      if (val instanceof Integer) {
        writeInt(((Integer) val).intValue());
        return true;
      } else if (val instanceof Long) {
        writeLong(((Long) val).longValue());
        return true;
      } else if (val instanceof Float) {
        writeFloat(((Float) val).floatValue());
        return true;
      } else if (val instanceof Double) {
        daos.writeByte(DOUBLE);
        daos.writeDouble(((Double) val).doubleValue());
        return true;
      } else if (val instanceof Byte) {
        daos.writeByte(BYTE);
        daos.writeByte(((Byte) val).intValue());
        return true;
      } else if (val instanceof Short) {
        daos.writeByte(SHORT);
        daos.writeShort(((Short) val).intValue());
        return true;
      }
      return false;

    } else if (val instanceof Date) {
      daos.writeByte(DATE);
      daos.writeLong(((Date) val).getTime());
      return true;
    } else if (val instanceof Boolean) {
      if ((Boolean) val) daos.writeByte(BOOL_TRUE);
      else daos.writeByte(BOOL_FALSE);
      return true;
    } else if (val instanceof byte[]) {
      writeByteArray((byte[]) val, 0, ((byte[]) val).length);
      return true;
    } else if (val instanceof ByteBuffer) {
      ByteBuffer buf = (ByteBuffer) val;
      writeByteArray(buf.array(), buf.position(), buf.limit() - buf.position());
      return true;
    } else if (val == END_OBJ) {
      writeTag(END);
      return true;
    }
    return false;
  }
Beispiel #22
0
 public void writeFloat(float val) throws IOException {
   daos.writeByte(FLOAT);
   daos.writeFloat(val);
 }
Beispiel #23
0
 protected void endRecord(long startRecordPosition) throws IOException {
   fos.writeInt((int) (fos.size() - startRecordPosition));
   numRecords++;
 }
Beispiel #24
0
 public void writeByteArray(byte[] arr, int offset, int len) throws IOException {
   writeTag(BYTEARR, len);
   daos.write(arr, offset, len);
 }
Beispiel #25
0
 public void writeTag(byte tag) throws IOException {
   daos.writeByte(tag);
 }
Beispiel #26
0
 /** returns the current position in the log file */
 public long position() {
   synchronized (this) {
     return fos.size();
   }
 }
Beispiel #27
0
  TransactionLog(File tlogFile, Collection<String> globalStrings, boolean openExisting) {
    boolean success = false;
    try {
      if (debug) {
        log.debug(
            "New TransactionLog file="
                + tlogFile
                + ", exists="
                + tlogFile.exists()
                + ", size="
                + tlogFile.length()
                + ", openExisting="
                + openExisting);
      }

      this.tlogFile = tlogFile;
      raf = new RandomAccessFile(this.tlogFile, "rw");
      long start = raf.length();
      channel = raf.getChannel();
      os = Channels.newOutputStream(channel);
      fos = new FastOutputStream(os, new byte[65536], 0);
      // fos = FastOutputStream.wrap(os);

      if (openExisting) {
        if (start > 0) {
          readHeader(null);
          raf.seek(start);
          assert channel.position() == start;
          fos.setWritten(start); // reflect that we aren't starting at the beginning
          assert fos.size() == channel.size();
        } else {
          addGlobalStrings(globalStrings);
        }
      } else {
        if (start > 0) {
          log.warn("New transaction log already exists:" + tlogFile + " size=" + raf.length());
          return;
        }

        if (start > 0) {
          raf.setLength(0);
        }
        addGlobalStrings(globalStrings);
      }

      success = true;

      assert ObjectReleaseTracker.track(this);

    } catch (IOException e) {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
    } finally {
      if (!success && raf != null) {
        try {
          raf.close();
        } catch (Exception e) {
          log.error("Error closing tlog file (after error opening)", e);
        }
      }
    }
  }