Example #1
1
  /**
   * Maps blah file with a random offset and checks to see if read from the ByteBuffer gets the
   * right line number
   */
  private static void testRead() throws Exception {
    StringBuilder sb = new StringBuilder();
    sb.setLength(4);

    for (int x = 0; x < 1000; x++) {
      try (FileInputStream fis = new FileInputStream(blah)) {
        FileChannel fc = fis.getChannel();

        long offset = generator.nextInt(10000);
        long expectedResult = offset / CHARS_PER_LINE;
        offset = expectedResult * CHARS_PER_LINE;

        MappedByteBuffer b = fc.map(MapMode.READ_ONLY, offset, 100);

        for (int i = 0; i < 4; i++) {
          byte aByte = b.get(i);
          sb.setCharAt(i, (char) aByte);
        }

        int result = Integer.parseInt(sb.toString());
        if (result != expectedResult) {
          err.println("I expected " + expectedResult);
          err.println("I got " + result);
          throw new Exception("Read test failed");
        }
      }
    }
  }
 public static void main(String[] args) throws Exception {
   fc = new RandomAccessFile("test.dat", "rw").getChannel();
   MappedByteBuffer out = fc.map(FileChannel.MapMode.READ_WRITE, 0, LENGTH);
   for (int i = 0; i < LENGTH; i++) out.put((byte) 'x');
   new LockAndModify(out, 0, 0 + LENGTH / 3);
   new LockAndModify(out, LENGTH / 2, LENGTH / 2 + LENGTH / 4);
 }
  @Override
  public void scrubFile(FileChannel file) throws IOException, ScrubException {
    if (!Machos.isMacho(file)) {
      return;
    }

    long size = file.size();
    MappedByteBuffer map = file.map(FileChannel.MapMode.READ_WRITE, 0, size);

    try {
      Machos.setUuid(map, ZERO_UUID);
    } catch (Machos.MachoException e) {
      throw new ScrubException(e.getMessage());
    }
    map.rewind();

    Hasher hasher = Hashing.sha1().newHasher();
    while (map.hasRemaining()) {
      hasher.putByte(map.get());
    }

    map.rewind();
    try {
      Machos.setUuid(map, Arrays.copyOf(hasher.hash().asBytes(), 16));
    } catch (Machos.MachoException e) {
      throw new ScrubException(e.getMessage());
    }
  }
Example #4
0
  /* (non-Javadoc)
   * @see com.ongraphdb.store.DiskSore1#start()
   */
  public void start() throws IOException {
    File file = new File(dataFileName);
    boolean newStore = file.exists() ? false : true;

    RandomAccessFile dataFile = new RandomAccessFile(dataFileName, "rw");
    dataFile.setLength(initialFileSize);
    dataChannel = dataFile.getChannel();
    dataLock = dataChannel.lock();
    mappedDataBuffer = dataChannel.map(MapMode.READ_WRITE, 0, dataMappedMemorySize);

    if (newStore) {
      nextPosition = NEXT_BYTES;
      mappedDataBuffer.putInt(nextPosition);
    } else {
      nextPosition = mappedDataBuffer.getInt();
    }
    shutdownHookThread =
        new Thread() {
          public void run() {
            try {
              mappedDataBuffer.force();
              dataLock.release();
              dataChannel.close();
              unmap(mappedDataBuffer);
            } catch (Exception e) {
              e.printStackTrace();
            }
          }
        };
    Runtime.getRuntime().addShutdownHook(shutdownHookThread);
  }
  public static ArrayList<String> getData() throws IOException {
    File file = new File("new_dataset_10000.txt");
    FileInputStream f = new FileInputStream(file);
    int SIZE = (int) file.length();
    byte[] barray = new byte[SIZE];
    FileChannel ch = f.getChannel();
    MappedByteBuffer mb = ch.map(FileChannel.MapMode.READ_ONLY, 0L, ch.size());
    int i = 0;

    while (mb.hasRemaining()) {
      barray[i] = mb.get();
      i++;
    }

    ArrayList<String> valToSort = new ArrayList<String>();
    String str = new String(barray);
    Scanner s = new Scanner(str);

    while (s.hasNext()) {
      valToSort.add(s.next());
    }

    //		Heap heap = new Heap(valToSort);
    //		heap.HeapSort();
    //		for(int i1 = 1 ; i1 < valToSort.size(); i1++)
    //		{
    //			System.out.println(valToSort.get(i1));
    //		}
    return valToSort;
  }
Example #6
0
  /**
   * This is the entry point to the whole file version function set. It takes the filename as a
   * parameter and returns the version. If the version can't be returned for whatever reason, an
   * IOException is thrown.
   *
   * @param filename The filename to get the version of.
   * @return The version of the file, as an integer.
   * @throws IOException If the version can't be retrieved for some reason.
   */
  public static int getVersion(String filename, boolean byteorder) throws IOException {
    /* psStart is the pointer to the first byte in the PE data.  The byte is pointed to at 0x3c */
    int peStart;
    /* The signature of the pe file, PE\0\0 */
    int peSignature;
    /* The number of sections (.data, .text, .rsrc, etc) */
    short numberOfSections;
    /* A pointer to the "optional" header (which will always be present in .exe files */
    int ptrOptionalHeader;

    /* The file we're reading from.*/
    MappedByteBuffer file =
        new FileInputStream(filename)
            .getChannel()
            .map(FileChannel.MapMode.READ_ONLY, 0, new File(filename).length());
    /* Set the file ordering to little endian */
    file.order(ByteOrder.LITTLE_ENDIAN);

    /* The start of the PE is pointed at by the 0x3c'th byte */
    peStart = file.getInt(PE_START);

    /* The first 4 bytes are the signature */
    peSignature = file.getInt(peStart + 0);

    /* Verify that it's a valid pe file.  IF not, throw an exception */
    if (peSignature != 0x00004550) throw new IOException("Invalid PE file!");

    /* The number of sections is the short starting at the 6th byte */
    numberOfSections = file.getShort(peStart + 6);

    /* Get a pointer to the optional header */
    ptrOptionalHeader = peStart + 24;

    return processOptionalHeader(file, ptrOptionalHeader, numberOfSections, byteorder);
  }
Example #7
0
  /*Function to read file */
  public byte[] getFile(String fname) throws Exception {

    /*FileInputStream and FileChannel for performance. */
    FileInputStream in = new FileInputStream(fname);
    FileChannel ch = in.getChannel();
    MappedByteBuffer mb = ch.map(FileChannel.MapMode.READ_ONLY, 0L, ch.size());
    long l = (new File(fname)).length();

    /*Currently, supported max size is 20MB*/
    if (l > MAX_SIZE) {
      // errorMessage("File size too large. Max file size allowed
      // is"+(Integer.MAX_VALUE/1000)+"KB");
      return null;
    }

    byte[] barray = new byte[(int) l];
    int nGet;
    /*Read the file in to barray*/
    while (mb.hasRemaining()) {
      nGet = Math.min(mb.remaining(), Integer.MAX_VALUE);
      mb.get(barray, 0, nGet);
    }
    if (in != null) in.close();

    /*Return barray*/
    return barray;
  }
Example #8
0
  private void writeWayNodes(List<Integer> waynodes, int compressionType, MappedByteBuffer buffer) {
    if (!waynodes.isEmpty() && waynodes.size() % 2 == 0) {
      Iterator<Integer> waynodeIterator = waynodes.iterator();
      buffer.putInt(waynodeIterator.next());
      buffer.putInt(waynodeIterator.next());

      while (waynodeIterator.hasNext()) {
        switch (compressionType) {
          case 0:
            buffer.putInt(waynodeIterator.next().intValue());
            buffer.putInt(waynodeIterator.next().intValue());
            break;
          case 1:
            buffer.put(Serializer.getSignedThreeBytes(waynodeIterator.next().intValue()));
            buffer.put(Serializer.getSignedThreeBytes(waynodeIterator.next().intValue()));
            break;
          case 2:
            buffer.putShort(waynodeIterator.next().shortValue());
            buffer.putShort(waynodeIterator.next().shortValue());
            break;
          case 3:
            buffer.put(waynodeIterator.next().byteValue());
            buffer.put(waynodeIterator.next().byteValue());
            break;
        }
      }
    }
  }
 // return byte offset of next IFD
 private IFDData readIFD(long byteOffset) throws IOException {
   MappedByteBuffer entries = makeReadOnlyBuffer(byteOffset, 2);
   int numEntries = entries.getChar();
   IFDData data = new IFDData();
   for (int i = 0; i < numEntries; i++) {
     IFDEntry entry = readDirectoryEntry(12 * i + 2 + byteOffset);
     if (entry.tag == MM_METADATA) {
       data.mdOffset = entry.value;
     } else if (entry.tag == STRIP_OFFSETS) {
       data.pixelOffset = entry.value;
     } else if (entry.tag == STRIP_BYTE_COUNTS) {
       data.bytesPerImage = entry.value;
     } else if (entry.tag == BITS_PER_SAMPLE) {
       if (entry.value <= 8) {
         data.bytesPerPixel = 1;
       } else if (entry.value <= 16) {
         data.bytesPerPixel = 2;
       } else {
         data.bytesPerPixel = 3;
       }
     } else if (entry.tag == MM_METADATA_LENGTH) {
       data.mdLength = entry.value;
     }
   }
   MappedByteBuffer next = makeReadOnlyBuffer(byteOffset + 2 + 12 * numEntries, 4);
   data.nextIFDAdress = unsignInt(next.getInt());
   return data;
 }
Example #10
0
 private int getNextPosition(int increment) {
   int result = nextPosition;
   nextPosition = nextPosition + increment;
   mappedDataBuffer.position(0);
   mappedDataBuffer.putInt(nextPosition);
   return result;
 }
Example #11
0
  /**
   * 从语料库中随机读取一行字符
   *
   * @return
   * @throws UnsupportedEncodingException
   */
  String readRandomLine(EncodingSet encoding) throws UnsupportedEncodingException {
    String line = null;

    // 从语料库文件中随机取一个字节
    int startIndex = NumberUtil.getRandomInt(-1, fileSize);
    byte b = mapBuf.get(startIndex);

    while (true) {
      try {
        if ((char) b == '\n') {
          // 拿到第一个\n和第二个\n之间的串
          byte[] lineBytes =
              ByteBufferUtil.getBytesAbsoluteBeforeEOFChar(mapBuf, startIndex + 1, '\n');
          line = new String(lineBytes, encoding.getEncode()).trim();
          if (line.length() <= 1) {
            startIndex += lineBytes.length + 1;
            b = mapBuf.get(startIndex);
            continue;
          }

          break;
        } else {
          b = mapBuf.get(++startIndex);
        }
      } catch (IndexOutOfBoundsException e) {
        line = readRandomLine(encoding);
        break;
      }
    }

    return line;
  }
Example #12
0
  private synchronized void appendCurrentBuffer(byte[] buf, int offset, int length)
      throws IOException {
    if (!TFS.requestSpace(length)) {
      mCanWrite = false;

      String msg =
          "Local tachyon worker does not have enough "
              + "space ("
              + length
              + ") or no worker for "
              + FILE.FID
              + " "
              + BLOCK_ID;
      if (PIN) {
        TFS.outOfMemoryForPinFile(FILE.FID);
        throw new IOException(msg);
      }

      throw new IOException(msg);
    }

    MappedByteBuffer out = mLocalFileChannel.map(MapMode.READ_WRITE, mInFileBytes, length);
    out.put(buf, 0, length);
    mInFileBytes += length;
  }
Example #13
0
  public String getFileBinaryBase64(String fileName) throws APIException {
    if ((new File(fileName)).exists()) {
      FileInputStream stream = null;
      try {
        stream = new FileInputStream(new File(fileName));
        FileChannel fc = stream.getChannel();
        MappedByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
        /* Instead of using default, pass in a decoder. */
        byte[] b = new byte[bb.remaining()];
        bb.get(b);

        return Base64.encodeBytes(b);
      } catch (Exception e) {
        throw new APIException(fileName + " could not have its files extracte!");
      } finally {
        try {
          stream.close();
        } catch (Exception e) {
          throw new APIException(fileName + " could not be closed!");
        }
      }
    } else {
      throw new APIException(fileName + " doesn't exist!");
    }
  }
Example #14
0
  /**
   * Mapped File way MappedByteBuffer 可以在处理大文件时,提升性能
   *
   * @param filename
   * @return
   * @throws IOException
   */
  public static byte[] toByteArray3(String filePath) throws IOException {

    FileChannel fc = null;
    RandomAccessFile rf = null;
    try {
      rf = new RandomAccessFile(filePath, "r");
      fc = rf.getChannel();
      MappedByteBuffer byteBuffer = fc.map(MapMode.READ_ONLY, 0, fc.size()).load();
      // System.out.println(byteBuffer.isLoaded());
      byte[] result = new byte[(int) fc.size()];
      if (byteBuffer.remaining() > 0) {
        // System.out.println("remain");
        byteBuffer.get(result, 0, byteBuffer.remaining());
      }
      return result;
    } catch (IOException e) {
      e.printStackTrace();
      throw e;
    } finally {
      try {
        rf.close();
        fc.close();
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
  }
Example #15
0
  public void loadDump(File dumpFile, File lineFile, File dumpInfoFile)
      throws FileNotFoundException, IOException {

    FileInputStream vertexStream = new FileInputStream(dumpFile);
    MappedByteBuffer inVertex =
        vertexStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, dumpFile.length());
    vertices = inVertex.asFloatBuffer();

    FileInputStream vertexLineStream = new FileInputStream(lineFile);
    MappedByteBuffer inLine;
    inLine = vertexLineStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, lineFile.length());
    stripVertexCounts = inLine.asIntBuffer();

    Scanner scanner = new Scanner(dumpInfoFile);

    String line;

    line = scanner.nextLine();
    minX = Float.parseFloat(line);
    line = scanner.nextLine();
    maxX = Float.parseFloat(line);
    line = scanner.nextLine();
    minY = Float.parseFloat(line);
    line = scanner.nextLine();
    maxY = Float.parseFloat(line);
  }
Example #16
0
  /**
   * This reads the optional header and returns the version, or throws an IOException if the version
   * could not be found.
   *
   * @param file The file we're reading from.
   * @param ptrOptionalHeader A pointer to the optional header.
   * @param numberOfSections The number of sections that we will need to process.
   * @return The version, always.
   * @throws IOException If the version couldn't be found.
   */
  private static int processOptionalHeader(
      MappedByteBuffer file, int ptrOptionalHeader, int numberOfSections, boolean byteorder)
      throws IOException {
    /* Set to true if this is a PE+ file.  Some addresses are slightly different.  Untested!  */
    boolean plus;
    /* The number of RVA entries.  We don't care what the entries are, so we just skip right over them.  */
    int numberOfRvaAndSizes;
    /* A pointer to the table of sections.  This, along with the number of sections, is passed to the next
     * function. */
    int ptrSectionTable;
    /* The version, which will eventually be returned. */
    int version;

    /* PE+ files have the first ("magic") byte set to 0x20b */
    plus = file.getShort(ptrOptionalHeader) == 0x020b;
    /* Get the RVA counts from the optional header */
    numberOfRvaAndSizes = file.getInt(ptrOptionalHeader + (plus ? 108 : 92));

    /* The optional header is 96 bytes, and each RVA is 8 bytes.  Skip over them all. */
    ptrSectionTable = ptrOptionalHeader + 96 + (numberOfRvaAndSizes * 8);

    /* Get the version from the sections */
    version = processSections(file, ptrSectionTable, numberOfSections, byteorder);

    /* If the version wasn't found, throw an exception */
    if (version == 0) throw new IOException("Couldn't find .rsrc section!");

    return version;
  }
Example #17
0
    private Pair<Long, byte[]> queryLog(HttpExchange t, Map<String, String> paramMap)
        throws IOException {

      String fileParam = paramMap.get(HttpserverUtils.HTTPSERVER_LOGVIEW_PARAM_LOGFILE);
      if (StringUtils.isBlank(fileParam)) {
        handlFailure(t, "Bad Request, Params Error, no log file name.");
        return null;
      }

      String logFile = Joiner.on(File.separator).join(logDir, fileParam);
      FileChannel fc = null;
      MappedByteBuffer fout = null;
      long fileSize = 0;
      byte[] ret = "Failed to get data".getBytes();
      try {
        fc = new RandomAccessFile(logFile, "r").getChannel();

        fileSize = fc.size();

        long position = fileSize - HttpserverUtils.HTTPSERVER_LOGVIEW_PAGESIZE;
        try {
          String posStr = paramMap.get(HttpserverUtils.HTTPSERVER_LOGVIEW_PARAM_POS);
          if (StringUtils.isBlank(posStr) == false) {
            long pos = Long.valueOf(posStr);

            position = pos;
          }
        } catch (Exception e) {
          LOG.warn("Invalide position ");
        }
        if (position < 0) {
          position = 0L;
        }

        long size = Math.min(fileSize - position, HttpserverUtils.HTTPSERVER_LOGVIEW_PAGESIZE);

        LOG.info("logview " + logFile + ", position=" + position + ", size=" + size);
        fout = fc.map(FileChannel.MapMode.READ_ONLY, position, size);

        ret = new byte[(int) size];
        fout.get(ret);
        return new Pair<Long, byte[]>(fileSize, ret);

      } catch (FileNotFoundException e) {
        LOG.warn(e);
        handlFailure(t, "Bad Request, Failed to find " + fileParam);
        return null;

      } catch (IOException e) {
        LOG.warn(e);
        handlFailure(t, "Bad Request, Failed to open " + fileParam);
        return null;
      } finally {
        fout = null;
        if (fc != null) {
          IOUtils.closeQuietly(fc);
        }
      }
    }
  protected void sendMessage(String message) {
    buffer.position(0);
    for (int i = 0; i < message.length(); ++i) {
      buffer.putChar(message.charAt(i));
    }

    buffer.putChar('\0');
  }
Example #19
0
 /* (non-Javadoc)
  * @see com.ongraphdb.store.DiskSore1#shutdown()
  */
 public void shutdown() throws IOException {
   mappedDataBuffer.force();
   mappedDataBuffer.clear();
   dataLock.release();
   dataChannel.close();
   unmap(mappedDataBuffer);
   Runtime.getRuntime().removeShutdownHook(shutdownHookThread);
 }
Example #20
0
 private void writeRecordHeaderToMappedBuffer(RecordHeader header) {
   mappedDataBuffer.position(header.getPosition());
   mappedDataBuffer.putInt(header.getDataSize());
   mappedDataBuffer.put(header.getFragmented());
   if (header.isFragmented()) {
     mappedDataBuffer.putInt(header.getNextPos());
   }
 }
 private JSONObject readJSONObject(long offset, long length) throws IOException, JSONException {
   MappedByteBuffer mdBuffer = makeReadOnlyBuffer(offset, length);
   StringBuffer sBuffer = new StringBuffer();
   for (int i = 0; i < length; i++) {
     sBuffer.append((char) mdBuffer.get(i));
   }
   return new JSONObject(sBuffer.toString());
 }
 @SuppressWarnings("deprecation")
 @Override
 protected void writeToBuffer(MappedByteBuffer buffer, Tuple e) {
   KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
   buffer.putInt(kv.getLength() + Bytes.SIZEOF_INT);
   buffer.putInt(kv.getLength());
   buffer.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
 }
Example #23
0
 private void writeWordBlock(WordBlock block) throws IOException {
   MappedByteBuffer buffer = fileChannel.map(MapMode.READ_WRITE, block.position, block_size);
   byte[] data = block.encode();
   if (data.length > block_size) {
     throw new RuntimeException();
   }
   buffer.put(data);
   // buffer.force();
 }
Example #24
0
 public static void main(String[] args) throws Exception {
   MappedByteBuffer out =
       new RandomAccessFile("test.dat", "rw")
           .getChannel()
           .map(FileChannel.MapMode.READ_WRITE, 0, length);
   for (int i = 0; i < length; i++) out.put((byte) 'x');
   print("Finished writing");
   for (int i = length / 2; i < length / 2 + 6; i++) printnb((char) out.get(i));
 }
  private static void corrupt(DataSegment segment) throws IOException {
    final Map<String, Object> localLoadSpec = segment.getLoadSpec();
    final Path segmentPath = Paths.get(localLoadSpec.get("path").toString());

    final MappedByteBuffer buffer = Files.map(segmentPath.toFile(), FileChannel.MapMode.READ_WRITE);
    while (buffer.hasRemaining()) {
      buffer.put((byte) 0xFF);
    }
  }
  private IFDEntry readDirectoryEntry(long byteOffset) throws IOException {
    MappedByteBuffer ifd = makeReadOnlyBuffer(byteOffset, 12);
    char tag = ifd.getChar();
    char type = ifd.getChar();
    long count = unsignInt(ifd.getInt());
    long value = unsignInt(ifd.getInt());

    return (new IFDEntry(tag, type, count, value));
  }
Example #27
0
 /**
  * @param offset 地区记录的起始偏移
  * @return 地区名字符串
  */
 private String readArea(int offset) {
   mbb.position(offset);
   byte b = mbb.get();
   if (b == REDIRECT_MODE_1 || b == REDIRECT_MODE_2) {
     int areaOffset = readInt3();
     if (areaOffset == 0) return Message.unknown_area;
     else return readString(areaOffset);
   } else return readString(offset);
 }
Example #28
0
 private WordBlock readWordBlock(int position) throws IOException {
   MappedByteBuffer buffer = fileChannel.map(MapMode.READ_ONLY, position, block_size).load();
   WordBlock block = new WordBlock(position);
   byte[] data = new byte[block_size];
   buffer.get(data);
   block.setData(data);
   block.decode();
   return block;
 }
Example #29
0
 /**
  * 从offset位置读取四个字节的ip地址放入ip数组中,读取后的ip为big-endian格式,但是 文件中是little-endian形式,将会进行转换
  *
  * @param offset
  * @param ip
  */
 private void readIP(int offset, byte[] ip) {
   mbb.position(offset);
   mbb.get(ip);
   byte temp = ip[0];
   ip[0] = ip[3];
   ip[3] = temp;
   temp = ip[1];
   ip[1] = ip[2];
   ip[2] = temp;
 }
  protected String receiveMessage() {
    String message = new String();
    buffer.position(0);

    for (char c = buffer.getChar(); c != '\0'; c = buffer.getChar()) {
      message += c;
    }

    return message;
  }