public void write(Tag tag, RandomAccessFile raf, RandomAccessFile tempRaf)
throws CannotWriteException, IOException {
FileChannel fc = raf.getChannel();
int oldTagSize = 0;
if (tagExists(fc)) {
// read the length
if (!canOverwrite(raf))
throw new CannotWriteException("Overwritting of this kind of ID3v2 tag not supported yet");
fc.position(6);
ByteBuffer buf = ByteBuffer.allocate(4);
fc.read(buf);
oldTagSize = (buf.get(0) & 0xFF) << 21;
oldTagSize += (buf.get(1) & 0xFF) << 14;
oldTagSize += (buf.get(2) & 0xFF) << 7;
oldTagSize += buf.get(3) & 0xFF;
oldTagSize += 10;
// System.err.println("Old tag size: "+oldTagSize);
int newTagSize = tc.getTagLength(tag);
if (oldTagSize >= newTagSize) {
// replace
// System.err.println("Old ID32v Tag found, replacing the old
// tag");
fc.position(0);
fc.write(tc.convert(tag, oldTagSize - newTagSize));
// ID3v2 Tag Written
return;
}
}
// create new tag with padding
// System.err.println("Creating a new ID3v2 Tag");
fc.position(oldTagSize);
if (fc.size() > 15 * 1024 * 1024) {
FileChannel tempFC = tempRaf.getChannel();
tempFC.position(0);
tempFC.write(tc.convert(tag, Id3v2TagCreator.DEFAULT_PADDING));
tempFC.transferFrom(fc, tempFC.position(), fc.size() - oldTagSize);
fc.close();
} else {
ByteBuffer[] content = new ByteBuffer[2];
content[1] = ByteBuffer.allocate((int) fc.size());
fc.read(content[1]);
content[1].rewind();
content[0] = tc.convert(tag, Id3v2TagCreator.DEFAULT_PADDING);
fc.position(0);
fc.write(content);
}
}
public void shutdown() {
try {
mDirectoryFile.close();
mDataFile.close();
} catch (IOException ioe) {
System.err.println("LineServer: error closing files");
}
}
public String getLine(int lineNo) throws Exception {
if (lineNo <= 0 || lineNo > mNumLines) {
return null;
}
String line = null;
Jedis jedis = pool_.getResource();
try {
String lineNoStr = Integer.toString(lineNo);
line = jedis.hget(lineNoStr, LINE_KEY);
if (line == null) {
RecordId lineId = getRecordId(lineNo);
byte[] buffer = new byte[BLOCK_SIZE];
synchronized (mDataFile) {
mDataFile.seek(lineId.pageNo() * BLOCK_SIZE);
mDataFile.read(buffer);
}
ByteBuffer bb = ByteBuffer.wrap(buffer);
int numRecords = bb.getInt(BLOCK_SIZE - INT_SIZE);
int indexPos = BLOCK_SIZE - 3 * INT_SIZE;
int curLineNo = lineNo - lineId.slotNo() + 1;
String[] recordKeys = {PAGE_NO_KEY, SLOT_NO_KEY};
for (int i = 1; i < numRecords; i++, curLineNo++) {
int offset = bb.getInt(indexPos);
int len = bb.getInt(indexPos + INT_SIZE);
ByteBuffer linebb = ByteBuffer.wrap(buffer, offset, len);
byte[] lineBuf = new byte[linebb.remaining()];
linebb.get(lineBuf);
String lineStr = new String(lineBuf);
if (i == lineId.slotNo()) {
line = lineStr;
}
// Store in cache
lineNoStr = Integer.toString(curLineNo);
jedis.hdel(lineNoStr, recordKeys);
jedis.hset(lineNoStr, LINE_KEY, lineStr);
indexPos -= 2 * INT_SIZE;
}
}
if (line == null) {
throw new Exception("LineServer: Could not find line on page");
}
} catch (IOException ioe) {
System.err.println("LineServer: IOException on line retrieval");
throw ioe;
} finally {
pool_.returnResource(jedis);
}
return line;
}
/**
* Creates and initializes an SPV block store. Will create the given file if it's missing. This
* operation will block on disk.
*/
public SPVBlockStore(NetworkParameters params, File file) throws BlockStoreException {
checkNotNull(file);
this.params = checkNotNull(params);
try {
this.numHeaders = DEFAULT_NUM_HEADERS;
boolean exists = file.exists();
// Set up the backing file.
randomAccessFile = new RandomAccessFile(file, "rw");
long fileSize = getFileSize();
if (!exists) {
log.info("Creating new SPV block chain file " + file);
randomAccessFile.setLength(fileSize);
} else if (randomAccessFile.length() != fileSize) {
throw new BlockStoreException(
"File size on disk does not match expected size: "
+ randomAccessFile.length()
+ " vs "
+ fileSize);
}
FileChannel channel = randomAccessFile.getChannel();
fileLock = channel.tryLock();
if (fileLock == null)
throw new BlockStoreException("Store file is already locked by another process");
// Map it into memory read/write. The kernel will take care of flushing writes to disk at the
// most
// efficient times, which may mean that until the map is deallocated the data on disk is
// randomly
// inconsistent. However the only process accessing it is us, via this mapping, so our own
// view will
// always be correct. Once we establish the mmap the underlying file and channel can go away.
// Note that
// the details of mmapping vary between platforms.
buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, fileSize);
// Check or initialize the header bytes to ensure we don't try to open some random file.
byte[] header;
if (exists) {
header = new byte[4];
buffer.get(header);
if (!new String(header, "US-ASCII").equals(HEADER_MAGIC))
throw new BlockStoreException("Header bytes do not equal " + HEADER_MAGIC);
} else {
initNewStore(params);
}
} catch (Exception e) {
try {
if (randomAccessFile != null) randomAccessFile.close();
} catch (IOException e2) {
throw new BlockStoreException(e2);
}
throw new BlockStoreException(e);
}
}
/**
* Assuming the file has an id3v2 tag, returns true if the tag can be overwritten. We cannot
* overwrite id3v2 tags not supported
*
* @param raf
* @return
* @throws IOException
*/
private boolean canOverwrite(RandomAccessFile raf) throws IOException {
raf.seek(3);
// Version du tag ID3v2.xx.xx
String versionHigh = raf.read() + "";
if (!(versionHigh.equals("4") || versionHigh.equals("3") || versionHigh.equals("2")))
return false; // only version 2.3.xx
// raf.read();
// int flag = raf.read() & 128;
// if (flag == 128)
// return false; //unsynchronised tags not supported
return true;
}
public static long checksumRandomAccessFile(Path filename) throws IOException {
try (RandomAccessFile file = new RandomAccessFile(filename.toFile(), "r")) {
long length = file.length();
CRC32 crc = new CRC32();
for (long p = 0; p < length; p++) {
file.seek(p);
int c = file.readByte();
crc.update(c);
}
return crc.getValue();
}
}
private void merge(SingleHit[] hits, String prefix, int chrom) throws IOException {
String postmp = getPositionsFname(prefix, chrom) + ".tmp";
String weightstmp = getWeightsFname(prefix, chrom) + ".tmp";
String lastmp = getLaSFname(prefix, chrom) + ".tmp";
RandomAccessFile positionsRAF = new RandomAccessFile(postmp, "rw");
RandomAccessFile weightsRAF = new RandomAccessFile(weightstmp, "rw");
RandomAccessFile lasRAF = new RandomAccessFile(lastmp, "rw");
int newsize = getPositionsBuffer().limit() + hits.length;
IntBP posfile =
new IntBP(positionsRAF.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, newsize * 4));
FloatBP weightfile =
new FloatBP(weightsRAF.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, newsize * 4));
IntBP lasfile =
new IntBP(lasRAF.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, newsize * 4));
int oldp = 0;
int newp = 0;
int pos = 0;
IntBP oldpositions = getPositionsBuffer();
FloatBP oldweights = getWeightsBuffer();
IntBP oldlas = getLASBuffer();
while (oldp < oldpositions.limit() || newp < hits.length) {
while (newp < hits.length
&& (oldp == oldpositions.limit() || hits[newp].pos <= oldpositions.get(oldp))) {
posfile.put(pos, hits[newp].pos);
weightfile.put(pos, hits[newp].weight);
lasfile.put(pos, Hits.makeLAS(hits[newp].length, hits[newp].strand));
newp++;
pos++;
}
while (oldp < oldpositions.limit()
&& (newp == hits.length || oldpositions.get(oldp) <= hits[newp].pos)) {
posfile.put(pos, oldpositions.get(oldp));
weightfile.put(pos, oldweights.get(oldp));
lasfile.put(pos, oldlas.get(oldp));
oldp++;
pos++;
}
// System.err.println(String.format("%d %d %d", pos, newp, oldp));
}
posfile = null;
weightfile = null;
lasfile = null;
oldpositions = null;
oldweights = null;
oldlas = null;
positionsRAF.close();
weightsRAF.close();
lasRAF.close();
/* ideally this part with the renames would atomic... */
(new File(postmp)).renameTo(new File(getPositionsFname(prefix, chrom)));
(new File(weightstmp)).renameTo(new File(getWeightsFname(prefix, chrom)));
(new File(lastmp)).renameTo(new File(getLaSFname(prefix, chrom)));
}
/**
* Read block from file.
*
* @param file - File to read.
* @param off - Marker position in file to start read from if {@code -1} read last blockSz bytes.
* @param blockSz - Maximum number of chars to read.
* @param lastModified - File last modification time.
* @return Read file block.
* @throws IOException In case of error.
*/
public static VisorFileBlock readBlock(File file, long off, int blockSz, long lastModified)
throws IOException {
RandomAccessFile raf = null;
try {
long fSz = file.length();
long fLastModified = file.lastModified();
long pos = off >= 0 ? off : Math.max(fSz - blockSz, 0);
// Try read more that file length.
if (fLastModified == lastModified && fSz != 0 && pos >= fSz)
throw new IOException(
"Trying to read file block with wrong offset: " + pos + " while file size: " + fSz);
if (fSz == 0)
return new VisorFileBlock(file.getPath(), pos, fLastModified, 0, false, EMPTY_FILE_BUF);
else {
int toRead = Math.min(blockSz, (int) (fSz - pos));
byte[] buf = new byte[toRead];
raf = new RandomAccessFile(file, "r");
raf.seek(pos);
int cntRead = raf.read(buf, 0, toRead);
if (cntRead != toRead)
throw new IOException(
"Count of requested and actually read bytes does not match [cntRead="
+ cntRead
+ ", toRead="
+ toRead
+ ']');
boolean zipped = buf.length > 512;
return new VisorFileBlock(
file.getPath(), pos, fSz, fLastModified, zipped, zipped ? zipBytes(buf) : buf);
}
} finally {
U.close(raf, null);
}
}
private RecordId getRecordId(int lineNo) throws IOException {
RecordId id = null;
Jedis jedis = pool_.getResource();
try {
int pageNo, slotNo;
String lineNoStr = Integer.toString(lineNo);
String pageNoStr = jedis.hget(lineNoStr, PAGE_NO_KEY);
String slotNoStr = jedis.hget(lineNoStr, SLOT_NO_KEY);
if (pageNoStr != null && slotNoStr != null) {
id = new RecordId(Integer.parseInt(pageNoStr), Integer.parseInt(slotNoStr));
} else { // Was not cached
// Read directory page containing line in question
byte[] buffer = new byte[BLOCK_SIZE];
int absIndex = (lineNo - 1) * 2 * INT_SIZE;
int relativeIndex = absIndex % BLOCK_SIZE;
int bytesRead = 0;
synchronized (mDirectoryFile) {
mDirectoryFile.seek(absIndex - relativeIndex);
bytesRead = mDirectoryFile.read(buffer);
}
ByteBuffer bb = ByteBuffer.wrap(buffer);
// Cache all entries in the page
int curLineNo = lineNo - relativeIndex / (2 * INT_SIZE);
for (int i = 0; i < bytesRead / (2 * INT_SIZE); i++, curLineNo++) {
pageNo = bb.getInt();
slotNo = bb.getInt();
if (curLineNo == lineNo) {
id = new RecordId(pageNo, slotNo);
}
lineNoStr = Integer.toString(curLineNo);
jedis.hset(lineNoStr, PAGE_NO_KEY, Integer.toString(pageNo));
jedis.hset(lineNoStr, SLOT_NO_KEY, Integer.toString(slotNo));
}
}
if (id == null) {
throw new IOException("LineServer: Could not find record id in directory");
}
} finally {
pool_.returnResource(jedis);
}
return id;
}
public RandomAccessFile delete(RandomAccessFile raf, RandomAccessFile tempRaf)
throws IOException {
FileChannel fc = raf.getChannel();
fc.position(0);
if (!tagExists(fc)) return raf;
fc.position(6);
ByteBuffer b = ByteBuffer.allocate(4);
fc.read(b);
b.rewind();
int tagSize = (b.get() & 0xFF) << 21;
tagSize += (b.get() & 0xFF) << 14;
tagSize += (b.get() & 0xFF) << 7;
tagSize += b.get() & 0xFF;
FileChannel tempFC = tempRaf.getChannel();
tempFC.position(0);
fc.position(tagSize + 10);
// Here we will try to skip eventual trash afer the tag and before the
// audio data
b = ByteBuffer.allocate(4);
int skip = 0;
while (fc.read(b) != -1) {
if ((b.get(0) & 0xFF) == 0xFF
&& (b.get(1) & 0xE0) == 0xE0
&& (b.get(1) & 0x06) != 0
&& (b.get(2) & 0xF0) != 0xF0
&& (b.get(2) & 0x08) != 0x08) {
fc.position(fc.position() - 4);
break;
}
fc.position(fc.position() - 3);
b.rewind();
skip++;
}
tempFC.transferFrom(fc, 0, fc.size() - tagSize - 10 - skip);
return tempRaf;
}
private void readPalette() {
RandomAccessFile rIn = null;
ByteBuffer buf = null;
int i;
try {
if (paletteFile != null) {
// see if the file exists, if not, set it to the default palette.
File file = new File(paletteFile);
numPaletteEntries = (int) (file.length() / 4);
buf = ByteBuffer.allocate(numPaletteEntries * 4);
rIn = new RandomAccessFile(paletteFile, "r");
FileChannel inChannel = rIn.getChannel();
inChannel.position(0);
inChannel.read(buf);
// Check the byte order.
buf.order(ByteOrder.LITTLE_ENDIAN);
buf.rewind();
IntBuffer ib = buf.asIntBuffer();
paletteData = new int[numPaletteEntries];
ib.get(paletteData);
ib = null;
}
} catch (Exception e) {
System.err.println("Caught exception: " + e.toString());
System.err.println(e.getStackTrace());
} finally {
if (rIn != null) {
try {
rIn.close();
} catch (Exception e) {
}
}
}
}
public void resort(String prefix, int chrom) throws IOException {
IntBP positions = getPositionsBuffer();
FloatBP weights = getWeightsBuffer();
IntBP las = getLASBuffer();
long indices[] = new long[positions.limit()];
for (int i = 0; i < indices.length; i++) {
long v = positions.get(i);
v <<= 32;
v |= i;
indices[i] = v;
}
Arrays.sort(indices);
String postmp = getPositionsFname(prefix, chrom) + ".tmp";
String weightstmp = getWeightsFname(prefix, chrom) + ".tmp";
String lastmp = getLaSFname(prefix, chrom) + ".tmp";
RandomAccessFile positionsRAF = new RandomAccessFile(postmp, "rw");
RandomAccessFile weightsRAF = new RandomAccessFile(weightstmp, "rw");
RandomAccessFile lasRAF = new RandomAccessFile(lastmp, "rw");
int newsize = getPositionsBuffer().limit();
IntBP posfile =
new IntBP(positionsRAF.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, newsize * 4));
FloatBP weightfile =
new FloatBP(weightsRAF.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, newsize * 4));
IntBP lasfile =
new IntBP(lasRAF.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, newsize * 4));
for (int i = 0; i < indices.length; i++) {
int index = (int) (indices[i] & 0xffffffffL);
int pos = (int) (indices[i] >> 32);
posfile.put(i, pos);
weightfile.put(i, weights.get(index));
lasfile.put(i, las.get(index));
}
posfile = null;
weightfile = null;
lasfile = null;
positionsRAF.close();
weightsRAF.close();
lasRAF.close();
/* ideally this part with the renames would atomic... */
(new File(postmp)).renameTo(new File(getPositionsFname(prefix, chrom)));
(new File(weightstmp)).renameTo(new File(getWeightsFname(prefix, chrom)));
(new File(lastmp)).renameTo(new File(getLaSFname(prefix, chrom)));
}
public static void main(String args[]) throws Exception {
String inputFile = "samplein.txt";
String outputFile = "sampleout.txt";
RandomAccessFile inf = new RandomAccessFile(inputFile, "r");
RandomAccessFile outf = new RandomAccessFile(outputFile, "rw");
long inputLength = new File(inputFile).length();
FileChannel inc = inf.getChannel();
FileChannel outc = outf.getChannel();
MappedByteBuffer inputData = inc.map(FileChannel.MapMode.READ_ONLY, 0, inputLength);
Charset latin1 = Charset.forName("ISO-8859-1");
CharsetDecoder decoder = latin1.newDecoder();
CharsetEncoder encoder = latin1.newEncoder();
CharBuffer cb = decoder.decode(inputData);
// Process char data here
ByteBuffer outputData = encoder.encode(cb);
outc.write(outputData);
inf.close();
outf.close();
}
/**
* Decode file charset.
*
* @param f File to process.
* @return File charset.
* @throws IOException in case of error.
*/
public static Charset decode(File f) throws IOException {
SortedMap<String, Charset> charsets = Charset.availableCharsets();
String[] firstCharsets = {
Charset.defaultCharset().name(), "US-ASCII", "UTF-8", "UTF-16BE", "UTF-16LE"
};
Collection<Charset> orderedCharsets = U.newLinkedHashSet(charsets.size());
for (String c : firstCharsets)
if (charsets.containsKey(c)) orderedCharsets.add(charsets.get(c));
orderedCharsets.addAll(charsets.values());
try (RandomAccessFile raf = new RandomAccessFile(f, "r")) {
FileChannel ch = raf.getChannel();
ByteBuffer buf = ByteBuffer.allocate(4096);
ch.read(buf);
buf.flip();
for (Charset charset : orderedCharsets) {
CharsetDecoder decoder = charset.newDecoder();
decoder.reset();
try {
decoder.decode(buf);
return charset;
} catch (CharacterCodingException ignored) {
}
}
}
return Charset.defaultCharset();
}
/** @param args */
public static void main(String[] args) {
FileChannel fc = null;
RandomAccessFile raf = null;
// StringBuilder sb;
if (args.length != 1) {
System.out.println("Usage: Ntfs filename");
System.exit(1);
}
/*
sb = new StringBuilder();
int[] foo = {129,4,229,33};
for (int b: foo) {
sb.insert(0,String.format("%02X", b));
}
System.out.println(sb.toString());
System.exit(0);
*/
try {
raf = new RandomAccessFile(args[0], "r");
fc = raf.getChannel();
Filesystem fs = new Filesystem(fc);
fs.demo();
// fs.displayFs();
} catch (FileNotFoundException x) {
System.out.println("FNF exp: " + x.getMessage());
} catch (IOException x) {
System.out.println("IO exp: " + x.getMessage());
} finally {
if (raf != null)
try {
raf.close();
} catch (IOException e) {
e.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
}
}
}
@Override
public void close() throws BlockStoreException {
try {
buffer.force();
if (System.getProperty("os.name").toLowerCase().contains("win")) {
log.info("Windows mmap hack: Forcing buffer cleaning");
WindowsMMapHack.forceRelease(buffer);
}
buffer = null; // Allow it to be GCd and the underlying file mapping to go away.
randomAccessFile.close();
} catch (IOException e) {
throw new BlockStoreException(e);
}
}
public static void writeSingleHits(
IntBP positions, FloatBP weights, IntBP las, String prefix, int chrom) throws IOException {
String postmp = getPositionsFname(prefix, chrom) + ".tmp";
String weightstmp = getWeightsFname(prefix, chrom) + ".tmp";
String lastmp = getLaSFname(prefix, chrom) + ".tmp";
RandomAccessFile positionsRAF = new RandomAccessFile(postmp, "rw");
RandomAccessFile weightsRAF = new RandomAccessFile(weightstmp, "rw");
RandomAccessFile lasRAF = new RandomAccessFile(lastmp, "rw");
Bits.sendBytes(positions.bb, 0, positions.bb.limit(), positionsRAF.getChannel());
Bits.sendBytes(weights.bb, 0, weights.bb.limit(), weightsRAF.getChannel());
Bits.sendBytes(las.bb, 0, las.bb.limit(), lasRAF.getChannel());
positionsRAF.close();
weightsRAF.close();
lasRAF.close();
/* ideally this part with the renames would atomic... */
(new File(postmp)).renameTo(new File(getPositionsFname(prefix, chrom)));
(new File(weightstmp)).renameTo(new File(getWeightsFname(prefix, chrom)));
(new File(lastmp)).renameTo(new File(getLaSFname(prefix, chrom)));
}
private void writeToOffset(File file, long offset, byte[] bytes) throws IOException {
RandomAccessFile f = new RandomAccessFile(file, "rw");
f.seek(offset);
f.write(bytes);
f.close();
}
private void append(SingleHit[] hits, String prefix, int chrom) throws IOException {
RandomAccessFile positionsRAF = new RandomAccessFile(getPositionsFname(prefix, chrom), "rw");
RandomAccessFile weightsRAF = new RandomAccessFile(getWeightsFname(prefix, chrom), "rw");
RandomAccessFile lasRAF = new RandomAccessFile(getLaSFname(prefix, chrom), "rw");
positionsRAF.seek(positionsRAF.length());
weightsRAF.seek(weightsRAF.length());
lasRAF.seek(lasRAF.length());
for (int i = 0; i < hits.length; i++) {
SingleHit h = hits[i];
positionsRAF.writeInt(h.pos);
weightsRAF.writeFloat(h.weight);
lasRAF.writeInt(makeLAS(h.length, h.strand));
}
positionsRAF.close();
weightsRAF.close();
lasRAF.close();
}
