@Test
public void testGCSChannelCloseIdempotent() throws IOException {
SeekableByteChannel channel =
new GoogleCloudStorageReadChannel(
null, "dummybucket", "dummyobject", null, new ClientRequestHelper<StorageObject>());
channel.close();
channel.close();
}
private static void loadValueIntoBuffer(int idx, ByteBuffer buffer) throws IOException {
Path path = PATH_A[idx];
do {
SeekableByteChannel bc = EdisonGPIO.gpioLinuxPins.provider.newByteChannel(path, readOptions);
buffer.clear();
while (bc.read(buffer) >= 0) {}
bc.close();
buffer.flip();
// if length is 0 read this again.
} while (buffer.limit() == 0 || (buffer.get(0) < '0'));
}
private static void testSequential() throws IOException {
long start = System.currentTimeMillis();
try (SeekableByteChannel c = Files.newByteChannel(testFile, READ)) {
while (true) {
b.clear();
int nbytes = c.read(b);
if (nbytes <= 0) break;
}
}
long end = System.currentTimeMillis();
logger.info("testSequential {} ms", end - start);
}
public static void writeValue(int port, ByteBuffer data, EdisonPinManager d) {
try {
SeekableByteChannel channel = d.provider.newByteChannel(d.gpioValue[port], i2cOptions);
do {
channel.write(data);
} while (data.hasRemaining()); // Caution, TODO: this is blocking.
data.flip();
channel.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private static void testReopenAndSeek() throws IOException {
long position = 0;
long start = System.currentTimeMillis();
while (true) {
try (SeekableByteChannel c = Files.newByteChannel(testFile, READ)) {
b.clear();
c.position(position);
int nbytes = c.read(b);
if (nbytes <= 0) break;
position += nbytes;
}
}
long end = System.currentTimeMillis();
logger.info("testReopenAndSeek {} ms", end - start);
}
public static String readChars(SeekableByteChannel channel, int length) throws IOException {
ByteBuffer bb = ByteBuffer.allocate(length);
bb.order(ByteOrder.LITTLE_ENDIAN);
channel.read(bb);
bb.rewind();
return readChars(bb, bb.remaining());
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (b == null || b.length < off + len) {
throw new IOException("buffer does not match size");
}
checkRanges();
if (currentRange == null || counter == -1) {
return -1; // ende
}
long mLen = (currentRange[1] - counter);
// das sollte nicht sein nach einem rangecheck
if (mLen == 0) {
return -1;
}
int xlen = Math.min(len - off, (int) mLen);
ByteBuffer bb = ByteBuffer.wrap(b);
bb.position(off);
bb.limit(xlen);
channel.read(bb);
counter += xlen;
checkRanges();
return xlen;
}
public static int readBit(int idx) {
try {
ByteBuffer buffer = readBitBuffer[idx];
SeekableByteChannel bc =
EdisonGPIO.gpioLinuxPins.provider.newByteChannel(
EdisonGPIO.gpioLinuxPins.gpioValue[idx], readOptions);
buffer.clear();
while (bc.read(buffer) == 0) {} // only need 1
bc.close();
buffer.flip();
return buffer.get() & 0x1;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public ByteBuffer next() {
try {
if (channel == null) {
channel = Files.newByteChannel(filePath, StandardOpenOption.READ);
LOG.debug("Opened file {}", filePath);
}
buffer.clear();
int read = channel.read(buffer);
if (read < 0) throw new NoSuchElementException();
if (LOG.isDebugEnabled()) LOG.debug("Read {} bytes from {}", read, filePath);
position += read;
if (!hasNext()) close();
buffer.flip();
return buffer;
} catch (NoSuchElementException x) {
close();
throw x;
} catch (Exception x) {
close();
throw (NoSuchElementException) new NoSuchElementException().initCause(x);
}
}
@Override
public void close() {
try {
if (channel != null) channel.close();
} catch (Exception x) {
LOG.ignore(x);
}
}
/** Takes care of resource cleanup if the decoder never had a chance to finish. */
@Override
public void close() {
try {
bytechannel.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
decoderthreadpool.shutdownNow();
}
// this method read every char at every 10 length position.
private void executeRead() throws IOException {
System.out.println("executeRead started");
try (SeekableByteChannel channel =
Files.newByteChannel(
Paths.get("c:/temp/channelExample.txt"), EnumSet.of(StandardOpenOption.READ))) {
ByteBuffer buffer = ByteBuffer.allocate(1);
long position = 0;
while (true) {
buffer.clear();
channel.position(position);
if (channel.read(buffer) < 0) break;
buffer.flip();
System.out.println(
"position : "
+ position
+ ", char : "
+ Charset.defaultCharset().decode(buffer)); // print chars from every 10 positions.
position += 10;
}
}
}
public static void main(String args[]) {
int count;
Path filepath = null;
// First, obtain a path to the file.
try {
filepath = Paths.get("test.txt");
} catch (InvalidPathException e) {
System.out.println("Path Error " + e);
return;
}
// Next, obtain a channel to that file within a try-with-resources block.
try (SeekableByteChannel fChan = Files.newByteChannel(filepath)) {
// Allocate a buffer.
ByteBuffer mBuf = ByteBuffer.allocate(128);
do {
// Read a buffer.
count = fChan.read(mBuf);
// Stop when end of file is reached.
if (count != -1) {
// Rewind the buffer so that it can be read.
mBuf.rewind();
// Read bytes from the buffer and show
// them on the screen as characters.
for (int i = 0; i < count; i++) System.out.print((char) mBuf.get());
}
} while (count != -1);
System.out.println();
} catch (IOException e) {
System.out.println("I/O Error " + e);
}
}
private void executeWrite() throws IOException {
System.out.println("executeWrite started.");
try (SeekableByteChannel channel =
Files.newByteChannel(
Paths.get("c:/temp/channelExample.txt"),
EnumSet.of(
StandardOpenOption.CREATE,
StandardOpenOption.WRITE,
StandardOpenOption.TRUNCATE_EXISTING))) {
String data =
"This is a sample data ========================================"
+ "This is a sample data ========================================"
+ "This is a sample data ========================================"
+ "This is a sample data ========================================"
+ "This is a sample data ========================================\r\n";
byte[] byteData = data.getBytes();
ByteBuffer buf = ByteBuffer.allocate(10);
int offset = 0;
int length = buf.capacity();
// read every 10 bytes and put the data into buffer and write it.
while (true) {
buf.clear(); // clear buffer to starting use.
if ((offset + length) >= byteData.length) {
length = byteData.length - offset;
}
buf.put(byteData, offset, length);
buf.flip(); // you should call this method to read data from buffer.
System.out.print(Charset.defaultCharset().decode(buf));
buf
.rewind(); // rewind the position because the above line used this buffer so position is
// at the end of data.
channel.write(buf);
offset += length;
if (offset >= byteData.length) break;
}
System.out.println("writing data to channel finished");
}
}
private static SeekableByteChannel addHeader(SeekableByteChannel input) {
try {
int total = (int) input.size();
final String comment =
"@HD\tVN:1.0 SO:unsorted\n"
+ "@SQ\tSN:chr1\tLN:101\n"
+ "@SQ\tSN:chr2\tLN:101\n"
+ "@SQ\tSN:chr3\tLN:101\n"
+ "@RG\tID:0\tSM:JP was here\n";
byte[] commentBuf = comment.getBytes();
ByteBuffer buf = ByteBuffer.allocate(total + commentBuf.length);
buf.put(commentBuf);
input.position(0);
while (input.read(buf) > 0) {
// read until EOF
}
buf.flip();
return new SeekableByteChannelFromBuffer(buf);
} catch (IOException x) {
throw new RuntimeException(x);
}
}
@Override
public synchronized int read() throws IOException {
if (!hasMore()) {
throw new EOFException("limit reached");
}
ByteBuffer buffer = ByteBuffer.allocate(1);
if (channel.read(buffer) != 1) {
throw new EOFException("buffer read faild");
}
buffer.flip();
int b = buffer.get();
counter += 1;
checkRanges();
return b;
}
public void testWindows() throws Exception {
// ==== Testing Windows filesystem ===
// two options to write - WRITE, WRITE + APPEND. Simple writes just rewrite current values
SeekableByteChannel s =
Files.newByteChannel(
Paths.get("test.txt"),
StandardOpenOption.WRITE,
StandardOpenOption.READ,
StandardOpenOption.CREATE); // , StandardOpenOption.APPEND);
ByteBuffer bb = ByteBuffer.wrap("12345".getBytes());
s.truncate(0); // clear file
s.position(0); // start position
s.write(bb);
bb.position(2);
s.write(bb);
//
s.position(0);
byte[] bbrb = new byte[(int) s.size()];
s.read(ByteBuffer.wrap(bbrb));
System.out.println(new String(bbrb)); // should be 12345345
s.close();
}
public void checkRanges() throws IOException {
if (currentRange != null) {
if (counter >= currentRange[1]) {
counter = -1;
}
}
if (counter == -1) {
if (!ranges.isEmpty()) {
currentRange = ranges.remove(0);
// reset channel
channel.position(currentRange[0]);
counter = 0;
} else {
currentRange = null;
}
}
}
public static void testZIP() throws Exception {
// ==== Testing ZIP filesystem ===
// two options to write - WRITE, WRITE + APPEND. Simple writes just rewrite current values
URI uri = URI.create("jar:file:/test.zip!/test.txt");
Map<String, String> env = new HashMap<>();
env.put("create", "true");
FileSystem zipfs = FileSystems.newFileSystem(uri, env);
Path pz = zipfs.provider().getPath(uri);
SeekableByteChannel s =
Files.newByteChannel(
pz,
StandardOpenOption.WRITE,
StandardOpenOption.CREATE,
StandardOpenOption.APPEND); // , StandardOpenOption.APPEND);
ByteBuffer bb = ByteBuffer.wrap("12345".getBytes());
// s.truncate(0);//clear file - unsupported
// s.position(0);//start position - unsupported
s.write(bb);
bb.position(2);
s.write(bb);
// SeekableByteChannel s1 = Files.newByteChannel(pz, StandardOpenOption.WRITE,
// StandardOpenOption.CREATE, StandardOpenOption.APPEND);//, StandardOpenOption.APPEND);
// s1.write(ByteBuffer.wrap("bb".getBytes()));
//
s.close();
// s1.close();
s =
Files.newByteChannel(
pz,
StandardOpenOption.READ,
StandardOpenOption.CREATE); // , StandardOpenOption.APPEND);
byte[] bbrb = new byte[(int) s.size()];
s.read(ByteBuffer.wrap(bbrb));
System.out.println(new String(bbrb)); // should be 12345345
s.close();
}
public FilePartInputStream(SeekableByteChannel channel) throws IOException {
this(channel, new long[] {0, channel.size()});
}
@Override
public void close() throws IOException {
channel.close();
}
private static void processArchive(
final Path path, final Set<Archive> output, final SettingsModelFactory settingsFactory)
throws IOException, StarDBException {
Archive originalArchive = new Archive(path);
if (!originalArchive.extract()) {
throw new IOException("Could not extract archive.");
}
Set<ArchiveFile> usedPaks = new HashSet<>();
for (ArchiveFile file : originalArchive.getFiles()) {
if (FileHelper.getExtension(file.getPath()).equals("modinfo")) {
JsonObject o = JsonObject.readFrom(new String(file.getData()));
String preformattedPath = o.get("path").asString().replaceAll("\"", "");
if (preformattedPath.startsWith("./")) {
preformattedPath = preformattedPath.replace("./", "");
}
if (preformattedPath.startsWith(".")) {
preformattedPath = preformattedPath.replace(".", "");
}
if (preformattedPath.startsWith("/")) {
preformattedPath = preformattedPath.replace("/", "");
}
Path modinfoPath = file.getPath();
if (modinfoPath.getNameCount() > 2) {
modinfoPath = modinfoPath.subpath(0, modinfoPath.getNameCount() - 1);
} else if (modinfoPath.getNameCount() > 1) {
modinfoPath = modinfoPath.getName(0);
} else {
modinfoPath = Paths.get("");
}
Archive outputArchive =
new Archive(
settingsFactory
.getInstance()
.getPropertyPath("modsdir")
.resolve(Paths.get(o.get("name").asString() + ".zip")));
if (file.getPath().getNameCount() == 1) {
outputArchive.addFile(new ArchiveFile(file.getData(), file.getPath(), false));
log.debug(
"'{}' -> '{}' relativized to '{}'", modinfoPath, file.getPath(), file.getPath());
} else {
outputArchive.addFile(
new ArchiveFile(
file.getData(), modinfoPath.relativize(file.getPath()).normalize(), false));
log.debug(
"'{}' -> '{}' relativized to '{}'",
modinfoPath,
file.getPath(),
modinfoPath.relativize(file.getPath()).normalize());
}
Path assetsPath = modinfoPath.resolve(Paths.get(preformattedPath));
log.debug("Assets path for modinfo '{}': {}", file.getPath(), assetsPath);
if (originalArchive.getFile(assetsPath) != null
&& !assetsPath.toString().isEmpty()
&& !originalArchive.getFile(assetsPath).isFolder()) {
if (FileHelper.identifyType(originalArchive.getFile(assetsPath).getData())
.equals("pak")) {
log.debug("Assets for mod '{}' identified as .pak file: {}", modinfoPath, assetsPath);
usedPaks.add(originalArchive.getFile(assetsPath));
ArchiveFile modinfo = outputArchive.getFile(".modinfo");
JsonObject o2 = JsonObject.readFrom(new String(modinfo.getData()));
o2.set("path", "assets");
modinfo.setData(o2.toString().getBytes());
// TODO Update StarDB to let me pass in a byte array instead of needing to open a file
Path tempPath = Paths.get("tempPak" + System.nanoTime());
SeekableByteChannel tempPakFile =
Files.newByteChannel(tempPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
tempPakFile.write(ByteBuffer.wrap(originalArchive.getFile(assetsPath).getData()));
tempPakFile.close();
AssetDatabase database = AssetDatabase.open(tempPath);
for (String assetFile : database.getFileList()) {
outputArchive.addFile(
new ArchiveFile(
database.getAsset(assetFile), Paths.get("assets/" + assetFile), false));
log.trace("Asset extracted: {} | {}", assetFile, Paths.get("assets/" + assetFile));
}
Files.deleteIfExists(tempPath);
}
} else {
ArchiveFile modinfo = outputArchive.getFile(".modinfo");
JsonObject o2 = JsonObject.readFrom(new String(modinfo.getData()));
o2.set("path", "assets");
modinfo.setData(o2.toString().getBytes());
log.debug("Assets for mod '{}' is a standard assets folder: {}", modinfoPath, assetsPath);
for (ArchiveFile f2 : originalArchive.getFiles()) {
if ((assetsPath.toString().isEmpty() || f2.getPath().startsWith(assetsPath))
&& !f2.isFolder()
&& !f2.getPath().toString().endsWith(".modinfo")) {
if (modinfoPath.toString().isEmpty()) {
if (f2.getPath().getNameCount() == 1) {
if (!f2.getPath().startsWith(assetsPath)) {
outputArchive.addFile(
new ArchiveFile(
f2.getData(),
Paths.get("assets/").resolve(f2.getPath()).normalize(),
false));
log.trace(
"'{}' -> '{}' relativized to '{}'",
modinfoPath,
f2.getPath(),
Paths.get("assets/").resolve(f2.getPath()).normalize());
} else {
outputArchive.addFile(new ArchiveFile(f2.getData(), f2.getPath(), false));
log.trace(
"'{}' -> '{}' relativized to '{}'",
modinfoPath,
f2.getPath(),
f2.getPath());
}
} else {
if (!f2.getPath().startsWith(assetsPath)) {
outputArchive.addFile(
new ArchiveFile(
f2.getData(),
Paths.get("assets/").resolve(f2.getPath()).normalize(),
false));
log.trace(
"'{}' -> '{}' relativized to '{}'",
modinfoPath,
f2.getPath(),
Paths.get("assets/").resolve(f2.getPath()).normalize());
} else {
outputArchive.addFile(new ArchiveFile(f2.getData(), f2.getPath(), false));
log.trace(
"'{}' -> '{}' relativized to '{}'",
modinfoPath,
f2.getPath(),
f2.getPath());
}
}
} else {
if (f2.getPath().getNameCount() == 1) {
if (!modinfoPath.relativize(f2.getPath()).startsWith("assets")) {
outputArchive.addFile(
new ArchiveFile(
f2.getData(), Paths.get("assets/").resolve(f2.getPath()), false));
log.trace(
"'{}' -> '{}' relativized to '{}'",
modinfoPath,
f2.getPath(),
Paths.get("assets/").resolve(f2.getPath()));
} else {
outputArchive.addFile(new ArchiveFile(f2.getData(), f2.getPath(), false));
log.trace(
"'{}' -> '{}' relativized to '{}'",
modinfoPath,
f2.getPath(),
f2.getPath());
}
} else {
if (!modinfoPath.relativize(f2.getPath()).startsWith("assets")) {
outputArchive.addFile(
new ArchiveFile(
f2.getData(),
Paths.get("assets/")
.resolve(modinfoPath.relativize(f2.getPath()).normalize()),
false));
log.trace(
"'{}' -> '{}' relativized to '{}'",
modinfoPath,
f2.getPath(),
Paths.get("assets/")
.resolve(modinfoPath.relativize(f2.getPath()).normalize()));
} else {
outputArchive.addFile(
new ArchiveFile(
f2.getData(), modinfoPath.relativize(f2.getPath()).normalize(), false));
log.trace(
"'{}' -> '{}' relativized to '{}'",
modinfoPath,
f2.getPath(),
modinfoPath.relativize(f2.getPath()).normalize());
}
}
}
}
}
}
outputArchive.writeToFile(
settingsFactory
.getInstance()
.getPropertyPath("modsdir")
.resolve(Paths.get(o.get("name").asString() + ".zip"))
.toFile()); // TODO
output.add(outputArchive);
}
}
for (ArchiveFile file : originalArchive.getFiles()) {
if (!usedPaks.contains(file)
&& !file.isFolder()
&& FileHelper.identifyType(file.getData()).equals("pak")) {
log.debug("Additional .pak identified: {}", file.getPath());
Path tempPath = Paths.get("tempPak" + System.nanoTime());
SeekableByteChannel tempPakFile =
Files.newByteChannel(tempPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
tempPakFile.write(ByteBuffer.wrap(file.getData()));
tempPakFile.close();
AssetDatabase database = AssetDatabase.open(tempPath);
log.debug(database.getFileList());
if (database.getAsset("/pak.modinfo") != null) {
log.debug("{} has a .modinfo file, parsing into mod.", file.getPath());
processPakFile(tempPath, output, settingsFactory);
} else {
log.debug("{} does not contain a .modinfo file, skipping.", file.getPath());
}
Files.deleteIfExists(tempPath);
}
}
}
