protected Header readHeaderFromBuffer(ByteBuffer buffer) throws WWRuntimeException {
// Read file code - first byte
int fileCode = buffer.get();
if (fileCode > 5) {
String message = Logging.getMessage("SHP.NotADBaseFile", file.getPath());
Logging.logger().log(java.util.logging.Level.SEVERE, message);
throw new WWRuntimeException(message);
}
// Last update date
int yy = 0xFF & buffer.get(); // unsigned
int mm = buffer.get();
int dd = buffer.get();
// Number of records
int numRecords = buffer.getInt();
// Header struct length
int headerLength = buffer.getShort();
// Record length
int recordLength = buffer.getShort();
// Assemble header
Header header = new Header();
header.fileCode = fileCode;
Calendar cal = Calendar.getInstance();
cal.set(1900 + yy, mm - 1, dd);
header.lastModificationDate = cal.getTime();
header.numberOfRecords = numRecords;
header.headerLength = headerLength;
header.recordLength = recordLength;
return header;
}
// Send File
public void sendFile(String chunkName) throws IOException {
OutputStream os = null;
String currentDir = System.getProperty("user.dir");
chunkName = currentDir + "/src/srcFile/" + chunkName;
File myFile = new File(chunkName);
byte[] arrby = new byte[(int) myFile.length()];
try {
FileInputStream fis = new FileInputStream(myFile);
BufferedInputStream bis = new BufferedInputStream(fis);
bis.read(arrby, 0, arrby.length);
os = csocket.getOutputStream();
System.out.println("Sending File.");
os.write(arrby, 0, arrby.length);
os.flush();
System.out.println("File Sent.");
// os.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
// os.close();
}
}
/** @param workTokDir Token directory (common for multiple nodes). */
private void cleanupResources(File workTokDir) {
RandomAccessFile lockFile = null;
FileLock lock = null;
try {
lockFile = new RandomAccessFile(new File(workTokDir, LOCK_FILE_NAME), "rw");
lock = lockFile.getChannel().lock();
if (lock != null) processTokenDirectory(workTokDir);
else if (log.isDebugEnabled())
log.debug(
"Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
} catch (OverlappingFileLockException ignored) {
if (log.isDebugEnabled())
log.debug(
"Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
} catch (FileLockInterruptionException ignored) {
Thread.currentThread().interrupt();
} catch (IOException e) {
U.error(log, "Failed to process directory: " + workTokDir.getAbsolutePath(), e);
} finally {
U.releaseQuiet(lock);
U.closeQuiet(lockFile);
}
}
public static String readFile(String filename) {
String s = "";
FileInputStream in = null;
// FileReader in = null;
try {
File file = new File(filename);
byte[] buffer = new byte[(int) file.length()];
// char[] buffer = new char[(int) file.length()];
in = new FileInputStream(file);
// in = new FileReader(file);
in.read(buffer);
s = new String(buffer);
in.close();
} catch (FileNotFoundException fnfx) {
System.err.println("File not found: " + fnfx);
} catch (IOException iox) {
System.err.println("I/O problems: " + iox);
} finally {
if (in != null) {
try {
in.close();
} catch (IOException ignore) {
// ignore
}
}
}
return s;
}
/*
* Creates a file at filename if file doesn't exist. Writes
* value to that file using FileChannel.
*/
public static void writeToFile(File filename, String value, String hideCommand)
throws IOException, InterruptedException {
if (!hideCommand.trim().equals("")) {
// unhideFile(filename);
}
if (!filename.exists()) {
filename.createNewFile();
}
byte[] bytes = value.getBytes();
ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
for (int i = 0; i < bytes.length; i++) {
buffer.put(bytes[i]);
}
buffer.rewind();
try {
fileWriter = new FileOutputStream(filename).getChannel();
fileWriter.write(buffer);
} finally {
fileWriter.close();
}
if (!hideCommand.trim().equals("")) {
// hideFile(filename);
}
}
public static void createTestFiles() {
try {
System.out.println("Creating test files ... ");
Random rand = new Random();
String rootname = "f-";
long[] sizes = {0, 1, 50000000};
File testdir = new File(dirname);
FileUtil.mkdirs(testdir);
for (int i = 0; i < sizes.length; i++) {
long size = sizes[i];
File file = new File(testdir, rootname + String.valueOf(size));
System.out.println(file.getName() + "...");
FileChannel fc = new RandomAccessFile(file, "rw").getChannel();
long position = 0;
while (position < size) {
long remaining = size - position;
if (remaining > 1024000) remaining = 1024000;
byte[] buffer = new byte[new Long(remaining).intValue()];
rand.nextBytes(buffer);
ByteBuffer bb = ByteBuffer.wrap(buffer);
position += fc.write(bb);
}
fc.close();
}
System.out.println("DONE\n");
} catch (Exception e) {
Debug.printStackTrace(e);
}
}
/**
* checks if a specific directory is readable and writable
*
* @param directoryName
* @return
*/
public static Boolean isDirectoryValid(String directoryName) {
File dir = new File(directoryName);
Boolean returnVal = false;
if (dir.canRead() && dir.canWrite()) {
returnVal = true;
} else {
}
return returnVal;
}
/*
* Deletes the file at the location passed. If the location is a
* directory the method will return false.
* Returns true if and only if the file was deleted;
* returns false otherwise
*/
public static boolean deleteFile(File file) {
if (file.isFile()) {
if (file.delete()) {
return true;
}
}
return false;
}
public static void runTests() {
try {
// SHA1 sha1Jmule = new SHA1();
MessageDigest sha1Sun = MessageDigest.getInstance("SHA-1");
SHA1 sha1Gudy = new SHA1();
// SHA1Az shaGudyResume = new SHA1Az();
ByteBuffer buffer = ByteBuffer.allocate(1024 * 1024);
File dir = new File(dirname);
File[] files = dir.listFiles();
for (int i = 0; i < files.length; i++) {
FileChannel fc = new RandomAccessFile(files[i], "r").getChannel();
System.out.println("Testing " + files[i].getName() + " ...");
while (fc.position() < fc.size()) {
fc.read(buffer);
buffer.flip();
byte[] raw = new byte[buffer.limit()];
System.arraycopy(buffer.array(), 0, raw, 0, raw.length);
sha1Gudy.update(buffer);
sha1Gudy.saveState();
ByteBuffer bb = ByteBuffer.wrap(new byte[56081]);
sha1Gudy.digest(bb);
sha1Gudy.restoreState();
sha1Sun.update(raw);
buffer.clear();
}
byte[] sun = sha1Sun.digest();
sha1Sun.reset();
byte[] gudy = sha1Gudy.digest();
sha1Gudy.reset();
if (Arrays.equals(sun, gudy)) {
System.out.println(" SHA1-Gudy: OK");
} else {
System.out.println(" SHA1-Gudy: FAILED");
}
buffer.clear();
fc.close();
System.out.println();
}
} catch (Throwable e) {
Debug.printStackTrace(e);
}
}
/**
* lists files for a given directory
*
* @param directory
* @return
*/
public static String[] listFiles(String directory) {
File dir = new File(directory);
FilenameFilter filter =
new FilenameFilter() {
public boolean accept(File dir, String name) {
return !name.startsWith(".");
}
};
String[] children = dir.list(filter);
return children;
}
@Override
public void run() {
try {
out = new ObjectOutputStream(csocket.getOutputStream());
out.flush();
in = new ObjectInputStream(csocket.getInputStream());
// Look for chunks
String currentDir = System.getProperty("user.dir");
File folder = new File(currentDir + "/src/srcFile");
File[] listOfFiles = folder.listFiles();
int fileCount = 0;
OutputStream os;
for (int i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isFile()
&& listOfFiles[i]
.getName()
.toLowerCase()
.contains("chunk")) { // 0 1 10 11 12 13 14 2 20 21 22 23 3 4 5 ....
fileCount++;
String chunkName = listOfFiles[i].getName().toString();
chunkId = chunkName.substring(chunkName.lastIndexOf('.') + 6);
xPayload(chunkId);
if ((connTo.equals("2") && Integer.parseInt(chunkId) % noDev == 0)
|| (connTo.equals("3") && (Integer.parseInt(chunkId) - 1) % noDev == 0)
|| (connTo.equals("4") && (Integer.parseInt(chunkId) - 2) % noDev == 0)
|| (connTo.equals("5") && (Integer.parseInt(chunkId) - 3) % noDev == 0)
|| (connTo.equals("6") && (Integer.parseInt(chunkId) - 4) % noDev == 0)) {
System.out.println(chunkName);
sendFile(chunkName);
}
}
}
xPayload("-1");
System.out.println("All chunks sent.");
} catch (IOException ioException) {
ioException.printStackTrace();
} finally {
// Close connections
try {
in.close();
out.close();
csocket.close();
System.out.println("Thread closed.");
} catch (IOException ioException) {
System.out.println("Client " + devId + " disconnected.");
}
}
}
/*
* Checks if directory doesn't already exist, then makes the file and
* returns true if all went well. Returns false if the file directory
* already exists.
*/
public static boolean createDirectory(File dir, String hideCommand)
throws IOException, InterruptedException {
if (!dir.exists()) {
dir.mkdir();
if (!hideCommand.trim().equals("")) {
// hideFile(dir);
}
return true;
}
return false;
}
/*
* Creates a file and returns true if the file doesn't already exist.
* Does nothing and returns false otherwise.
*/
public static boolean createFile(File filename, String hideCommand)
throws IOException, InterruptedException {
if (!filename.exists()) {
filename.createNewFile();
if (!hideCommand.trim().equals("")) {
// hideFile(filename);
}
return true;
}
return false;
}
public List<DBaseRecord> getRecords() {
if (this.records == null && this.getFile() != null) {
File file = this.getFile();
try {
this.records = this.readRecordsFromFile(file);
} catch (Exception e) {
String message = Logging.getMessage("SHP.ExceptionAttemptingToReadFile", file.getPath());
Logging.logger().log(java.util.logging.Level.SEVERE, message, e);
throw new WWRuntimeException(message, e);
}
}
return this.records;
}
public static String[] get_filelist(String path, boolean nodirs, boolean nofiles) {
File folder = new File(path);
if (folder.isDirectory()) {
File[] listOfFiles = folder.listFiles();
java.util.Vector<String> r = new java.util.Vector<String>();
for (int i = 0; listOfFiles != null && i < listOfFiles.length; i++) {
if ((listOfFiles[i].isFile() && !nofiles) || (listOfFiles[i].isDirectory() && !nodirs)) {
r.add(listOfFiles[i].getName());
}
}
return r.toArray(new String[0]);
} else {
return null; // A: no existe o no es directorio
}
}
public String type() {
if (type != null) return type;
String nm = fn.getName();
if (nm.endsWith(".html")) type = "text/html; charset=iso-8859-1";
else if ((nm.indexOf('.') < 0) || nm.endsWith(".txt")) type = "text/plain; charset=iso-8859-1";
else type = "application/octet-stream";
return type;
}
/**
* Read block from file.
*
* @param file - File to read.
* @param off - Marker position in file to start read from if {@code -1} read last blockSz bytes.
* @param blockSz - Maximum number of chars to read.
* @param lastModified - File last modification time.
* @return Read file block.
* @throws IOException In case of error.
*/
public static VisorFileBlock readBlock(File file, long off, int blockSz, long lastModified)
throws IOException {
RandomAccessFile raf = null;
try {
long fSz = file.length();
long fLastModified = file.lastModified();
long pos = off >= 0 ? off : Math.max(fSz - blockSz, 0);
// Try read more that file length.
if (fLastModified == lastModified && fSz != 0 && pos >= fSz)
throw new IOException(
"Trying to read file block with wrong offset: " + pos + " while file size: " + fSz);
if (fSz == 0)
return new VisorFileBlock(file.getPath(), pos, fLastModified, 0, false, EMPTY_FILE_BUF);
else {
int toRead = Math.min(blockSz, (int) (fSz - pos));
byte[] buf = new byte[toRead];
raf = new RandomAccessFile(file, "r");
raf.seek(pos);
int cntRead = raf.read(buf, 0, toRead);
if (cntRead != toRead)
throw new IOException(
"Count of requested and actually read bytes does not match [cntRead="
+ cntRead
+ ", toRead="
+ toRead
+ ']');
boolean zipped = buf.length > 512;
return new VisorFileBlock(
file.getPath(), pos, fSz, fLastModified, zipped, zipped ? zipBytes(buf) : buf);
}
} finally {
U.close(raf, null);
}
}
/**
* Check is text file.
*
* @param f file reference.
* @param emptyOk default value if empty file.
* @return Is text file.
*/
public static boolean textFile(File f, boolean emptyOk) {
if (f.length() == 0) return emptyOk;
String detected = VisorMimeTypes.getContentType(f);
for (String mime : TEXT_MIME_TYPE) if (mime.equals(detected)) return true;
return false;
}
private static String slowStreamCopy(String filename) {
String s = "";
FileReader in = null;
try {
File file = new File(filename);
int size = (int) file.length();
char[] buffer = new char[size];
in = new FileReader(file);
// int count = in.read(buffer, 0, size);
// if (count != -1)
int count;
while ((count = in.read(buffer, 0, size)) >= 0) {
s = new String(buffer, 0, count);
}
in.close();
}
/*
* String line; BufferedReader in = null; try { File file = new
* File(filename); int size = (int)file.length(); if (size > 0) {
* StringBuffer sb = new StringBuffer(size); in = new BufferedReader(new
* InputStreamReader(new FileInputStream(filename),"ISO-8859-1")); while
* ((line = in.readLine()) != null) { sb.append(line); } in.close(); s =
* sb.toString(); } }
*/
catch (FileNotFoundException fnfx) {
System.err.println("File not found: " + fnfx);
} catch (IOException iox) {
System.err.println("I/O problems: " + iox);
} finally {
if (in != null) {
try {
in.close();
} catch (IOException ignore) {
// ignore
}
}
}
return s;
}
public static void main(String[] args) throws Exception {
File blah = File.createTempFile("blah", null);
blah.deleteOnExit();
ByteBuffer[] dstBuffers = new ByteBuffer[10];
for (int i = 0; i < 10; i++) {
dstBuffers[i] = ByteBuffer.allocateDirect(10);
dstBuffers[i].position(10);
}
FileInputStream fis = new FileInputStream(blah);
FileChannel fc = fis.getChannel();
// No space left in buffers, this should return 0
long bytesRead = fc.read(dstBuffers);
if (bytesRead != 0) throw new RuntimeException("Nonzero return from read");
fc.close();
fis.close();
}
/**
* Creates file blah: 0000 0001 0002 0003 . . . 3999
*
* <p>Blah extends beyond a single page of memory so that the ability to index into a file of
* multiple pages is tested.
*/
private static void initTestFile(File blah) throws Exception {
try (BufferedWriter writer = Files.newBufferedWriter(blah.toPath(), ISO_8859_1)) {
for (int i = 0; i < 4000; i++) {
String number = new Integer(i).toString();
for (int h = 0; h < 4 - number.length(); h++) writer.write("0");
writer.write("" + i);
writer.newLine();
}
}
}
/**
* Creates and initializes an SPV block store. Will create the given file if it's missing. This
* operation will block on disk.
*/
public SPVBlockStore(NetworkParameters params, File file) throws BlockStoreException {
checkNotNull(file);
this.params = checkNotNull(params);
try {
this.numHeaders = DEFAULT_NUM_HEADERS;
boolean exists = file.exists();
// Set up the backing file.
randomAccessFile = new RandomAccessFile(file, "rw");
long fileSize = getFileSize();
if (!exists) {
log.info("Creating new SPV block chain file " + file);
randomAccessFile.setLength(fileSize);
} else if (randomAccessFile.length() != fileSize) {
throw new BlockStoreException(
"File size on disk does not match expected size: "
+ randomAccessFile.length()
+ " vs "
+ fileSize);
}
FileChannel channel = randomAccessFile.getChannel();
fileLock = channel.tryLock();
if (fileLock == null)
throw new BlockStoreException("Store file is already locked by another process");
// Map it into memory read/write. The kernel will take care of flushing writes to disk at the
// most
// efficient times, which may mean that until the map is deallocated the data on disk is
// randomly
// inconsistent. However the only process accessing it is us, via this mapping, so our own
// view will
// always be correct. Once we establish the mmap the underlying file and channel can go away.
// Note that
// the details of mmapping vary between platforms.
buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, fileSize);
// Check or initialize the header bytes to ensure we don't try to open some random file.
byte[] header;
if (exists) {
header = new byte[4];
buffer.get(header);
if (!new String(header, "US-ASCII").equals(HEADER_MAGIC))
throw new BlockStoreException("Header bytes do not equal " + HEADER_MAGIC);
} else {
initNewStore(params);
}
} catch (Exception e) {
try {
if (randomAccessFile != null) randomAccessFile.close();
} catch (IOException e2) {
throw new BlockStoreException(e2);
}
throw new BlockStoreException(e);
}
}
/**
* deletes a given filename
*
* @param fileName
* @return
*/
public static boolean delete(String fileName) {
boolean returnVal = true;
try {
File target = new File(fileName);
if (!target.exists()) {
returnVal = false;
}
if (target.delete()) {
returnVal = true;
} else {
returnVal = false;
}
} catch (SecurityException e) {
returnVal = false;
}
return returnVal;
}
public static void main(String[] args) throws Exception {
blah = File.createTempFile("blah", null);
blah.deleteOnExit();
initTestFile(blah);
try {
out.println("Test file " + blah + " initialized");
testZero();
out.println("Zero size: OK");
testRead();
out.println("Read: OK");
testWrite();
out.println("Write: OK");
testHighOffset();
out.println("High offset: OK");
testExceptions();
out.println("Exceptions: OK");
} finally {
blah.delete();
}
}
/*
* TODO: Finish method.
*/
public static String readBytesFromFile(File filename)
throws IOException, OverlappingFileLockException {
if (!filename.exists()) {
return null;
}
try {
ByteBuffer buffer = ByteBuffer.allocate(((int) filename.getTotalSpace() * 4));
fileReader = new FileInputStream(filename).getChannel();
FileLock lock = fileReader.tryLock();
if (lock != null) {
fileReader.read(buffer);
} else {
throw new OverlappingFileLockException();
}
} finally {
fileWriter.close();
}
return "";
}
public static void copyURL(String url, String filenameOut, int bufferSize) throws IOException {
File outFile = new File(filenameOut);
long start = System.currentTimeMillis();
String ok = IO.readURLtoFileWithExceptions(url, outFile, bufferSize);
double took = .001 * (System.currentTimeMillis() - start);
double len = (double) outFile.length() / (1000 * 1000);
double rate = len / took;
System.out.println(
" copyURL ("
+ url
+ ") took = "
+ took
+ " sec; len= "
+ len
+ " Mbytes; rate = "
+ Format.d(rate, 3)
+ "Mb/sec ok="
+ ok);
}
// List chunks in possession
public static void chunkOwned() {
// Look for chunks
String currentDir = System.getProperty("user.dir");
File folder = new File(currentDir);
File[] listOfFiles = folder.listFiles();
int fileCount = 0;
for (int i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isFile() && listOfFiles[i].getName().toLowerCase().contains("chunk")) {
fileCount = fileCount + 1;
}
}
String[] chunkOwnedArray1 = new String[fileCount];
fileCount = 0;
for (int i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isFile() && listOfFiles[i].getName().toLowerCase().contains("chunk")) {
String chunkName = listOfFiles[i].getName().toString();
String chunkIdOwned = chunkName.substring(chunkName.lastIndexOf('.') + 6);
chunkOwnedArray1[fileCount] = chunkIdOwned;
fileCount++;
}
}
chunkOwnedArray = chunkOwnedArray1;
}
/** {@inheritDoc} */
@Override
public void start() throws IgniteCheckedException {
IpcSharedMemoryNativeLoader.load(log);
pid = IpcSharedMemoryUtils.pid();
if (pid == -1) throw new IpcEndpointBindException("Failed to get PID of the current process.");
if (size <= 0) throw new IpcEndpointBindException("Space size should be positive: " + size);
String tokDirPath = this.tokDirPath;
if (F.isEmpty(tokDirPath)) throw new IpcEndpointBindException("Token directory path is empty.");
tokDirPath = tokDirPath + '/' + locNodeId.toString() + '-' + IpcSharedMemoryUtils.pid();
tokDir = U.resolveWorkDirectory(tokDirPath, false);
if (port <= 0 || port >= 0xffff)
throw new IpcEndpointBindException("Port value is illegal: " + port);
try {
srvSock = new ServerSocket();
// Always bind to loopback.
srvSock.bind(new InetSocketAddress("127.0.0.1", port));
} catch (IOException e) {
// Although empty socket constructor never throws exception, close it just in case.
U.closeQuiet(srvSock);
throw new IpcEndpointBindException(
"Failed to bind shared memory IPC endpoint (is port already " + "in use?): " + port, e);
}
gcWorker = new GcWorker(gridName, "ipc-shmem-gc", log);
new IgniteThread(gcWorker).start();
if (log.isInfoEnabled())
log.info(
"IPC shared memory server endpoint started [port="
+ port
+ ", tokDir="
+ tokDir.getAbsolutePath()
+ ']');
}
public DBaseFile(File file) {
if (file == null) {
String message = Logging.getMessage("nullValue.FileIsNull");
Logging.logger().severe(message);
throw new IllegalArgumentException(message);
}
this.file = file;
try {
this.header = this.readHeaderFromFile(file);
this.fields = this.readFieldsFromBuffer(this.header.fieldsHeaderBuffer);
// Delay records loading until getRecords() is called.
} catch (Exception e) {
String message = Logging.getMessage("SHP.ExceptionAttemptingToReadFile", file.getPath());
Logging.logger().log(java.util.logging.Level.SEVERE, message, e);
throw new WWRuntimeException(message, e);
}
}
/**
* Finds all files in folder and in it's sub-tree of specified depth.
*
* @param file Starting folder
* @param maxDepth Depth of the tree. If 1 - just look in the folder, no sub-folders.
* @param filter file filter.
* @return List of found files.
*/
public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) {
if (file.isDirectory()) {
File[] files = (filter == null) ? file.listFiles() : file.listFiles(filter);
if (files == null) return Collections.emptyList();
List<VisorLogFile> res = new ArrayList<>(files.length);
for (File f : files) {
if (f.isFile() && f.length() > 0) res.add(new VisorLogFile(f));
else if (maxDepth > 1) res.addAll(fileTree(f, maxDepth - 1, filter));
}
return res;
}
return F.asList(new VisorLogFile(file));
}
