public void writeStream(DataInputStream input) throws IOException, NoSuchAlgorithmException { try { int bytesRead = 0; int bytesToSkip = 0; while (true) { final int value = input.readUnsignedByte(); bytesRead++; check.nextByte(value); writeRaw(value); final int weakChecksum = check.weakChecksum(); String strongChecksum = null; if (bytesToSkip > 0) { bytesToSkip--; } else { Map<String, Integer> weakMatches = inputBlocks.get(weakChecksum); if (weakMatches != null) { strongChecksum = check.strongChecksum(); Integer previousOffset = weakMatches.get(strongChecksum); if (previousOffset != null) { snipRawBuffer(); System.err.println( "Using previously remembered : " + previousOffset + " : " + (bytesRead - BLOCK_SIZE)); writeBlock(previousOffset); bytesToSkip = BLOCK_SIZE - 1; } } } if ((bytesRead % BLOCK_SIZE) == 0) { Map<String, Integer> weakMatches = inputBlocks.get(weakChecksum); if (weakMatches == null) { weakMatches = new HashMap<String, Integer>(); inputBlocks.put(weakChecksum, weakMatches); } if (strongChecksum == null) { strongChecksum = check.strongChecksum(); } if (!weakMatches.containsKey(strongChecksum)) { weakMatches.put(strongChecksum, bytesRead - BLOCK_SIZE); System.err.println( "Remembering : " + weakChecksum + " : " + strongChecksum + " : " + (bytesRead - BLOCK_SIZE)); } } } } catch (EOFException e) { flushRaw(); } System.err.println("Original data: " + rawChunks); System.err.println("Reused data: " + blockChunks); }
public static FileDesc loadFile(Path root, Path file, int blocSize) throws NoSuchAlgorithmException, FileNotFoundException, IOException { MessageDigest md = MessageDigest.getInstance("SHA-512"); MessageDigest fileMd = MessageDigest.getInstance("SHA-512"); FileDesc desc = new FileDesc(file.toString(), null, null); List<Bloc> list = new ArrayList<Bloc>(); try (FileInputStream fis = new FileInputStream(root.resolve(file).toString())) { byte[] buf = new byte[blocSize]; byte[] h; int s; while ((s = fis.read(buf)) != -1) { int c; while (s < buf.length && (c = fis.read()) != -1) buf[s++] = (byte) c; fileMd.update(buf, 0, s); // padding byte p = 0; while (s < buf.length) buf[s++] = ++p; h = md.digest(buf); Bloc bloc = new Bloc(RollingChecksum.compute(buf), new Hash(h)); list.add(bloc); } h = fileMd.digest(); desc.fileHash = new Hash(h); desc.blocs = list.toArray(new Bloc[0]); } return desc; }