private static byte[] getZipDigest(ZipFile zipFile, ZipEntry ze, boolean zip)
throws ZipException, IOException, Exception {
System.out.println("z2e: " + ze.toString());
InputStream is = null;
if (zip) is = zipFile.getInputStream(ze);
else is = new FileInputStream(testDirName + ze.toString());
MessageDigest sha = MessageDigest.getInstance("SHA-1");
byte[] buf = new byte[1024];
int count;
while ((count = is.read(buf)) != -1) {
System.out.println("Count: " + count);
sha.update(buf, 0, count);
}
is.close();
return sha.digest();
}
public void fetchDoc(String filename) throws IOException {
ZipEntry ze = null;
FileInputStream fis = new FileInputStream("d:/search/tdt3.zip");
ZipFile zipFile = new ZipFile("d:/search/tdt3.zip");
ZipInputStream zis = new ZipInputStream(fis);
while ((ze = zis.getNextEntry()) != null) {
if (!ze.isDirectory())
if (ze.toString().equals(filename)) {
InputStream input = zipFile.getInputStream(ze);
Reader r = new InputStreamReader(input);
StringWriter sw = new StringWriter();
char[] buffer = new char[1024];
for (int n; (n = r.read(buffer)) != -1; ) sw.write(buffer, 0, n);
// & | regular expression stripping off the text
Pattern p = Pattern.compile(" & | & |&|&");
Matcher m = p.matcher(" & ");
m.reset(sw.toString());
XMLReader(m.replaceAll(" & "));
r.close();
}
}
}
@Override
public List<Profile> loadProfiles() {
if (DEBUG) System.out.println("JarProfilesLoader.loadProfiles: " + getJarPath());
if (getJarPath() == null) {
throw new BuildException("jarPath is null!");
}
List<Profile> profiles = new ArrayList<Profile>();
Map<String, Profile> profilesByName = new HashMap<String, Profile>();
InputStream in =
Thread.currentThread().getContextClassLoader().getResourceAsStream(getJarPath());
ZipInputStream zin = new ZipInputStream(in);
ZipEntry entry = null;
try {
while ((entry = zin.getNextEntry()) != null) {
if (DEBUG)
System.out.println(
" Jar Entry [isDirectory:" + entry.isDirectory() + "] >>> " + entry.getName());
if (DEBUG) System.out.println(" " + entry.toString());
if (entry.getName().matches("\\AMETA-INF.*") || entry.isDirectory()) {
if (DEBUG) System.out.println(" skippin!");
continue;
}
if (!entry.getName().matches("\\A[\\w\\-\\.\\:]+/\\d{5}.*\\.sql\\z")) {
throw new BuildException("Unrecognized file in sql jar: " + entry.getName());
}
String profileName = entry.getName().substring(0, entry.getName().indexOf("/"));
if (DEBUG) System.out.println(" Profile: " + profileName);
Profile profile = profilesByName.get(profileName);
if (profile == null) {
if (DEBUG) System.out.println(" Adding Profile");
profile = new Profile();
profile.setName(profileName);
profiles.add(profile);
profilesByName.put(profileName, profile);
}
String versionNameFull = entry.getName().substring(entry.getName().indexOf("/") + 1);
if (DEBUG) System.out.println(" Full version Name: " + versionNameFull);
String versionNum = versionNameFull.substring(0, 5);
if (DEBUG) System.out.println(" Version Num: " + versionNum);
String versionNameComment =
versionNameFull.length() > 9
? versionNameFull.replaceFirst("\\.sql", "").substring(6)
: "";
byte[] sqlBytes = new byte[(int) entry.getSize()];
if (DEBUG) System.out.println(" Reading " + entry.getSize() + " bytes");
int count;
byte[] sqlBuf = new byte[2048];
int totalCount = 0;
while ((count = zin.read(sqlBuf)) != -1) {
if (DEBUG)
System.out.println(
" arrayCopy(sqlBuf, 0, sqlBytes," + totalCount + "," + count + ")");
System.arraycopy(sqlBuf, 0, sqlBytes, totalCount, count);
totalCount += count;
}
if (DEBUG) System.out.println(" Total bytes read: " + totalCount);
String sql = new String(sqlBytes);
if (DEBUG) System.out.println("==========SQL:=========================================");
if (DEBUG) System.out.println(sql);
if (DEBUG) System.out.println("======================================================");
Revision revision = new Revision(new Version(versionNum));
revision.setUpgradeScriptTemplate(sql);
revision.assignUpgradeScriptTemplateChecksum();
revision.setName(versionNameComment);
profile.getRevisions().add(revision);
}
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
return profiles;
}
private static void DocParser(String tdt3) throws Exception {
FileInputStream fis = new FileInputStream(tdt3);
ZipInputStream zis = new ZipInputStream(fis);
ZipFile zipFile = new ZipFile(tdt3);
ZipEntry ze;
// Parsing process
double start = System.currentTimeMillis();
while ((ze = zis.getNextEntry()) != null) {
if (!ze.isDirectory()) {
docID++;
// split index file into 2K files each to avoid:
// 1. java heap lack in JVM on index creation
// 2. yet, it works to decrease searching time
if (docID % 2000 == 0) {
idFile++;
try {
WriteTermDocIndex("d:/search/term_doc_index_" + idFile + ".txt");
// WriteDocTermIndex("d:/search/doc_term_index_"+idFile+".txt");
// WriteTermDocPosIndex("d:/search/term_doc_pos_index_"+idFile+".txt");
} catch (IOException e) {
e.printStackTrace();
}
}
// Parse to XML
InputStream input = zipFile.getInputStream(ze);
Reader r = new InputStreamReader(input);
StringWriter sw = new StringWriter();
char[] buffer = new char[1024];
for (int n; (n = r.read(buffer)) != -1; ) sw.write(buffer, 0, n);
// & | regular expression stripping off the text
Pattern p = Pattern.compile(" & | & |&|&");
Matcher m = p.matcher(" & ");
m.reset(sw.toString());
XMLReader(m.replaceAll(" & "));
// Documents' IDs mapping
tMapDocs.put(docID, ze.toString());
r.close();
}
}
zis.close();
idFile++;
try {
WriteDocMap();
WritePostingList();
WriteTermDocIndex("d:/search/term_doc_index_" + idFile + ".txt");
// WriteDocTermIndex("d:/search/doc_term_index_"+idFile+".txt");
// WriteTermDocPosIndex("d:/search/term_doc_pos_index_"+idFile+".txt");
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println();
double end = ((System.currentTimeMillis() - start) / 1000 / 60);
System.out.println("Total documents : " + tMapDocs.size() + " terms.");
System.out.println("Total posting list : " + PostList.size() + " terms.");
System.out.println("Overall process time : " + end + " minutes.");
System.out.println("Indexing completed.");
PostList.clear();
tMapDocs.clear();
termtodocindex.clear();
// doctotermindex.clear();
// termtodocposindex.clear();
}
public static void upgrade(File wrFile) throws Exception {
System.out.println("Installing/upgrading Myna in '" + wrFile.toString() + "'...");
wrFile.mkdirs();
File web_inf = new File(wrFile.toURI().resolve("WEB-INF"));
boolean isUpgrade = false;
File backupDir = null;
if (web_inf.exists()) {
String dateString =
new java.text.SimpleDateFormat("MM-dd-yyyy_HH.mm.ss.S").format(new Date());
String backupBase = "WEB-INF/upgrade_backups/backup_" + dateString;
backupDir = new File(wrFile.toURI().resolve(backupBase));
backupDir.mkdirs();
isUpgrade = true;
System.out.println("Backups stored in " + backupDir);
// backup entire /myna folder because we're wiping it out
FileUtils.copyDirectory(
new File(wrFile.toURI().resolve("myna")), new File(backupDir.toURI().resolve("myna")));
FileUtils.deleteDirectory(new File(wrFile.toURI().resolve("myna")));
}
if (isJar) {
String jarFilePath = classUrl.substring(classUrl.indexOf(":") + 1, classUrl.indexOf("!"));
File jarFile = new File(new java.net.URL(jarFilePath).toURI());
ZipFile zipFile = new ZipFile(jarFile);
for (ZipEntry entry : java.util.Collections.list(zipFile.entries())) {;
File outputFile =
new File(wrFile.toURI().resolve(java.net.URLEncoder.encode(entry.getName(), "UTF-8")));
File backupFile = null;
if (isUpgrade) {
backupFile =
new File(
backupDir.toURI().resolve(java.net.URLEncoder.encode(entry.getName(), "UTF-8")));
}
if (entry.isDirectory()) {
outputFile.mkdirs();
if (isUpgrade) backupFile.mkdirs();
} else {
if (isUpgrade && outputFile.exists()) {
java.io.InputStream sourceIS = zipFile.getInputStream(entry);
java.io.InputStream targetIS = FileUtils.openInputStream(outputFile);
boolean isSame = IOUtils.contentEquals(sourceIS, targetIS);
sourceIS.close();
targetIS.close();
if (isSame
|| entry.toString().equals("index.html")
|| entry.toString().equals("application.sjs")
|| entry.toString().equals("WEB-INF/classes/general.properties")
|| entry.toString().startsWith("WEB-INF/myna/ds")) {
continue;
} else {
System.out.println("...backing up " + entry);
FileUtils.copyFile(outputFile, backupFile, true);
// outputFile.copyTo(backupFile);
// fusebox.upgradeLog("Backup: " + backupFile);
}
}
java.io.InputStream is = zipFile.getInputStream(entry);
java.io.OutputStream os = FileUtils.openOutputStream(outputFile);
IOUtils.copyLarge(is, os);
is.close();
os.close();
}
}
zipFile.close();
// FileUtils.deleteDirectory()
System.out.println("Done unpacking.");
}
}
public boolean extractThemAll(String filePath) {
File selectedFile = new File(filePath);
if (selectedFile.isDirectory()) {
for (File child : selectedFile.listFiles()) {
extractThemAll(child.toString());
}
} else if (filePath.endsWith(".txt.gz")) {
/**
* Passes filepPath as the source, and filePath without the .gz as the destination file to
* extract to.
*/
unGZipThemAll(filePath, filePath.substring(0, filePath.lastIndexOf(".")));
} else if (filePath.endsWith(".zip")) {
String destDir = filePath;
destDir = destDir.substring(0, destDir.lastIndexOf("."));
logger("destDir is: " + destDir);
File destDirectory = new File(destDir);
try {
ZipFile zipFile = new ZipFile(selectedFile);
Enumeration<?> enu = zipFile.entries();
while (enu.hasMoreElements()) {
ZipEntry zipEntry = (ZipEntry) enu.nextElement();
// avoid extracting unnecessary XML files
if (!zipEntry.toString().endsWith(".xml")) {
String name = zipEntry.getName(); // returns full path within the ZIP Folder
long size = zipEntry.getSize();
long compressedSize = zipEntry.getCompressedSize();
File extractedFile = new File(destDirectory, name);
if (name.endsWith("/")) {
extractedFile.mkdirs();
continue;
}
unZipThemAll(zipFile, zipEntry, extractedFile);
if (zipEntry.getName().endsWith(".gz")) {
String efName = extractedFile.toString();
unGZipThemAll(efName, efName.substring(0, efName.lastIndexOf(".")));
extractedFile.delete();
}
} // END OF XML IF
} // END OF ITERATION THROUGH THE ZIP CONTAINER
zipFile.close();
} catch (ZipException ex) {
Logger.getLogger(GCExtractor.class.getName()).log(Level.SEVERE, null, ex);
} catch (IOException ex) {
Logger.getLogger(GCExtractor.class.getName()).log(Level.SEVERE, null, ex);
}
}
return true;
} // END OF UNZIPTHEMALL
protected void processJarFile(final File file) throws Exception {
if (verbose) {
log("processing " + file.toURI());
}
final File tempFile =
File.createTempFile(file.getName(), null, new File(file.getAbsoluteFile().getParent()));
try {
final ZipInputStream zip = new ZipInputStream(new FileInputStream(file));
try {
final FileOutputStream fout = new FileOutputStream(tempFile);
try {
final ZipOutputStream out = new ZipOutputStream(fout);
ZipEntry entry;
while ((entry = zip.getNextEntry()) != null) {
byte bytes[] = getBytes(zip);
if (!entry.isDirectory()) {
final DataInputStream din = new DataInputStream(new ByteArrayInputStream(bytes));
if (din.readInt() == CLASS_MAGIC) {
bytes = process(bytes);
} else {
if (verbose) {
log("ignoring " + entry.toString());
}
}
}
final ZipEntry outEntry = new ZipEntry(entry.getName());
outEntry.setMethod(entry.getMethod());
outEntry.setComment(entry.getComment());
outEntry.setSize(bytes.length);
if (outEntry.getMethod() == ZipEntry.STORED) {
final CRC32 crc = new CRC32();
crc.update(bytes);
outEntry.setCrc(crc.getValue());
outEntry.setCompressedSize(bytes.length);
}
out.putNextEntry(outEntry);
out.write(bytes);
out.closeEntry();
zip.closeEntry();
}
out.close();
} finally {
fout.close();
}
} finally {
zip.close();
}
if (file.delete()) {
final File newFile = new File(tempFile.getAbsolutePath());
if (!newFile.renameTo(file)) {
throw new IOException("can not rename " + tempFile + " to " + file);
}
} else {
throw new IOException("can not delete " + file);
}
} finally {
tempFile.delete();
}
}
