public static void main(final String[] args) throws IOException, URISyntaxException {
DesktopSetup.deploy();
Json.installComponent("com.jfixby.cmns.adopted.gdx.json.RedJson");
final File words_folder = LocalFileSystem.ApplicationHome().child("words");
final ChildrenList words_files = words_folder.listDirectChildren();
final WordsSorter sorter = new WordsSorter(false);
for (int i = 0; i < words_files.size(); i++) {
final File file = words_files.getElementAt(i);
final String file_name = file.getName();
if (file_name.startsWith("#")) {
continue;
}
L.d("reading", file_name);
final String data = file.readToString();
L.d("parsing", file_name);
final WordCollectorFile content = Json.deserializeFromString(WordCollectorFile.class, data);
final List<WordCollector> split = Collections.newList(content.values);
L.d(" adding", split.size());
sorter.addOthers(split);
}
sorter.filter(1);
sorter.sort();
// sorter.print();
final Collection<WordCollector> terms_list = sorter.list();
terms_list.print("terms_list", 0, 500);
final Random random = new Random();
final int EXTRACTIONS = 100;
final int NUMBER_OF_TERMS = 8;
for (int i = 0; ; i++) {
final List<String> batch = Collections.newList();
for (int k = 0; k < EXTRACTIONS; k++) {
final String request = generateRequest(NUMBER_OF_TERMS, terms_list, random);
batch.add(request);
}
batch.print("batch generated");
for (int k = 0; k < batch.size(); k++) {
final String request = batch.getElementAt(k);
final String request_url = template + request.replaceAll(" ", "+");
L.d(request, request_url);
openUrl(request_url);
Sys.sleep(10 * 1000);
}
}
}
public static FolderSupportingIndex rebuild(final FolderSupportingIndexBuilderParams params)
throws IOException {
final File file = params.getTarget();
if (!file.isFolder()) {
Err.reportError("Is not folder " + file);
}
if (params.getDebug()) {
L.d(file);
}
final ChildrenList children = file.listDirectChildren();
// children.print("children");
final FolderSupportingIndex desc = new FolderSupportingIndex();
Collections.scanCollection(
children,
new CollectionScanner<File>() {
@Override
public void scanElement(final File e, final long i) {
if (e.getName().startsWith(FolderSupportingIndex.FILE_NAME)) {
return;
}
final FolderSupportingIndexEntry entry = new FolderSupportingIndexEntry();
try {
entry.name = e.getName();
entry.is_file = e.isFile();
entry.is_folder = e.isFolder();
entry.lastModified = e.lastModified();
entry.size = e.getSize();
if (!params.ignoreHashSum() && e.isFile()) {
entry.hash = e.calculateHash().getMD5HashHexString();
}
desc.entries.put(entry.name, entry);
if (e.isFolder()) {
final FolderSupportingIndexBuilderParams paramsNext = params.copy();
paramsNext.setTarget(e);
if (params.rebuidOnlyForRoot()) {
paramsNext.setNoOutput(true);
} else {
paramsNext.setNoOutput(params.noOutput());
// paramsNext.noOutput = params.noOutput;
}
paramsNext.setIgnoreHashSum(params.ignoreHashSum());
paramsNext.setDebug(params.getDebug());
if (params.recoursive()) {
final FolderSupportingIndex sublevel = rebuild(paramsNext);
desc.children.put(entry.name, sublevel);
}
}
} catch (final IOException e1) {
Err.reportError(e1);
}
}
});
if (!params.noOutput()) {
FolderSupportingIndex deckCheck = null;
{
final File desc_file = file.child(FolderSupportingIndex.FILE_NAME);
L.d("writing", desc_file);
final FileOutputStream os = desc_file.newOutputStream();
os.open();
HTTPOperator.encode(desc, os);
os.close();
final ByteArray dataCheck = desc_file.readBytes();
deckCheck = HTTPOperator.decode(dataCheck);
}
{
final File desc_file_json = file.child(FolderSupportingIndex.FILE_NAME + ".json");
L.d("writing", desc_file_json);
final JsonString stringData = Json.serializeToString(desc);
final JsonString testStringData = Json.serializeToString(deckCheck);
final String s1 = stringData.toString();
final String s2 = testStringData.toString();
if (!s1.equals(s2)) {
L.e(" stringData " + s1.length());
L.e("testStringData " + s2.length());
L.e(desc_file_json);
// for (int k = 0; k < s1.length(); k++) {
// final char c1 = s1.charAt(k);
// final char c2 = s2.charAt(k);
// if (c1 != c2) {
//// L.d_appendChars("#");L.d_appendChars("#");L.d_appendChars("#");
//// L.d_appendChars("" + c1);
//// L.d_appendChars("#");
// } else {
//// L.d_appendChars("" + c1);
// }
// }
L.e("decoder fails");
}
final String data = stringData.toString();
desc_file_json.writeString(data);
}
}
return desc;
}