public static void main(final String[] args) throws IOException, URISyntaxException {
DesktopSetup.deploy();
Json.installComponent("com.jfixby.cmns.adopted.gdx.json.RedJson");
final File words_folder = LocalFileSystem.ApplicationHome().child("words");
final ChildrenList words_files = words_folder.listDirectChildren();
final WordsSorter sorter = new WordsSorter(false);
for (int i = 0; i < words_files.size(); i++) {
final File file = words_files.getElementAt(i);
final String file_name = file.getName();
if (file_name.startsWith("#")) {
continue;
}
L.d("reading", file_name);
final String data = file.readToString();
L.d("parsing", file_name);
final WordCollectorFile content = Json.deserializeFromString(WordCollectorFile.class, data);
final List<WordCollector> split = Collections.newList(content.values);
L.d(" adding", split.size());
sorter.addOthers(split);
}
sorter.filter(1);
sorter.sort();
// sorter.print();
final Collection<WordCollector> terms_list = sorter.list();
terms_list.print("terms_list", 0, 500);
final Random random = new Random();
final int EXTRACTIONS = 100;
final int NUMBER_OF_TERMS = 8;
for (int i = 0; ; i++) {
final List<String> batch = Collections.newList();
for (int k = 0; k < EXTRACTIONS; k++) {
final String request = generateRequest(NUMBER_OF_TERMS, terms_list, random);
batch.add(request);
}
batch.print("batch generated");
for (int k = 0; k < batch.size(); k++) {
final String request = batch.getElementAt(k);
final String request_url = template + request.replaceAll(" ", "+");
L.d(request, request_url);
openUrl(request_url);
Sys.sleep(10 * 1000);
}
}
}
@Override
public void addEntry(final Entry entry) throws IOException {
final String table_name = this.sql_table_name;
final List<String> keys = Collections.newList();
final String stm = "INSERT INTO " + table_name + " " + this.paramString(entry, keys, "(", ")");
final MySQLConnection connection = this.db.obtainConnection();
connection.checkIsOpen();
try {
final Connection mysql_connection = connection.getConnection();
final PreparedStatement statement = mysql_connection.prepareStatement(stm);
for (int i = 0; i < keys.size(); i++) {
final String key = keys.getElementAt(i);
final String value = entry.getValue(key);
statement.setString(i + 1, value);
}
statement.execute();
} catch (final SQLException e) {
e.printStackTrace();
throw new IOException(e);
} finally {
this.db.releaseConnection(connection);
}
}
@Override
public void replaceEntries(final List<Entry> batch) throws IOException {
if (batch.size() == 0) {
return;
}
final Entry entry0 = batch.getElementAt(0);
final String table_name = this.sql_table_name;
final List<String> keys = Collections.newList();
final String stm = "REPLACE " + table_name + " " + this.paramString(entry0, keys, "(", ")");
final MySQLConnection connection = this.db.obtainConnection();
connection.checkIsOpen();
try {
final Connection mysql_connection = connection.getConnection();
final PreparedStatement statement = mysql_connection.prepareStatement(stm);
for (int b = 0; b < batch.size(); b++) {
final Entry entry = batch.getElementAt(b);
for (int i = 0; i < keys.size(); i++) {
final String key = keys.getElementAt(i);
final String value = entry.getValue(key);
statement.setString(i + 1, value);
}
statement.addBatch();
}
statement.executeBatch();
} catch (final SQLException e) {
e.printStackTrace();
throw new IOException(e);
} finally {
this.db.releaseConnection(connection);
}
}
public static void main(final String[] args) {
DesktopSetup.deploy();
final RelativePath target = JUtils.newRelativePath();
final AmazonS3 s3 = new AmazonS3Client(new ProfileCredentialsProvider());
final ListObjectsRequest request = new ListObjectsRequest().withBucketName("jfix.by");
final String prefix = "/";
request.withPrefix("wp-content");
request.setDelimiter("");
final ObjectListing objectListing = s3.listObjects(request);
final List<String> prefixes = Collections.newList(objectListing.getCommonPrefixes());
prefixes.print("prefixes");
final List<S3ObjectSummary> summs = Collections.newList(objectListing.getObjectSummaries());
}
private List<Entry> collectResult(final ResultSet result) throws SQLException, IOException {
final List<Entry> entries = Collections.newList();
final TableSchema schema = this.getSchema();
final Collection<String> columns = schema.getColumns();
while (result.next()) {
final Entry entry = this.readEntry(result, columns);
entries.add(entry);
}
return entries;
}
private static String generateRequest(
final int n, final Collection<WordCollector> terms_list, final Random random) {
String result = "";
final Set<String> stack = Collections.newSet();
for (; stack.size() < n; ) {
double d1 = random.nextDouble();
d1 = FloatMath.power(d1, 1.0);
int index = (int) (terms_list.size() * d1);
index = (int) IntegerMath.limit(0, index, terms_list.size());
final String term = terms_list.getElementAt(index).getWord();
stack.addAll(JUtils.split(term, " "));
}
// stack.print("");
for (int k = 0; k < stack.size(); k++) {
result = result + " " + stack.getElementAt(k);
}
return result;
}
public static FolderSupportingIndex rebuild(final FolderSupportingIndexBuilderParams params)
throws IOException {
final File file = params.getTarget();
if (!file.isFolder()) {
Err.reportError("Is not folder " + file);
}
if (params.getDebug()) {
L.d(file);
}
final ChildrenList children = file.listDirectChildren();
// children.print("children");
final FolderSupportingIndex desc = new FolderSupportingIndex();
Collections.scanCollection(
children,
new CollectionScanner<File>() {
@Override
public void scanElement(final File e, final long i) {
if (e.getName().startsWith(FolderSupportingIndex.FILE_NAME)) {
return;
}
final FolderSupportingIndexEntry entry = new FolderSupportingIndexEntry();
try {
entry.name = e.getName();
entry.is_file = e.isFile();
entry.is_folder = e.isFolder();
entry.lastModified = e.lastModified();
entry.size = e.getSize();
if (!params.ignoreHashSum() && e.isFile()) {
entry.hash = e.calculateHash().getMD5HashHexString();
}
desc.entries.put(entry.name, entry);
if (e.isFolder()) {
final FolderSupportingIndexBuilderParams paramsNext = params.copy();
paramsNext.setTarget(e);
if (params.rebuidOnlyForRoot()) {
paramsNext.setNoOutput(true);
} else {
paramsNext.setNoOutput(params.noOutput());
// paramsNext.noOutput = params.noOutput;
}
paramsNext.setIgnoreHashSum(params.ignoreHashSum());
paramsNext.setDebug(params.getDebug());
if (params.recoursive()) {
final FolderSupportingIndex sublevel = rebuild(paramsNext);
desc.children.put(entry.name, sublevel);
}
}
} catch (final IOException e1) {
Err.reportError(e1);
}
}
});
if (!params.noOutput()) {
FolderSupportingIndex deckCheck = null;
{
final File desc_file = file.child(FolderSupportingIndex.FILE_NAME);
L.d("writing", desc_file);
final FileOutputStream os = desc_file.newOutputStream();
os.open();
HTTPOperator.encode(desc, os);
os.close();
final ByteArray dataCheck = desc_file.readBytes();
deckCheck = HTTPOperator.decode(dataCheck);
}
{
final File desc_file_json = file.child(FolderSupportingIndex.FILE_NAME + ".json");
L.d("writing", desc_file_json);
final JsonString stringData = Json.serializeToString(desc);
final JsonString testStringData = Json.serializeToString(deckCheck);
final String s1 = stringData.toString();
final String s2 = testStringData.toString();
if (!s1.equals(s2)) {
L.e(" stringData " + s1.length());
L.e("testStringData " + s2.length());
L.e(desc_file_json);
// for (int k = 0; k < s1.length(); k++) {
// final char c1 = s1.charAt(k);
// final char c2 = s2.charAt(k);
// if (c1 != c2) {
//// L.d_appendChars("#");L.d_appendChars("#");L.d_appendChars("#");
//// L.d_appendChars("" + c1);
//// L.d_appendChars("#");
// } else {
//// L.d_appendChars("" + c1);
// }
// }
L.e("decoder fails");
}
final String data = stringData.toString();
desc_file_json.writeString(data);
}
}
return desc;
}