public static void main(final String[] args) throws IOException, URISyntaxException { DesktopSetup.deploy(); Json.installComponent("com.jfixby.cmns.adopted.gdx.json.RedJson"); final File words_folder = LocalFileSystem.ApplicationHome().child("words"); final ChildrenList words_files = words_folder.listDirectChildren(); final WordsSorter sorter = new WordsSorter(false); for (int i = 0; i < words_files.size(); i++) { final File file = words_files.getElementAt(i); final String file_name = file.getName(); if (file_name.startsWith("#")) { continue; } L.d("reading", file_name); final String data = file.readToString(); L.d("parsing", file_name); final WordCollectorFile content = Json.deserializeFromString(WordCollectorFile.class, data); final List<WordCollector> split = Collections.newList(content.values); L.d(" adding", split.size()); sorter.addOthers(split); } sorter.filter(1); sorter.sort(); // sorter.print(); final Collection<WordCollector> terms_list = sorter.list(); terms_list.print("terms_list", 0, 500); final Random random = new Random(); final int EXTRACTIONS = 100; final int NUMBER_OF_TERMS = 8; for (int i = 0; ; i++) { final List<String> batch = Collections.newList(); for (int k = 0; k < EXTRACTIONS; k++) { final String request = generateRequest(NUMBER_OF_TERMS, terms_list, random); batch.add(request); } batch.print("batch generated"); for (int k = 0; k < batch.size(); k++) { final String request = batch.getElementAt(k); final String request_url = template + request.replaceAll(" ", "+"); L.d(request, request_url); openUrl(request_url); Sys.sleep(10 * 1000); } } }
@Override public void addEntry(final Entry entry) throws IOException { final String table_name = this.sql_table_name; final List<String> keys = Collections.newList(); final String stm = "INSERT INTO " + table_name + " " + this.paramString(entry, keys, "(", ")"); final MySQLConnection connection = this.db.obtainConnection(); connection.checkIsOpen(); try { final Connection mysql_connection = connection.getConnection(); final PreparedStatement statement = mysql_connection.prepareStatement(stm); for (int i = 0; i < keys.size(); i++) { final String key = keys.getElementAt(i); final String value = entry.getValue(key); statement.setString(i + 1, value); } statement.execute(); } catch (final SQLException e) { e.printStackTrace(); throw new IOException(e); } finally { this.db.releaseConnection(connection); } }
@Override public void replaceEntries(final List<Entry> batch) throws IOException { if (batch.size() == 0) { return; } final Entry entry0 = batch.getElementAt(0); final String table_name = this.sql_table_name; final List<String> keys = Collections.newList(); final String stm = "REPLACE " + table_name + " " + this.paramString(entry0, keys, "(", ")"); final MySQLConnection connection = this.db.obtainConnection(); connection.checkIsOpen(); try { final Connection mysql_connection = connection.getConnection(); final PreparedStatement statement = mysql_connection.prepareStatement(stm); for (int b = 0; b < batch.size(); b++) { final Entry entry = batch.getElementAt(b); for (int i = 0; i < keys.size(); i++) { final String key = keys.getElementAt(i); final String value = entry.getValue(key); statement.setString(i + 1, value); } statement.addBatch(); } statement.executeBatch(); } catch (final SQLException e) { e.printStackTrace(); throw new IOException(e); } finally { this.db.releaseConnection(connection); } }
public static void main(final String[] args) { DesktopSetup.deploy(); final RelativePath target = JUtils.newRelativePath(); final AmazonS3 s3 = new AmazonS3Client(new ProfileCredentialsProvider()); final ListObjectsRequest request = new ListObjectsRequest().withBucketName("jfix.by"); final String prefix = "/"; request.withPrefix("wp-content"); request.setDelimiter(""); final ObjectListing objectListing = s3.listObjects(request); final List<String> prefixes = Collections.newList(objectListing.getCommonPrefixes()); prefixes.print("prefixes"); final List<S3ObjectSummary> summs = Collections.newList(objectListing.getObjectSummaries()); }
private List<Entry> collectResult(final ResultSet result) throws SQLException, IOException { final List<Entry> entries = Collections.newList(); final TableSchema schema = this.getSchema(); final Collection<String> columns = schema.getColumns(); while (result.next()) { final Entry entry = this.readEntry(result, columns); entries.add(entry); } return entries; }
private static String generateRequest( final int n, final Collection<WordCollector> terms_list, final Random random) { String result = ""; final Set<String> stack = Collections.newSet(); for (; stack.size() < n; ) { double d1 = random.nextDouble(); d1 = FloatMath.power(d1, 1.0); int index = (int) (terms_list.size() * d1); index = (int) IntegerMath.limit(0, index, terms_list.size()); final String term = terms_list.getElementAt(index).getWord(); stack.addAll(JUtils.split(term, " ")); } // stack.print(""); for (int k = 0; k < stack.size(); k++) { result = result + " " + stack.getElementAt(k); } return result; }
public static FolderSupportingIndex rebuild(final FolderSupportingIndexBuilderParams params) throws IOException { final File file = params.getTarget(); if (!file.isFolder()) { Err.reportError("Is not folder " + file); } if (params.getDebug()) { L.d(file); } final ChildrenList children = file.listDirectChildren(); // children.print("children"); final FolderSupportingIndex desc = new FolderSupportingIndex(); Collections.scanCollection( children, new CollectionScanner<File>() { @Override public void scanElement(final File e, final long i) { if (e.getName().startsWith(FolderSupportingIndex.FILE_NAME)) { return; } final FolderSupportingIndexEntry entry = new FolderSupportingIndexEntry(); try { entry.name = e.getName(); entry.is_file = e.isFile(); entry.is_folder = e.isFolder(); entry.lastModified = e.lastModified(); entry.size = e.getSize(); if (!params.ignoreHashSum() && e.isFile()) { entry.hash = e.calculateHash().getMD5HashHexString(); } desc.entries.put(entry.name, entry); if (e.isFolder()) { final FolderSupportingIndexBuilderParams paramsNext = params.copy(); paramsNext.setTarget(e); if (params.rebuidOnlyForRoot()) { paramsNext.setNoOutput(true); } else { paramsNext.setNoOutput(params.noOutput()); // paramsNext.noOutput = params.noOutput; } paramsNext.setIgnoreHashSum(params.ignoreHashSum()); paramsNext.setDebug(params.getDebug()); if (params.recoursive()) { final FolderSupportingIndex sublevel = rebuild(paramsNext); desc.children.put(entry.name, sublevel); } } } catch (final IOException e1) { Err.reportError(e1); } } }); if (!params.noOutput()) { FolderSupportingIndex deckCheck = null; { final File desc_file = file.child(FolderSupportingIndex.FILE_NAME); L.d("writing", desc_file); final FileOutputStream os = desc_file.newOutputStream(); os.open(); HTTPOperator.encode(desc, os); os.close(); final ByteArray dataCheck = desc_file.readBytes(); deckCheck = HTTPOperator.decode(dataCheck); } { final File desc_file_json = file.child(FolderSupportingIndex.FILE_NAME + ".json"); L.d("writing", desc_file_json); final JsonString stringData = Json.serializeToString(desc); final JsonString testStringData = Json.serializeToString(deckCheck); final String s1 = stringData.toString(); final String s2 = testStringData.toString(); if (!s1.equals(s2)) { L.e(" stringData " + s1.length()); L.e("testStringData " + s2.length()); L.e(desc_file_json); // for (int k = 0; k < s1.length(); k++) { // final char c1 = s1.charAt(k); // final char c2 = s2.charAt(k); // if (c1 != c2) { //// L.d_appendChars("#");L.d_appendChars("#");L.d_appendChars("#"); //// L.d_appendChars("" + c1); //// L.d_appendChars("#"); // } else { //// L.d_appendChars("" + c1); // } // } L.e("decoder fails"); } final String data = stringData.toString(); desc_file_json.writeString(data); } } return desc; }