public static void printUsage(Options options) { String usage = String.format("%s [options] <dir_path>", TOOL_NAME); String header = System.lineSeparator() + "Bulk load the sstables found in the directory <dir_path> to the configured cluster." + "The parent directories of <dir_path> are used as the target keyspace/table name. " + "So for instance, to load an sstable named Standard1-g-1-Data.db into Keyspace1/Standard1, " + "you will need to have the files Standard1-g-1-Data.db and Standard1-g-1-Index.db into a directory /path/to/Keyspace1/Standard1/."; String footer = System.lineSeparator() + "You can provide cassandra.yaml file with -f command line option to set up streaming throughput, client and server encryption options. " + "Only stream_throughput_outbound_megabits_per_sec, server_encryption_options and client_encryption_options are read from yaml. " + "You can override options read from cassandra.yaml with corresponding command line options."; new HelpFormatter().printHelp(usage, header, options, footer); }
private synchronized void print( String url, String startTime, String endTime, int fileSize, String cacheBool, int statusCode) { String entry = url + " " + startTime + " " + endTime + " " + fileSize + " " + cacheBool + " " + statusCode + System.lineSeparator(); try { Files.write(Paths.get("log.txt"), entry.getBytes(), StandardOpenOption.APPEND); } catch (IOException ex) { Logger.getLogger(doComms.class.getName()).log(Level.SEVERE, null, ex); } }
final class HttpRequest implements Runnable { static final String CRLF = "\r\n"; static final String NEWLINE = System.lineSeparator(); static final int CHUNCKED_BYTES = 1; private File rootDirectory; private File defaultPage; private SynchronizedQueue<Socket> socketRequestsQueue; public String fullPathForFile; private Crawler serverCrawler; private String execResults = "/execResult.html"; // Constructor public HttpRequest( File rootDirectory, File defaultPage, SynchronizedQueue<Socket> socketRequestsQueue, int threadNumber, Crawler crawler) { this.rootDirectory = rootDirectory; this.defaultPage = defaultPage; this.socketRequestsQueue = socketRequestsQueue; this.serverCrawler = crawler; } // Implement the run() method of the Runnable interface. public void run() { try { processRequest(); } catch (Exception e) { System.out.println("Request generating error: " + e); } } private void processRequest() throws Exception { while (true) { Socket socket = socketRequestsQueue.take(); DataOutputStream socketOutputStream = new DataOutputStream(socket.getOutputStream()); HtmlRequest htmlRequest = readRequest(socket); // If the request is empty than the socket was closed on the other side if (htmlRequest.equals(null)) { try { socket.close(); } catch (Exception e) { System.out.println("Error on trying to close socket on empty request: " + e.toString()); } continue; } HtmlResponse responseToClient; if (!htmlRequest.isLegalRequest) { // The request format is illegal responseToClient = respond400(htmlRequest); } else if (!legalRequestType(htmlRequest)) { // The request method is unimplemented responseToClient = respond501(htmlRequest); } else if (directRequestToResultPages(htmlRequest)) { responseToClient = respond403(htmlRequest); } else { if (!htmlRequest.type.equals("TRACE") && !htmlRequest.type.equals("POST")) { boolean isFileLegal = false; try { isFileLegal = checkIfRequestedFileLegal(htmlRequest); } catch (IOException e) { responseToClient = respond500(htmlRequest); } if (!isFileLegal) { responseToClient = respond404(htmlRequest); } else { responseToClient = respond200(htmlRequest); } } else { responseToClient = respond200(htmlRequest); } } try { // Send the status line. socketOutputStream.writeBytes(responseToClient.getStatusLine()); // Send the content type line. socketOutputStream.writeBytes(responseToClient.getContentType()); // Send content length. if (!htmlRequest.isChunked) { socketOutputStream.writeBytes(responseToClient.getContentLengthLine()); } if (htmlRequest.isChunked) { socketOutputStream.writeBytes(responseToClient.getTransferEncoding()); } // Send a blank line to indicate the end of the header lines. socketOutputStream.writeBytes(CRLF); } catch (Exception e) { System.out.println("Writing the header caused an error" + e.toString()); } // Send the content of the HTTP. if (!htmlRequest.type.equals("HEAD")) { sendEntityBodyToClient(socketOutputStream, responseToClient, htmlRequest.isChunked); } // Close streams and socket. try { socketOutputStream.close(); socket.close(); } catch (Exception e) { System.out.println("closing the socket caused an error"); } } } private boolean directRequestToResultPages(HtmlRequest htmlRequest) { String filename = htmlRequest.requestedFile; if (filename.length() < Crawler.RESULTS_PATH_WEB.length()) { return false; } if (filename .substring(0, Crawler.RESULTS_PATH_WEB.length()) .equalsIgnoreCase(Crawler.RESULTS_PATH_WEB)) { String referrer = null; for (String line : htmlRequest.parsedRequest) { if (line.startsWith("Referer:")) { // Referer: http://xxx.xxx.xxx.xxx/ if (line.length() < 16) { return false; } String[] splitReferer = line.substring(16, line.length()).split("/"); if (splitReferer.length < 2 || splitReferer[1].equalsIgnoreCase(defaultPage.getName()) || splitReferer[1].equalsIgnoreCase(Crawler.RESULTS_PATH_WEB.replaceAll("/", "")) || splitReferer[1].equalsIgnoreCase(execResults.replaceAll("/", ""))) { return false; } } } if (referrer == null) { return true; } } return false; } private boolean legalRequestType(HtmlRequest htmlRequest) { if (htmlRequest.type == null) { return false; } if (!htmlRequest.type.equals("POST") && !htmlRequest.type.equals("GET") && !htmlRequest.type.equals("HEAD") && !htmlRequest.type.equals("TRACE")) { return false; } return true; } private HtmlResponse respond200(HtmlRequest htmlRequest) throws IOException, InterruptedException { HtmlResponse response200 = new HtmlResponse(); byte[] bodyInBytes; if (htmlRequest.type.equals("TRACE")) { bodyInBytes = htmlRequest.unparsedRequest.getBytes(); } else if (htmlRequest.type.equals("POST")) { if (htmlRequest.requestedFile.equals("/params_info.html")) { bodyInBytes = makeTable(htmlRequest.parametersInRequestBody); } else if (htmlRequest.requestedFile.equals(execResults)) { System.out.println( "Parameters for Crawler : " + htmlRequest.parametersInRequestBody.toString()); if (serverCrawler.isBusy()) { bodyInBytes = prepareDefaultPage("Crawler is busy"); } else { String crawlerInputCheckResults = checkCrawlerInput(htmlRequest); if (crawlerInputCheckResults == null) { bodyInBytes = activateCrawler(htmlRequest); Thread crawlerThread = new Thread(serverCrawler); crawlerThread.start(); } else { bodyInBytes = prepareDefaultPage(crawlerInputCheckResults); } } } else { bodyInBytes = null; } } else { bodyInBytes = readFileForResponse(htmlRequest); } response200.setEntityBody(bodyInBytes); response200.setStatus(htmlRequest.httpVersion, 200); String contentType; if (!htmlRequest.type.equals("POST")) { contentType = getContentTypeFromFile(htmlRequest.requestedFile); } else { contentType = getContentTypeFromFile(htmlRequest.requestedFile); } response200.setContentTypeLine(contentType); return response200; } private String checkCrawlerInput(HtmlRequest htmlRequest) { String result = null; String domain = htmlRequest.parametersInRequestBody.get("Domain"); String domainFound = Crawler.ParseURL(domain); if (domainFound.charAt(domainFound.length() - 1) == '\\') { domainFound = domainFound.substring(0, domainFound.length() - 1); } try { ClientRequest clientRequest = new ClientRequest(domainFound, ClientRequest.getRequest); if (clientRequest.responseHeaderFields == null) { return "Error connecting to: " + domain + "\n"; } } catch (Exception e) { System.out.println("checkCrawlerInput: clientRequest generated error."); result = "Error connecting to: " + domain + "\n" + e.toString(); e.printStackTrace(); } return result; } private byte[] activateCrawler(HtmlRequest htmlRequest) throws IOException { byte[] bodyInBytes; String domain = htmlRequest.parametersInRequestBody.get("Domain"); boolean ignoreRobots = false; boolean performPortScan = false; if (htmlRequest.parametersInRequestBody.containsKey("portscan")) { performPortScan = true; } if (htmlRequest.parametersInRequestBody.containsKey("robots.txt")) { ignoreRobots = true; } boolean isConfigureSucceeded = serverCrawler.ConfigureCrawler(domain, ignoreRobots, performPortScan); if (isConfigureSucceeded) { bodyInBytes = prepareDefaultPage("Crawler started succesfuly"); System.out.println("Domain is: " + domain); System.out.println("Perform port scan: " + performPortScan); System.out.println("Ignore robots.txt: " + ignoreRobots); } else { bodyInBytes = prepareDefaultPage("Crawler is already running"); } return bodyInBytes; } private byte[] readFileForResponse(HtmlRequest htmlRequest) throws IOException { if (htmlRequest.requestedFile.equals("/") || htmlRequest.requestedFile.equals("/" + defaultPage.getName())) { fullPathForFile = rootDirectory.getCanonicalPath() + "\\" + defaultPage.getName(); return prepareDefaultPage(null); } else { fullPathForFile = rootDirectory.getCanonicalPath() + htmlRequest.requestedFile; } File file = new File(fullPathForFile); byte[] buffer = new byte[(int) file.length()]; FileInputStream fis = new FileInputStream(file); BufferedInputStream bis = new BufferedInputStream(fis); bis.read(buffer, 0, buffer.length); bis.close(); return buffer; } private byte[] prepareDefaultPage(String message) { String head = "<!doctype html><html lang=\"en\"><head><title> Crawler HTML site </title></head>" + "<link href=\"css/style.css\" rel=\"stylesheet\" /><body><div class=\"header\"><h1>Crawler</h1></div>"; String form; if (message != null) { form = "<div class=\"crawlerAnswer\"><h2>" + message + "</h2>" + "<a href=\"/\"><h3>Back to homepage</h3></a></div>"; } else if (serverCrawler.isBusy()) { form = "<div class=\"crawlerAnswer\"><h2>Crawler is already running</h2></div>"; } else { form = "<div class=\"crawlerForm\"><form id=\"generalform\" method=\"post\" action=\"execResult.html\" class=\"crawlerFormTable\">" + "<table><tr><td><h3>Enter Domain</h3></td><td><input type=\"text\" name=\"Domain\"></td></tr><tr>" + "<td><h3><input type=\"checkbox\" name=\"portscan\">Perform full TCP port scan</h3></td></tr><tr>" + "<td><h3><input type=\"checkbox\" name=\"robots.txt\">Disrespect robots.txt</h3></td></tr>" + "<tr><td></td><td><input type=\"submit\" value=\"Start Crawler\"></td></tr></table></form></div>"; } String resultPages = prepareResultPagesSection(); String finish = "</body></html>"; String result = head + form + resultPages + finish; return result.getBytes(); } private String prepareResultPagesSection() { String resultsPath; try { resultsPath = rootDirectory.getCanonicalPath() + Crawler.RESULTS_PATH_LOCAL; } catch (IOException e) { System.out.println("HTTPRequest: Error root directory" + rootDirectory.toString()); return ""; } StringBuilder result = new StringBuilder(); result.append("<div class=\"connectedDomains\"><ul>"); File resultsFolder = new File(resultsPath); if (resultsFolder.exists() && resultsFolder.isDirectory()) { File[] allFiles = resultsFolder.listFiles(); SimpleDateFormat format = new SimpleDateFormat("dd.MM.yyyy-HH:mm"); for (File file : allFiles) { String filename = file.getName(); String domain = Crawler.ResultsFilenameToDomain(filename); Date creationDate = Crawler.ResultsFilenameToDate(filename); String linkFormat = domain + "-" + format.format(creationDate); result.append("<li><a href="); result.append(Crawler.RESULTS_PATH_WEB); result.append(filename); result.append(">"); result.append(linkFormat); result.append("</a></li>"); } result.append("</ul></div>"); } return result.toString(); } private String getContentTypeFromFile(String requestedFile) { String type = "application/octet-stream"; // We will check if the file has a type like .bmp .jpg String[] splitFileRequest = requestedFile.split(Pattern.quote(".")); if (splitFileRequest.length < 2) return type; // If there is a type it is last String typeInRequest = splitFileRequest[splitFileRequest.length - 1]; if (typeInRequest.equalsIgnoreCase("html") || typeInRequest.equalsIgnoreCase("htm")) type = "text/html"; else if (typeInRequest.equalsIgnoreCase("bmp") || typeInRequest.equalsIgnoreCase("png") || typeInRequest.equalsIgnoreCase("jpg") || typeInRequest.equalsIgnoreCase("jpeg") || typeInRequest.equalsIgnoreCase("gif")) type = "image"; else if (typeInRequest.equalsIgnoreCase("ico")) type = "icon"; else if (typeInRequest.equalsIgnoreCase("css")) type = "text/css"; return type; } private HtmlResponse respond404(HtmlRequest htmlRequest) { HtmlResponse response404 = new HtmlResponse(); response404.setStatus(htmlRequest.httpVersion, 404); response404.setContentTypeLine("text/html"); String body404 = "<HTML><HEAD><TITLE>404 Not Found</TITLE></HEAD>" + "<BODY><H1>File Not Found</H1><H2>But have a nice day</H2></BODY></HTML>"; response404.setEntityBody(body404.getBytes()); return response404; } private HtmlResponse respond501(HtmlRequest htmlRequest) { HtmlResponse response501 = new HtmlResponse(); response501.setStatus(htmlRequest.httpVersion, 501); response501.setContentTypeLine("text/html"); String body501 = "<HTML><HEAD><TITLE>Unknown Method</TITLE></HEAD>" + "<BODY><H1>Unknown Method</H1><H2>But may you find knowledge in your path</H2></BODY></HTML>"; response501.setEntityBody(body501.getBytes()); return response501; } private HtmlResponse respond500(HtmlRequest htmlRequest) { HtmlResponse response500 = new HtmlResponse(); response500.setStatus(htmlRequest.httpVersion, 500); response500.setContentTypeLine("text/html"); String body500 = "<HTML><HEAD><TITLE>Internal Server Error</TITLE></HEAD>" + "<BODY><H1>Internal Server Error</H1><H2>This is totally our fault, so relax.</H2></BODY></HTML>"; response500.setEntityBody(body500.getBytes()); return response500; } private HtmlResponse respond400(HtmlRequest htmlRequest) { HtmlResponse response400 = new HtmlResponse(); response400.setStatus(htmlRequest.httpVersion, 400); response400.setContentTypeLine("text/html"); String body400 = "<HTML><HEAD><TITLE>Bad Request</TITLE></HEAD>" + "<BODY><H1>Bad Request</H1><H2>You are not bad, only the request, and requests can change.</H2></BODY></HTML>"; response400.setEntityBody(body400.getBytes()); return response400; } private HtmlResponse respond403(HtmlRequest htmlRequest) { HtmlResponse response403 = new HtmlResponse(); response403.setStatus(htmlRequest.httpVersion, 403); response403.setContentTypeLine("text/html"); String body403 = "<HTML><HEAD><TITLE>403 Forbidden</TITLE></HEAD>" + "<BODY><H1>Forbidden Request</H1><H2>How can something so wrong feel so right?</H2></BODY></HTML>"; response403.setEntityBody(body403.getBytes()); return response403; } private boolean checkIfRequestedFileLegal(HtmlRequest htmlRequest) throws IOException { String requestedFileStr = htmlRequest.requestedFile; // Check if it is default if (requestedFileStr.equals("/")) { htmlRequest.requestedFile = "/" + defaultPage.toString(); requestedFileStr = htmlRequest.requestedFile; } String requestedFileFullPath = rootDirectory.getCanonicalPath() + requestedFileStr; File requestedFile = new File(requestedFileFullPath); // Checking that the requested file path is under the root directory if (!requestedFile.getAbsolutePath().startsWith(rootDirectory.getCanonicalPath())) { return false; } // Checking if it is a directory if (requestedFile.isDirectory()) { return false; } // Check if the file exists if (!requestedFile.exists()) { return false; } return true; } private HtmlRequest readRequest(Socket socket) throws IOException { BufferedReader requestBufferedReader = new BufferedReader(new InputStreamReader(socket.getInputStream())); StringBuilder requestStringBuilder = new StringBuilder(); try { String line = requestBufferedReader.readLine(); while (!line.isEmpty()) { requestStringBuilder.append(line + NEWLINE); line = requestBufferedReader.readLine(); } } catch (IOException e) { System.out.println("An error occured while reading from the socket: " + e.toString()); } if (requestStringBuilder.toString().isEmpty()) { return null; } HtmlRequest htmlRequest = new HtmlRequest(requestStringBuilder.toString()); if (htmlRequest.type.equals("POST") || htmlRequest.type.equals("TRACE")) { htmlRequest.getParametersFromBody(requestBufferedReader); } return htmlRequest; } private void sendEntityBodyToClient( DataOutputStream socketOutputStream, HtmlResponse htmlResponse, boolean isChunked) throws IOException { byte[] content = htmlResponse.getEntityBody(); if (!isChunked) { try { socketOutputStream.write(content, 0, content.length); socketOutputStream.flush(); } catch (IOException e) { System.out.println("Writing the answer caused an error" + e.toString()); } } else { int currentIndexStart = 0; int currentIndexEnd = Math.min(CHUNCKED_BYTES - 1, content.length - 1); int lengthOfBytesSent = currentIndexEnd - currentIndexStart + 1; while (currentIndexStart < content.length - 1) { socketOutputStream.writeBytes(Integer.toHexString(lengthOfBytesSent) + CRLF); socketOutputStream.write(content, currentIndexStart, lengthOfBytesSent); socketOutputStream.writeBytes(CRLF); socketOutputStream.flush(); currentIndexStart = currentIndexEnd + 1; currentIndexEnd = Math.min(currentIndexStart + CHUNCKED_BYTES - 1, content.length - 1); lengthOfBytesSent = currentIndexEnd - currentIndexStart + 1; } socketOutputStream.writeBytes("0" + CRLF); socketOutputStream.writeBytes(CRLF); socketOutputStream.flush(); } } private byte[] makeTable(HashMap<String, String> map) { StringBuilder table = new StringBuilder(); // if there are no parameters if (map.isEmpty()) { table.append( "<!DOCTYPE html>" + "<HTML>" + "<HEAD><TITLE>" + "table" + "</TITLE></HEAD>" + "<BODY>" + "<H1>" + " no parameters" + "</H1><br><br><br>" + "</BODY></HTML>"); } else { table.append( "<!DOCTYPE html>" + "<HTML>" + "<HEAD lang='en'><TITLE>" + "table" + "</TITLE></HEAD>" + "<BODY>" + "<H1>" + " table with parameters the client sent" + "</H1><br><br><br>" + "<table border ='2' style='width:30%'>"); Iterator<String> keySetIterator = map.keySet().iterator(); while (keySetIterator.hasNext()) { String key = keySetIterator.next(); table.append("<tr>" + "<td>" + key + "</td>" + "<td>" + map.get(key) + "</td>" + "</tr>"); } table.append("</table>" + "</BODY></HTML>"); } byte[] result = table.toString().getBytes(); return result; } }