/**
   * Send the command to Solr using a GET
   *
   * @param queryString
   * @param url
   * @return
   * @throws IOException
   */
  private static String sendGetCommand(String queryString, String url) throws IOException {
    String results = null;
    HttpClient client = new HttpClient();
    GetMethod get = new GetMethod(url);
    get.setQueryString(queryString.trim());

    client.executeMethod(get);
    try {
      // Execute the method.
      int statusCode = client.executeMethod(get);

      if (statusCode != HttpStatus.SC_OK) {
        System.err.println("Method failed: " + get.getStatusLine());
        results = "Method failed: " + get.getStatusLine();
      }

      results = getStringFromStream(get.getResponseBodyAsStream());
    } catch (HttpException e) {
      System.err.println("Fatal protocol violation: " + e.getMessage());
      e.printStackTrace();
    } catch (IOException e) {
      System.err.println("Fatal transport error: " + e.getMessage());
      e.printStackTrace();
    } finally {
      // Release the connection.
      get.releaseConnection();
    }
    return results;
  }
 public static void main(String[] args) {
   HttpClient client = new HttpClient(); // Create an instance of HttpClient.
   GetMethod method = new GetMethod(url); // Create a method instance.	
   method
       .getParams()
       .setParameter(
           HttpMethodParams.RETRY_HANDLER,
           new DefaultHttpMethodRetryHandler(
               3, false)); // Provide custom retry handler is necessary
   try {
     int statusCode = client.executeMethod(method); // Execute the method.
     if (statusCode != HttpStatus.SC_OK) {
       System.err.println("Method failed: " + method.getStatusLine());
     }
     byte[] responseBody = method.getResponseBody(); // Read the response body.
     System.out.println(
         new String(
             responseBody)); // Deal with the response.// Use caution: ensure correct character
                             // encoding and is not binary data
   } catch (HttpException e) {
     System.err.println("Fatal protocol violation: " + e.getMessage());
     e.printStackTrace();
   } catch (IOException e) {
     System.err.println("Fatal transport error: " + e.getMessage());
     e.printStackTrace();
   } finally {
     method.releaseConnection(); // Release the connection.
   }
 }
 public List<DatastreamProfile> getDatastreamHistory(String pid, String dsId)
     throws HttpException, IOException, FedoraException, SAXException,
         ParserConfigurationException, XPathExpressionException {
   String url =
       this.fedoraBaseUrl + "/objects/" + pid + "/datastreams/" + dsId + "/history?format=xml";
   GetMethod get = new GetMethod(url);
   this.client.executeMethod(get);
   if (get.getStatusCode() == 200) {
     List<DatastreamProfile> history = new ArrayList<DatastreamProfile>();
     Document xml = this.getDocumentBuilder().parse(get.getResponseBodyAsStream());
     NodeList nl =
         (NodeList)
             this.getXPath()
                 .evaluate(
                     "/fedora-management:datastreamHistory/fedora-management:datastreamProfile",
                     xml,
                     XPathConstants.NODESET);
     for (int i = 0; i < nl.getLength(); i++) {
       Element el = (Element) nl.item(i);
       history.add(new DatastreamProfile(this, el));
     }
     return history;
   } else {
     throw new FedoraException("REST action \"" + url + "\" failed: " + get.getStatusLine());
   }
 }
  /**
   * Fetches the object profile XML and parse out the property value for the given property.
   *
   * @param pid the pid of the object to be queried
   * @param dsName the name of the datastream who's property is being queried
   * @param prop the property to query.
   * @return the value of the property
   */
  public String getDatastreamProperty(
      String pid, String dsName, DatastreamProfile.DatastreamProperty prop)
      throws HttpException, IOException, SAXException, ParserConfigurationException,
          FedoraException, XPathExpressionException {
    String url = this.fedoraBaseUrl + "/objects/" + pid + "/datastreams/" + dsName + "?format=xml";
    GetMethod get = new GetMethod(url);
    this.client.executeMethod(get);
    if (get.getStatusCode() == 200) {
      Document xml = this.getDocumentBuilder().parse(get.getResponseBodyAsStream());

      // compatible with fedora 3.4
      String value =
          (String)
              this.getXPath()
                  .evaluate(
                      "/fedora-management:datastreamProfile/fedora-management:"
                          + prop.getPropertyName(),
                      xml,
                      XPathConstants.STRING);
      if (value != null) {
        return value;
      } else {
        // compatible with fedora 3.2
        return (String)
            this.getXPath()
                .evaluate(
                    "/datastreamProfile/" + prop.getPropertyName(), xml, XPathConstants.STRING);
      }

    } else {
      throw new FedoraException("REST action \"" + url + "\" failed: " + get.getStatusLine());
    }
  }
Beispiel #5
0
  private String getXmlToken() {
    String token = null;
    HttpClient httpclient = getHttpClient();
    GetMethod get = new GetMethod(restUrl.getTokenUrl());
    get.addRequestHeader("Accept", "application/xml");

    NameValuePair userParam = new NameValuePair(USERNAME_PARAM, satelite.getUser());
    NameValuePair passwordParam = new NameValuePair(PASSWORD_PARAM, satelite.getPassword());
    NameValuePair[] params = new NameValuePair[] {userParam, passwordParam};

    get.setQueryString(params);
    try {
      int statusCode = httpclient.executeMethod(get);
      if (statusCode != HttpStatus.SC_OK) {
        logger.error("Method failed: " + get.getStatusLine());
      }
      token = parseToken(get.getResponseBodyAsString());
    } catch (HttpException e) {
      logger.error(e.getMessage());
    } catch (IOException e) {
      logger.error(e.getMessage());
    } catch (ParserConfigurationException e) {
      logger.error(e.getMessage());
    } catch (SAXException e) {
      logger.error(e.getMessage());
    } finally {
      get.releaseConnection();
    }

    return token;
  }
  /**
   * Retrieves the build number of this CloudTest server. Postcondition: The build number returned
   * is never null.
   */
  public VersionNumber getBuildNumber() throws IOException {
    if (url == null) {
      // User didn't enter a value in the Configure Jenkins page.
      // Nothing we can do.
      throw new IllegalStateException("No URL has been configured for this CloudTest server.");
    }

    final String[] v = new String[1];
    try {
      HttpClient hc = createClient();

      GetMethod get = new GetMethod(url);
      hc.executeMethod(get);

      if (get.getStatusCode() != 200) {
        throw new IOException(get.getStatusLine().toString());
      }

      SAXParser sp = SAXParserFactory.newInstance().newSAXParser();
      sp.parse(
          get.getResponseBodyAsStream(),
          new DefaultHandler() {
            @Override
            public InputSource resolveEntity(String publicId, String systemId)
                throws IOException, SAXException {
              if (systemId.endsWith(".dtd")) return new InputSource(new StringReader(""));
              return null;
            }

            @Override
            public void startElement(
                String uri, String localName, String qName, Attributes attributes)
                throws SAXException {
              if (qName.equals("meta")) {
                if ("buildnumber".equals(attributes.getValue("name"))) {
                  v[0] = attributes.getValue("content");
                  throw new SAXException("found");
                }
              }
            }
          });
      LOGGER.warning("Build number not found in " + url);
    } catch (SAXException e) {
      if (v[0] != null) return new VersionNumber(v[0]);

      LOGGER.log(Level.WARNING, "Failed to load " + url, e);
    } catch (ParserConfigurationException e) {
      throw new Error(e);
    }

    // If we reach this point, then we failed to extract the build number.
    throw new IOException(
        "Failed to extract build number from \'"
            + this.getDescriptor().getDisplayName()
            + "\': <"
            + url
            + ">.");
  }
  private String sendRequest(TicketsRequest request) throws HttpException, IOException {

    String html = "";
    GetMethod get = new GetMethod(url2);
    client.executeMethod(get);

    int statusCodeInit = client.executeMethod(get);
    if (statusCodeInit != HttpStatus.SC_OK) {
      logger.error("sendRequest method failed. Get method failed: " + get.getStatusLine());
      return null;
    } else {
      html = IOUtils.toString(get.getResponseBodyAsStream(), "UTF-8");
    }
    parseToken(html);

    client.getHttpConnectionManager().getParams().setSoTimeout(10000);
    PostMethod post = new PostMethod(URL);

    post.addRequestHeader("Accept", "*/*");
    post.addRequestHeader("Accept-Encoding", "gzip, deflate");
    post.addRequestHeader("Accept-Language", "uk,ru;q=0.8,en-US;q=0.5,en;q=0.3");
    //		post.addRequestHeader("Cache-Control", "no-cache");
    post.addRequestHeader("Connection", "keep-alive");
    //		post.addRequestHeader("Content-Length", "288"); //202. 196
    post.addRequestHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
    post.addRequestHeader("GV-Ajax", "1");
    post.addRequestHeader("GV-Referer", "http://booking.uz.gov.ua/");
    post.addRequestHeader("GV-Screen", "1920x1080");
    post.addRequestHeader("GV-Token", token);
    post.addRequestHeader("GV-Unique-Host", "1");
    post.addRequestHeader("Host", "booking.uz.gov.ua");
    //		post.addRequestHeader("Pragma", "no-cache");
    post.addRequestHeader("Origin", "http://booking.uz.gov.ua");
    post.addRequestHeader("Referer", "http://booking.uz.gov.ua/");
    post.addRequestHeader(
        "User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/39.0");

    post.addParameter("another_ec", "0");
    post.addParameter("date_dep", new SimpleDateFormat("dd.MM.yyyy").format(request.date));
    post.addParameter("search", "");
    post.addParameter("station_from", request.from.getName());
    post.addParameter("station_id_from", request.from.getStationId());
    post.addParameter("station_id_till", request.till.getStationId());
    post.addParameter("station_till", request.till.getName());
    post.addParameter("time_dep", "00:00");
    post.addParameter("time_dep_till", "");

    int statusCode = client.executeMethod(post);
    if (statusCode != HttpStatus.SC_OK) {
      logger.error("sendRequest method failed. Post method failed: " + post.getStatusLine());
      return null;
    }

    return post.getResponseBodyAsString();
  }
 public static void commit(String updateUrl) throws IOException {
   String url = updateUrl + "?stream.body=%3Ccommit/%3E";
   GetMethod get = new GetMethod(url);
   try {
     HttpClient client = new HttpClient();
     client.executeMethod(get);
     int status = get.getStatusCode();
     if (status != HttpStatus.SC_OK) {
       throw new RuntimeException("REST action \"" + url + "\" failed: " + get.getStatusLine());
     }
   } finally {
     get.releaseConnection();
   }
 }
  /* 下载 url 指向的网页 */
  public String downloadFile(String url) {
    String filePath = null;
    /* 1.生成 HttpClinet 对象并设置参数 */
    HttpClient httpClient = new HttpClient();
    // 设置 Http 连接超时 5s
    httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);

    /* 2.生成 GetMethod 对象并设置参数 */
    GetMethod getMethod = new GetMethod(url);
    // 设置 get 请求超时 5s
    getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
    // 设置请求重试处理
    getMethod
        .getParams()
        .setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());

    /* 3.执行 HTTP GET 请求 */
    try {
      int statusCode = httpClient.executeMethod(getMethod);
      // 判断访问的状态码
      if (statusCode != HttpStatus.SC_OK) {
        System.err.println("Method failed: " + getMethod.getStatusLine());
        filePath = null;
      }

      /* 4.处理 HTTP 响应内容 */
      byte[] responseBody = getMethod.getResponseBody(); // 读取为字节数组
      // 根据网页 url 生成保存时的文件名
      filePath =
          "temp\\" + getFileNameByUrl(url, getMethod.getResponseHeader("Content-Type").getValue());
      // createFile(filePath);
      FileUtils.createFile(filePath);
      saveToLocal(responseBody, filePath, false);
    } catch (HttpException e) {
      // 发生致命的异常,可能是协议不对或者返回的内容有问题
      System.out.println("Please check your provided http address!");
      e.printStackTrace();
    } catch (IOException e) {
      // 发生网络异常
      e.printStackTrace();
    } finally {
      // 释放连接
      getMethod.releaseConnection();
    }
    return filePath;
  }
Beispiel #10
0
 // ���� URL ָ�����ҳ
 public String downloadFile(String url) {
   String filePath = null;
   // 1.���� HttpClinet��������
   HttpClient httpClient = new HttpClient();
   // ���� HTTP���ӳ�ʱ 5s
   httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
   // 2.���� GetMethod��������
   GetMethod getMethod = new GetMethod(url);
   // ���� get����ʱ 5s
   getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
   // �����������Դ���
   getMethod
       .getParams()
       .setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
   // 3.ִ��GET����
   try {
     int statusCode = httpClient.executeMethod(getMethod);
     // �жϷ��ʵ�״̬��
     if (statusCode != HttpStatus.SC_OK) {
       System.err.println("Method failed: " + getMethod.getStatusLine());
       filePath = null;
     }
     // 4.���� HTTP ��Ӧ����
     byte[] responseBody = getMethod.getResponseBody(); // ��ȡΪ�ֽ�����
     // ������ҳ url ���ɱ���ʱ���ļ���
     filePath =
         "temp\\" + getFileNameByUrl(url, getMethod.getResponseHeader("Content-Type").getValue());
     File file = new File(filePath);
     if (!file.getParentFile().exists()) {
       file.getParentFile().mkdir();
     }
     saveToLocal(responseBody, filePath);
   } catch (HttpException e) {
     // �����������쳣��������Э�鲻�Ի��߷��ص�����������
     System.out.println("�������http��ַ�Ƿ���ȷ");
     e.printStackTrace();
   } catch (IOException e) {
     // ���������쳣
     e.printStackTrace();
   } finally {
     // �ͷ�����
     getMethod.releaseConnection();
   }
   return filePath;
 }
 public static String getGetResponse(String url) {
   String html = "";
   GetMethod getMethod = new GetMethod(url);
   try {
     int statusCode = httpClient.executeMethod(getMethod);
     if (statusCode != HttpStatus.SC_OK) {
       System.err.println("Method failed: " + getMethod.getStatusLine());
     }
     html = getMethod.getResponseBodyAsString();
   } catch (HttpException e) {
     e.printStackTrace();
   } catch (IOException e) {
     e.printStackTrace();
   } finally {
     getMethod.releaseConnection();
   }
   return html;
 }
Beispiel #12
0
  @Override
  public void process(Exchange exchange) throws Exception {
    String q = query;
    String location = exchange.getIn().getHeader(WeatherConstants.WEATHER_LOCATION, String.class);
    if (location != null) {
      q = getEndpoint().getConfiguration().getQuery(location);
    }

    HttpClient httpClient = ((WeatherComponent) getEndpoint().getComponent()).getHttpClient();
    GetMethod method = new GetMethod(q);
    try {
      log.debug("Going to execute the Weather query {}", q);
      int statusCode = httpClient.executeMethod(method);
      if (statusCode != HttpStatus.SC_OK) {
        throw new IllegalStateException(
            "Got the invalid http status value '"
                + method.getStatusLine()
                + "' as the result of the query '"
                + query
                + "'");
      }
      String weather =
          getEndpoint()
              .getCamelContext()
              .getTypeConverter()
              .mandatoryConvertTo(String.class, method.getResponseBodyAsStream());
      log.debug("Got back the Weather information {}", weather);

      if (ObjectHelper.isEmpty(weather)) {
        throw new IllegalStateException(
            "Got the unexpected value '" + weather + "' as the result of the query '" + q + "'");
      }

      String header = getEndpoint().getConfiguration().getHeaderName();
      if (header != null) {
        exchange.getIn().setHeader(header, weather);
      } else {
        exchange.getIn().setBody(weather);
      }
      exchange.getIn().setHeader(WeatherConstants.WEATHER_QUERY, q);
    } finally {
      method.releaseConnection();
    }
  }
Beispiel #13
0
  @Override
  public void emitTuples() {

    try {
      int statusCode = client.executeMethod(method);
      if (statusCode != HttpStatus.SC_OK) {
        logger.error("Method failed: " + method.getStatusLine());
      } else {
        InputStream istream = method.getResponseBodyAsStream();
        // Process response
        InputStreamReader isr = new InputStreamReader(istream);
        CSVReader reader = new CSVReader(isr);
        List<String[]> myEntries = reader.readAll();
        for (String[] stringArr : myEntries) {
          ArrayList<String> tuple = new ArrayList<String>(Arrays.asList(stringArr));
          if (tuple.size() != 4) {
            return;
          }
          // input csv is <Symbol>,<Price>,<Volume>,<Time>
          String symbol = tuple.get(0);
          double currentPrice = Double.valueOf(tuple.get(1));
          long currentVolume = Long.valueOf(tuple.get(2));
          String timeStamp = tuple.get(3);
          long vol = currentVolume;
          // Sends total volume in first tick, and incremental volume afterwards.
          if (lastVolume.containsKey(symbol)) {
            vol -= lastVolume.get(symbol);
          }

          if (vol > 0 || outputEvenIfZeroVolume) {
            price.emit(new KeyValPair<String, Double>(symbol, currentPrice));
            volume.emit(new KeyValPair<String, Long>(symbol, vol));
            time.emit(new KeyValPair<String, String>(symbol, timeStamp));
            lastVolume.put(symbol, currentVolume);
          }
        }
      }
      Thread.sleep(readIntervalMillis);
    } catch (InterruptedException ex) {
      logger.debug(ex.toString());
    } catch (IOException ex) {
      logger.debug(ex.toString());
    }
  }
Beispiel #14
0
 void loadBinary(String uri, int index, Target target) throws RepositoryException, IOException {
   GetMethod method = new GetMethod(uri);
   try {
     int statusCode = client.executeMethod(method);
     if (statusCode == DavServletResponse.SC_OK) {
       target.setStream(method.getResponseBodyAsStream());
     } else {
       throw ExceptionConverter.generate(
           new DavException(
               statusCode,
               ("Unable to load binary at "
                   + uri
                   + " - Status line = "
                   + method.getStatusLine().toString())));
     }
   } finally {
     method.releaseConnection();
   }
 }
  /**
   * Get data set from CKAN repository
   *
   * <p>4 Big Source --> Direct Link
   */
  @SuppressWarnings("unchecked")
  private void getPackageList() {
    HttpClient client = new HttpClient();
    LOG.info(split.getAction());
    GetMethod method = new GetMethod(split.getAction());
    method
        .getParams()
        .setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));

    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method
        .getParams()
        .setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");

    try {
      int statusCode = client.executeMethod(method);

      if (statusCode != HttpStatus.SC_OK) {
        throw new Exception("Method failed: " + method.getStatusLine());
      }
      byte[] responseBody = method.getResponseBody();
      LOG.debug(new String(responseBody));
      setOutput(new String(responseBody));

      Document document = Document.parse(getOutput());

      if (document.containsKey("result")) {

        ArrayList<Document> docs =
            (ArrayList<Document>) ((Document) document.get("result")).get("results");

        for (Document doc : docs) {
          this.dataset.add(doc.getString("id"));
        }

        LOG.info("SANTA MARIA CKAN3 RECORD READER found" + this.dataset.size());
      }
    } catch (Exception e) {
      LOG.error(e);
    } finally {
      method.releaseConnection();
    }
  }
  public static void executeHttpGet() throws Exception {
    // Create an instance of HttpClient.
    HttpClient client = new HttpClient();

    // Create a method instance.
    GetMethod method = new GetMethod(url);
    method.addRequestHeader("command", "addCredit");
    //	    method.addRequestHeader("value", "addCredit");

    // Provide custom retry handler is necessary
    method
        .getParams()
        .setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));

    try {
      // Execute the method.
      int statusCode = client.executeMethod(method);

      if (statusCode != HttpStatus.SC_OK) {
        System.err.println("Method failed: " + method.getStatusLine());
      }

      // Read the response body.
      byte[] responseBody = method.getResponseBody();
      System.out.println(responseBody);
      System.out.println(method.getResponseHeader("response"));

      // Deal with the response.
      // Use caution: ensure correct character encoding and is not binary data
      System.out.println(new String(responseBody));

    } catch (HttpException e) {
      System.err.println("Fatal protocol violation: " + e.getMessage());
      e.printStackTrace();
    } catch (IOException e) {
      System.err.println("Fatal transport error: " + e.getMessage());
      e.printStackTrace();
    } finally {
      // Release the connection.
      method.releaseConnection();
    }
  }
  /**
   * Test
   *
   * @param pArgs
   * @throws Exception
   */
  @SuppressWarnings("unchecked")
  public static void main(String[] pArgs) throws Exception {
    GetCountByCkan3("http://catalog.data.gov/api/action/package_search?start=0&rows=1");

    HttpClient client = new HttpClient();
    GetMethod method =
        new GetMethod("http://catalog.data.gov/api/action/package_search?start=0&rows=10");
    method
        .getParams()
        .setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));

    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method
        .getParams()
        .setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");

    try {
      int statusCode = client.executeMethod(method);

      if (statusCode != HttpStatus.SC_OK) {
        throw new Exception("Method failed: " + method.getStatusLine());
      }
      byte[] responseBody = method.getResponseBody();
      LOG.info(new String(responseBody));

      Document document = Document.parse(new String(responseBody));

      if (document.containsKey("result")) {

        ArrayList<Document> docs =
            (ArrayList<Document>) ((Document) document.get("result")).get("results");

        for (Document doc : docs) {
          LOG.info(doc.getString("id"));
        }
      }
    } catch (Exception e) {
      LOG.error(e);
    } finally {
      method.releaseConnection();
    }
  }
 public InputStream getDatastream(String pid, String dsName, String asOfDateTime)
     throws IOException {
   String url =
       this.fedoraBaseUrl
           + "/objects/"
           + pid
           + "/datastreams/"
           + dsName
           + "/content"
           + (asOfDateTime != null
               ? "?asOfDateTime=" + URLEncoder.encode(asOfDateTime, "UTF-8")
               : "");
   GetMethod get = new GetMethod(url);
   this.client.executeMethod(get);
   int status = this.client.executeMethod(get);
   if (status != HttpStatus.SC_CREATED && status != HttpStatus.SC_OK) {
     throw new RuntimeException("REST action \"" + url + "\" failed: " + get.getStatusLine());
   }
   return get.getResponseBodyAsStream();
 }
  @Override
  public String execute() throws Exception {

    request.setAttribute("ips", ips);
    request.setAttribute("startpage", pageno);

    String connip = ips[pageno];
    // 连接url
    String conpath = "/pool/status";
    HttpClient httpClient = new HttpClient();
    GetMethod getMethod = new GetMethod("http://" + connip + conpath);
    getMethod
        .getParams()
        .setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
    try {
      // 执行getMethod
      int statusCode = httpClient.executeMethod(getMethod);
      if (statusCode != HttpStatus.SC_OK) {
        System.err.println("Method failed: " + getMethod.getStatusLine());
      }
      // 读取内容
      byte[] responseBody = getMethod.getResponseBody();
      // 处理内容
      JSONObject jo = JSONObject.fromObject(new String(responseBody));

      cinfo = (ConnInfo) jo.toBean(jo, ConnInfo.class);
    } catch (HttpException e) {
      // 发生致命的异常,可能是协议不对或者返回的内容有问题
      System.err.println("Please check your provided http address!");
      e.printStackTrace();
      return ERROR;
    } catch (IOException e) {
      // 发生网络异常
      e.printStackTrace();
      return ERROR;
    } finally {
      // 释放连接
      getMethod.releaseConnection();
    }
    return SUCCESS;
  }
  /**
   * Returns list with all stations with their ID's from the server.
   *
   * @return List<Station>
   * @throws HttpException
   * @throws IOException
   */
  public List<Station> getAllStations() {
    if (stations != null && !stations.isEmpty()) {
      return stations;
    }

    String url = "http://booking.uz.gov.ua/purchase/station/";
    char letter;
    GetMethod get = null;
    String jsonResp = null;
    List<Station> myStations = new ArrayList<>();

    try {
      for (int i = 0; i < 32; i++) {
        letter = (char) (1072 + i); // 1072 - rus 'a' in ASCII
        String currentUrl = URIUtil.encodeQuery(url + letter);
        get = new GetMethod(currentUrl);
        int statusCode = client.executeMethod(get);
        if (statusCode != HttpStatus.SC_OK) {
          logger.error("getAllStations method failed. Get method failed: " + get.getStatusLine());
          return null;
        }
        jsonResp =
            IOUtils.toString(
                get.getResponseBodyAsStream(), "UTF-8"); // get.getResponseBodyAsString();
        StationsListJson resp = new ObjectMapper().readValue(jsonResp, StationsListJson.class);
        myStations.addAll(resp.getStations());
      }
    } catch (HttpException e) {
      logger.error("Could not retrieve stations." + e.getMessage(), e);
      throw new RuntimeException("Could not retrieve stations", e);
    } catch (IOException e) {
      logger.error("Could not retrieve stations." + e.getMessage(), e);
      throw new RuntimeException("Could not retrieve stations", e);
    }
    Collections.sort(myStations);
    return stations = myStations;
  }
  public boolean testHttpConnection() {
    boolean result = true;

    HttpClient httpclient = new HttpClient();
    GetMethod httpget = new GetMethod("https://www.verisign.com/");
    try {
      try {
        httpclient.executeMethod(httpget);
      } catch (Exception e) {
        result = false;
      }

      System.out.println(httpget.getStatusLine());

    } finally {
      httpget.releaseConnection();
    }

    if (httpget.getStatusCode() != 200) {
      result = false;
    }

    return result;
  }
  /**
   * Utility method to send message to kannel from this servlet.
   *
   * @param message
   */
  public void sendMessageToKannel(
      String message, String phoneNumber, String username, String password) {

    String smsMessage = "";
    try {
      smsMessage = URLEncoder.encode(message, "UTF-8");

    } catch (UnsupportedEncodingException ex) {
      logger.debug(
          SendSmsToKannelService.class
              + ":sendMessageToKannel() has thrown an exception\n"
              + ex.getMessage());
    }

    String kannelUrl = "http://10.210.7.193:13009/cgi-bin/sendsms";

    String kannelParam =
        "?username="******"&password="******"&to="
            + phoneNumber
            + "&text="
            + smsMessage
            + "&from=0802";

    HttpClient client = new HttpClient();

    GetMethod method = new GetMethod(kannelUrl + kannelParam);

    method
        .getParams()
        .setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));

    try {
      // Execute the method.
      int statusCode = client.executeMethod(method);

      if (statusCode != HttpStatus.SC_OK) {
        System.err.println("Method failed: " + method.getStatusLine());
      }

      // Read the response body.
      byte[] responseBody = method.getResponseBody();

      // Deal with the response.
      // Use caution: ensure correct character encoding and is not binary data
      System.out.println(new String(responseBody));

    } catch (HttpException e) {
      System.err.println("Fatal protocol violation: " + e.getMessage());
      e.printStackTrace();
    } catch (IOException e) {
      System.err.println("Fatal transport error: " + e.getMessage());
      e.printStackTrace();
    } finally {
      // Release the connection.
      method.releaseConnection();
    }
  }
  public static void main(String[] args) throws Exception {
    /**
     * Use the key and secret provided when you registered your application
     * http://www.flickr.com/services/api/keys/apply/
     */
    String key = "";
    String secret = "";

    /**
     * This example assumes that you have uploaded a photo to your Flickr account.
     *
     * <p>The photo id can be found in the URL of the photo
     * http://flickr.com/photos/xxxxxxx@N03/<use_this_photo_id>/
     */
    String photoId = "";

    /**
     * Request a frob to identify the login session. This call requires a signature. The signature
     * starts with your shared secret and is followed by your API key and the method name. The API
     * key and method name are prepended by the words "api_key" and "method" as shown in the
     * following line.
     */
    String methodGetFrob = "flickr.auth.getFrob";
    String sig = secret + "api_key" + key + "method" + methodGetFrob;

    /** The API signature must be MD5 encoded and appended to the request */
    String signature = MD5(sig);

    String request =
        "http://api.flickr.com/services/rest/?method="
            + methodGetFrob
            + "&api_key="
            + key
            + "&api_sig="
            + signature;
    System.out.println("GET frob request: " + request);

    HttpClient client = new HttpClient();
    GetMethod method = new GetMethod(request);

    // Send GET request
    int statusCode = client.executeMethod(method);

    if (statusCode != HttpStatus.SC_OK) {
      System.err.println("Method failed: " + method.getStatusLine());
    }

    InputStream rstream = null;

    // Get the response body
    rstream = method.getResponseBodyAsStream();

    // Retrieve the XML response to the frob request and get the frob value.
    Document response = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(rstream);

    String frob = null;

    // Check if frob is in the response
    NodeList frobResponse = response.getElementsByTagName("frob");
    Node frobNode = frobResponse.item(0);
    if (frobNode != null) {
      frob = frobNode.getTextContent();
      System.out.println("Successfully retrieved frob: " + frob);
    } else {
      printFlickrError(response);
      return;
    }
    System.out.println("Get frob XML response:");
    /**
     * Create a Flickr login link
     * http://www.flickr.com/services/auth/?api_key=[api_key]&perms=[perms]&frob=[frob]&api_sig=[api_sig]
     * We are using "write" for the perms value because we will be rotating an image.
     */
    sig = secret + "api_key" + key + "frob" + frob + "permswrite";
    signature = MD5(sig);
    request =
        "http://www.flickr.com/services/auth/?api_key="
            + key
            + "&perms=write&frob="
            + frob
            + "&api_sig="
            + signature;

    /** Copy/paste the generated link into your favorite web browser and follow the instructions */
    System.out.println(
        "Browse to the following flickr url to authenticate yourself and then press enter.");
    System.out.println(request);
    BufferedReader infile = new BufferedReader(new InputStreamReader(System.in));
    String line = infile.readLine();

    /**
     * Get auth token using frob. Once again, a signature is required for authenticated calls to the
     * Flickr API.
     */
    String methodGetToken = "flickr.auth.getToken";
    sig = secret + "api_key" + key + "frob" + frob + "method" + methodGetToken;
    signature = MD5(sig);
    request =
        "http://api.flickr.com/services/rest/?method="
            + methodGetToken
            + "&api_key="
            + key
            + "&frob="
            + frob
            + "&api_sig="
            + signature;
    System.out.println("Token request: " + request);

    method = new GetMethod(request);

    // Send GET request
    statusCode = client.executeMethod(method);

    if (statusCode != HttpStatus.SC_OK) {
      System.err.println("Method failed: " + method.getStatusLine());
    }

    rstream = null;

    // Get the response body
    rstream = method.getResponseBodyAsStream();
    /** Retrieve the XML response to the token request and get the token value */
    response = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(rstream);

    String token = null;

    // Check if token is in the response
    NodeList tokenResponse = response.getElementsByTagName("token");
    Node tokenNode = tokenResponse.item(0);
    if (tokenNode != null) {
      token = tokenNode.getTextContent();
      System.out.println("Successfully retrieved token: " + token);
    } else {
      printFlickrError(response);
      return;
    }

    /**
     * Now that we have authenticated with Flickr and obtained an auth token with write privileges,
     * it's time to rotate the image.
     *
     * <p>http://api.flickr.com/services/rest/?method=flickr.photos.transform.rotate&
     * api_key=[api_key]&photo_id=[photo_id]&degrees=180&auth_token=[auth_token]&api_sig=[api_sig]
     */
    String methodRotate = "flickr.photos.transform.rotate";
    sig =
        secret
            + "api_key"
            + key
            + "auth_token"
            + token
            + "degrees180"
            + "method"
            + methodRotate
            + "photo_id"
            + photoId;
    signature = MD5(sig);

    /**
     * Create POST data for rotate request. The Flickr API documentation specifies that calls to
     * flickr.photos.transform.rotate be made via HTTP POST.
     */
    request = "http://api.flickr.com/services/rest/";

    PostMethod pmethod = new PostMethod(request);

    pmethod.addParameter("method", methodRotate);
    pmethod.addParameter("photo_id", photoId);
    pmethod.addParameter("degrees", "180");
    pmethod.addParameter("api_key", key);
    pmethod.addParameter("auth_token", token);
    pmethod.addParameter("api_sig", signature);

    // Send POST request
    statusCode = client.executeMethod(pmethod);

    if (statusCode != HttpStatus.SC_OK) {
      System.err.println("Method failed: " + pmethod.getStatusLine());
    }

    rstream = null;

    // Get the response body
    rstream = pmethod.getResponseBodyAsStream();

    /** Retrieve the XML response to the rotate request and get the XML response */
    response = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(rstream);

    String photoid = null;

    // Check if photoid is in the response
    NodeList rotateResponse = response.getElementsByTagName("photoid");
    Node rotateNode = rotateResponse.item(0);
    if (rotateNode != null) {
      photoid = rotateNode.getTextContent();
      System.out.println("Successfully flipped photo " + photoid);
      System.out.println(
          "Refresh the photo in your web browser and you will see it has been flipped.");
    } else {
      printFlickrError(response);
      return;
    }
  }
Beispiel #24
0
  private void buscarOrganizacion(String token) {
    Organizacion organizacion = null;
    if (logger.isDebugEnabled()) logger.debug("Buscando Organizaciones para Satelite: " + satelite);

    HttpClient httpclient = getHttpClient();

    GetMethod get = new GetMethod(restUrl.getOrganizacionUrl());
    get.addRequestHeader("Accept", "application/xml");
    NameValuePair tokenParam = new NameValuePair("token", token);
    int id = 0;
    //		if(satelite.getLastOrgId() != null)
    //			id = satelite.getLastOrgId();

    int leap = idLeap;

    while (true) {
      id = id + 1;

      if (logger.isTraceEnabled()) {
        logger.trace("Buscando Organizacion con id: " + id + " en el Satelite: " + satelite);
      }

      NameValuePair passwordParam = new NameValuePair("id", String.valueOf(id));
      NameValuePair[] params = new NameValuePair[] {tokenParam, passwordParam};

      get.setQueryString(params);
      String queryString = get.getQueryString();

      int statusCode;
      try {
        if (logger.isTraceEnabled()) {
          logger.trace("Query String: " + queryString);
        }

        statusCode = httpclient.executeMethod(get);
        if (statusCode != HttpStatus.SC_OK) {
          logger.error("Method failed: " + get.getStatusLine());
        } else {
          String xmlString = get.getResponseBodyAsString();
          if (logger.isInfoEnabled()) {
            logger.info("Xml Received.");
          }

          String xmlHash = constructXmlHash(xmlString);

          organizacion = organizacionService.findByHash(xmlHash);
          if (organizacion == null) {
            organizacion = parseOrganizacion(xmlString);
            if (organizacion == null) {
              leap--;
              if (leap <= 0) {
                logger.info(
                    "Se llegó al fin de las Organizaciones. Saliendo del Satelite: " + satelite);
                break;
              } else {
                logger.info("Leap: " + leap);
              }
            } else {
              if (organizacion.getSatelites() == null) {
                organizacion.setSatelites(new HashSet<OrganizacionSatelite>());
              }
              boolean sateliteFound = false;
              for (OrganizacionSatelite sat : organizacion.getSatelites()) {
                if (sat.getSatelite().getName().equals(satelite.getName())) {
                  sateliteFound = true;
                  break;
                }
              }
              if (!sateliteFound) {
                OrganizacionSatelite newSat = new OrganizacionSatelite();
                newSat.setSatelite(satelite);
                newSat.setOrganizacion(organizacion);
                organizacion.getSatelites().add(newSat);
              }

              organizacion.setXmlHash(xmlHash);
              organizacionService.saveOrganization(organizacion);

              int numEmpresas = 0;
              if (satelite.getNumEmpresas() != null) {
                numEmpresas = satelite.getNumEmpresas();
              }
              numEmpresas++;

              Date now = new Date();
              satelite.setNumEmpresas(numEmpresas);
              satelite.setLastRetrieval(new Timestamp(now.getTime()));
              satelite.setLastOrgId(id);
              sateliteService.saveSatelite(satelite);
            }
          } else {
            if (logger.isInfoEnabled()) {
              logger.info("Organizacion with id: " + id + " already in centraldir");
            }
          }
        }
      } catch (HttpException e) {
        logger.error(e.getMessage());
      } catch (IOException e) {
        logger.error(e.getMessage());
      } catch (ParserConfigurationException e) {
        logger.error(e.getMessage());
      } catch (SAXException e) {
        logger.error(e.getMessage());
      } catch (ServiceException e) {
        logger.error(e.getMessage());
      } catch (NoSuchAlgorithmException e) {
        logger.error(e.getMessage());
      } finally {
        get.releaseConnection();
      }
    }
  }
  public static Document getRssDocument(Context context, InputStream is, Bundle feedBundle) {
    GetMethod method = null;
    BufferedReader in = null;

    try {
      // Download the feed
      // url is the bitstream contents
      in = new BufferedReader(new InputStreamReader(is));
      String url = in.readLine();

      // need to replace the feed prefix with the HTTP protocol ie replace feed:// with http://
      url = url.replaceAll(PackageDetectorStep.FEED_PREFIX, "http://");

      HttpClient client = new HttpClient();
      // Create a method instance.
      method = new GetMethod(url);

      // Provide custom retry handler is necessary
      method
          .getParams()
          .setParameter(
              HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));

      // Execute the method.
      int statusCode = client.executeMethod(method);

      if (statusCode != HttpStatus.SC_OK) {
        throw new Exception("Method failed: " + method.getStatusLine());
      }

      // Read the response body.
      byte[] responseBody = method.getResponseBody();

      // Store the response in a bitstream so that the ingester can use it later
      if (feedBundle != null) {
        BitstreamFormat bs_format = BitstreamFormat.findByShortDescription(context, "Text");
        BundleUtils.setBitstreamFromBytes(
            feedBundle, Constants.FEED_BUNDLE_CONTENTS_NAME, bs_format, responseBody, false);
      }

      // Now see if we have a RSS v2.0 document
      SAXBuilder builder = new SAXBuilder(false);
      Document xmlDoc = builder.build(new ByteArrayInputStream(responseBody));
      // XMLOutputter outputPretty = new XMLOutputter(Format.getPrettyFormat());
      // log.debug("Got RSS DOCUMENT:");
      // log.debug(outputPretty.outputString(xmlDoc));

      return xmlDoc;

    } catch (Exception e) {
      ExceptionLogger.logException(log, e);
    } finally {
      // Release the connection.
      method.releaseConnection();
      try {
        in.close();
      } catch (Exception e) {
      }
    }

    return null;
  }
  @Override
  protected void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    GetMethod httpGet = null;

    try {
      String url = RequestUtil.getParameter(request, PARAM_URL, defaultProxyUrl);
      String host = url.split("/")[2];

      // Get the proxy parameters
      // TODO: Add dependency injection to set proxy config from GeoNetwork settings, using also the
      // credentials configured
      String proxyHost = System.getProperty("http.proxyHost");
      String proxyPort = System.getProperty("http.proxyPort");

      // Get rest of parameters to pass to proxied url
      HttpMethodParams urlParams = new HttpMethodParams();

      Enumeration paramNames = request.getParameterNames();
      while (paramNames.hasMoreElements()) {
        String paramName = (String) paramNames.nextElement();
        if (!paramName.equalsIgnoreCase(PARAM_URL)) {
          urlParams.setParameter(paramName, request.getParameter(paramName));
        }
      }

      // Checks if allowed host
      if (!isAllowedHost(host)) {
        // throw new ServletException("This proxy does not allow you to access that location.");
        returnExceptionMessage(response, "This proxy does not allow you to access that location.");
        return;
      }

      if (url.startsWith("http://") || url.startsWith("https://")) {
        HttpClient client = new HttpClient();

        // Added support for proxy
        if (proxyHost != null && proxyPort != null) {
          client.getHostConfiguration().setProxy(proxyHost, Integer.valueOf(proxyPort));
        }

        httpGet = new GetMethod(url);
        httpGet.setParams(urlParams);
        client.executeMethod(httpGet);

        if (httpGet.getStatusCode() == HttpStatus.SC_OK) {
          Header contentType = httpGet.getResponseHeader(HEADER_CONTENT_TYPE);
          String[] contentTypesReturned = contentType.getValue().split(";");
          if (!isValidContentType(contentTypesReturned[0])) {
            contentTypesReturned = contentType.getValue().split(" ");
            if (!isValidContentType(contentTypesReturned[0])) {
              throw new ServletException("Status: 415 Unsupported media type");
            }
          }

          // Sets response contentType
          response.setContentType(getResponseContentType(contentTypesReturned));

          String responseBody = httpGet.getResponseBodyAsString().trim();

          PrintWriter out = response.getWriter();
          out.print(responseBody);

          out.flush();
          out.close();

        } else {
          returnExceptionMessage(
              response, "Unexpected failure: " + httpGet.getStatusLine().toString());
        }

        httpGet.releaseConnection();

      } else {
        // throw new ServletException("only HTTP(S) protocol supported");
        returnExceptionMessage(response, "only HTTP(S) protocol supported");
      }
    } catch (Exception e) {
      e.printStackTrace();
      // throw new ServletException("Some unexpected error occurred. Error text was: " +
      // e.getMessage());
      returnExceptionMessage(
          response, "Some unexpected error occurred. Error text was: " + e.getMessage());
    } finally {
      if (httpGet != null) httpGet.releaseConnection();
    }
  }
Beispiel #27
0
  /** Run method of the thread */
  public void run() {

    queue = manager.workQueue;
    while (manager.hasWorkLeft()) {

      working = false;
      // code to make the worker pause, if the pause button has been presed

      // if the stop signal has been given stop the thread
      if (stop) {
        return;
      }

      // this pasuses the thread
      synchronized (this) {
        while (pleaseWait) {
          try {
            wait();
          } catch (InterruptedException e) {
            return;
          } catch (Exception e) {
            e.printStackTrace();
          }
        }
      }

      GetMethod httpget = null;
      HeadMethod httphead = null;

      try {

        work = (WorkUnit) queue.take();
        working = true;
        url = work.getWork();
        int code = 0;

        String responce = "";
        String rawResponce = "";

        // if the work is a head request
        if (work.getMethod().equalsIgnoreCase("HEAD")) {
          if (Config.debug) {
            System.out.println("DEBUG Worker[" + threadId + "]: HEAD " + url.toString());
          }

          httphead = new HeadMethod(url.toString());

          // set the custom HTTP headers
          Vector HTTPheaders = manager.getHTTPHeaders();
          for (int a = 0; a < HTTPheaders.size(); a++) {
            HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
            /*
             * Host header has to be set in a different way!
             */
            if (httpHeader.getHeader().startsWith("Host")) {
              httphead.getParams().setVirtualHost(httpHeader.getValue());
            } else {
              httphead.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
            }
          }
          httphead.setFollowRedirects(Config.followRedirects);

          /*
           * this code is used to limit the number of request/sec
           */
          if (manager.isLimitRequests()) {
            while (manager.getTotalDone()
                    / ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
                > manager.getLimitRequestsTo()) {
              Thread.sleep(100);
            }
          }
          /*
           * Send the head request
           */
          code = httpclient.executeMethod(httphead);
          if (Config.debug) {
            System.out.println("DEBUG Worker[" + threadId + "]: " + code + " " + url.toString());
          }
          httphead.releaseConnection();

        }
        // if we are doing a get request
        else if (work.getMethod().equalsIgnoreCase("GET")) {
          // make the request;
          if (Config.debug) {
            System.out.println("DEBUG Worker[" + threadId + "]: GET " + url.toString());
          }
          httpget = new GetMethod(url.toString());

          // set the custom HTTP headers
          Vector HTTPheaders = manager.getHTTPHeaders();
          for (int a = 0; a < HTTPheaders.size(); a++) {

            HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
            /*
             * Host header has to be set in a different way!
             */
            if (httpHeader.getHeader().startsWith("Host")) {
              httpget.getParams().setVirtualHost(httpHeader.getValue());
            } else {
              httpget.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
            }
          }
          httpget.setFollowRedirects(Config.followRedirects);

          /*
           * this code is used to limit the number of request/sec
           */
          if (manager.isLimitRequests()) {
            while (manager.getTotalDone()
                    / ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
                > manager.getLimitRequestsTo()) {
              Thread.sleep(100);
            }
          }

          code = httpclient.executeMethod(httpget);

          if (Config.debug) {
            System.out.println("DEBUG Worker[" + threadId + "]: " + code + " " + url.toString());
          }

          // set up the input stream
          BufferedReader input =
              new BufferedReader(new InputStreamReader(httpget.getResponseBodyAsStream()));

          // save the headers into a string, used in viewing raw responce
          String rawHeader;
          rawHeader = httpget.getStatusLine() + "\r\n";
          Header headers[] = httpget.getResponseHeaders();

          StringBuffer buf = new StringBuffer();
          for (int a = 0; a < headers.length; a++) {
            buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
          }

          rawHeader = rawHeader + buf.toString();

          buf = new StringBuffer();
          // read in the responce body
          String line;
          while ((line = input.readLine()) != null) {
            buf.append("\r\n" + line);
          }
          responce = buf.toString();
          input.close();

          rawResponce = rawHeader + responce;
          // clean the responce

          // parse the html of what we have found

          if (Config.parseHTML && !work.getBaseCaseObj().isUseRegexInstead()) {
            Header contentType = httpget.getResponseHeader("Content-Type");

            if (contentType != null) {
              if (contentType.getValue().startsWith("text")) {
                manager.addHTMLToParseQueue(new HTMLparseWorkUnit(responce, work));
              }
            }
          }

          responce = FilterResponce.CleanResponce(responce, work);

          Thread.sleep(10);
          httpget.releaseConnection();
        } else {
          // There is no need to deal with requests other than HEAD or GET
        }

        // if we need to check the against the base case
        if (work.getMethod().equalsIgnoreCase("GET")
            && work.getBaseCaseObj().useContentAnalysisMode()) {
          if (code == 200) {
            if (Config.debug) {
              System.out.println(
                  "DEBUG Worker[" + threadId + "]: Base Case Check " + url.toString());
            }

            // TODO move this option to the Adv options
            // if the responce does not match the base case
            Pattern regexFindFile = Pattern.compile(".*file not found.*", Pattern.CASE_INSENSITIVE);

            Matcher m = regexFindFile.matcher(responce);

            // need to clean the base case of the item we are looking for
            String basecase =
                FilterResponce.removeItemCheckedFor(
                    work.getBaseCaseObj().getBaseCase(), work.getItemToCheck());

            if (m.find()) {
              // do nothing as we have a 404
            } else if (!responce.equalsIgnoreCase(basecase)) {
              if (work.isDir()) {
                if (Config.debug) {
                  System.out.println(
                      "DEBUG Worker[" + threadId + "]: Found Dir (base case)" + url.toString());
                }
                // we found a dir
                manager.foundDir(url, code, responce, basecase, rawResponce, work.getBaseCaseObj());
              } else {
                // found a file
                if (Config.debug) {
                  System.out.println(
                      "DEBUG Worker[" + threadId + "]: Found File (base case)" + url.toString());
                }
                manager.foundFile(
                    url,
                    code,
                    responce,
                    work.getBaseCaseObj().getBaseCase(),
                    rawResponce,
                    work.getBaseCaseObj());
              }
            }
          } else if (code == 404 || code == 400) {
            // again do nothing as it is not there
          } else {
            if (work.isDir()) {
              if (Config.debug) {
                System.out.println(
                    "DEBUG Worker[" + threadId + "]: Found Dir (base case)" + url.toString());
              }
              // we found a dir
              manager.foundDir(
                  url,
                  code,
                  responce,
                  work.getBaseCaseObj().getBaseCase(),
                  rawResponce,
                  work.getBaseCaseObj());
            } else {
              // found a file
              if (Config.debug) {
                System.out.println(
                    "DEBUG Worker[" + threadId + "]: Found File (base case)" + url.toString());
              }
              manager.foundFile(
                  url,
                  code,
                  responce,
                  work.getBaseCaseObj().getBaseCase(),
                  rawResponce,
                  work.getBaseCaseObj());
            }
            // manager.foundError(url, "Base Case Mode Error - Responce code came back as " + code +
            // " it should have been 200");
            // manager.workDone();
          }
        }
        /*
         * use the custom regex check instead
         */
        else if (work.getBaseCaseObj().isUseRegexInstead()) {
          Pattern regexFindFile = Pattern.compile(work.getBaseCaseObj().getRegex());

          Matcher m = regexFindFile.matcher(rawResponce);
          /*
          System.out.println("======Trying to find======");
          System.out.println(work.getBaseCaseObj().getRegex());
          System.out.println("======In======");
          System.out.println(responce);
          System.out.println("======/In======");
           */
          if (m.find()) {
            // do nothing as we have a 404
            if (Config.debug) {

              System.out.println(
                  "DEBUG Worker[" + threadId + "]: Regex matched so it's a 404, " + url.toString());
            }

          } else {
            if (Config.parseHTML) {
              Header contentType = httpget.getResponseHeader("Content-Type");

              if (contentType != null) {
                if (contentType.getValue().startsWith("text")) {
                  manager.addHTMLToParseQueue(new HTMLparseWorkUnit(rawResponce, work));
                }
              }
            }
            if (work.isDir()) {
              if (Config.debug) {
                System.out.println(
                    "DEBUG Worker[" + threadId + "]: Found Dir (regex) " + url.toString());
              }
              // we found a dir
              manager.foundDir(
                  url,
                  code,
                  responce,
                  work.getBaseCaseObj().getBaseCase(),
                  rawResponce,
                  work.getBaseCaseObj());
            } else {
              // found a file
              if (Config.debug) {
                System.out.println(
                    "DEBUG Worker[" + threadId + "]: Found File (regex) " + url.toString());
              }
              manager.foundFile(
                  url,
                  code,
                  responce,
                  work.getBaseCaseObj().getBaseCase(),
                  rawResponce,
                  work.getBaseCaseObj());
            }
            // manager.foundError(url, "Base Case Mode Error - Responce code came back as " + code +
            // " it should have been 200");
            // manager.workDone();
          }

        }
        // just check the responce code
        else {
          // if is not the fail code, a 404 or a 400 then we have a possible
          if (code != work.getBaseCaseObj().getFailCode()
              && code != 404
              && code != 0
              && code != 400) {
            if (work.getMethod().equalsIgnoreCase("HEAD")) {
              if (Config.debug) {
                System.out.println(
                    "DEBUG Worker[" + threadId + "]: Getting responce via GET " + url.toString());
              }
              rawResponce = "";

              httpget = new GetMethod(url.toString());
              Vector HTTPheaders = manager.getHTTPHeaders();
              for (int a = 0; a < HTTPheaders.size(); a++) {
                HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
                httpget.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
              }
              httpget.setFollowRedirects(Config.followRedirects);

              /*
               * this code is used to limit the number of request/sec
               */
              if (manager.isLimitRequests()) {
                while (manager.getTotalDone()
                        / ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
                    > manager.getLimitRequestsTo()) {
                  Thread.sleep(100);
                }
              }

              int newCode = httpclient.executeMethod(httpget);

              // in some cases the second get can return a different result, than the first head
              // request!
              if (newCode != code) {
                manager.foundError(
                    url,
                    "Return code for first HEAD, is different to the second GET: "
                        + code
                        + " - "
                        + newCode);
              }

              rawResponce = "";
              // build a string version of the headers
              rawResponce = httpget.getStatusLine() + "\r\n";
              Header headers[] = httpget.getResponseHeaders();

              StringBuffer buf = new StringBuffer();
              for (int a = 0; a < headers.length; a++) {
                buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
              }

              buf.append("\r\n");

              rawResponce = rawResponce + buf.toString();

              if (httpget.getResponseContentLength() > 0) {

                // get the http body
                BufferedReader input =
                    new BufferedReader(new InputStreamReader(httpget.getResponseBodyAsStream()));

                String line;

                String tempResponce = "";

                buf = new StringBuffer();
                while ((line = input.readLine()) != null) {
                  buf.append("\r\n" + line);
                }
                tempResponce = buf.toString();
                input.close();

                rawResponce = rawResponce + tempResponce;

                Header contentType = httpget.getResponseHeader("Content-Type");

                if (Config.parseHTML) {
                  contentType = httpget.getResponseHeader("Content-Type");

                  if (contentType != null) {
                    if (contentType.getValue().startsWith("text")) {
                      manager.addHTMLToParseQueue(new HTMLparseWorkUnit(tempResponce, work));
                    }
                  }
                }
              }

              httpget.releaseConnection();
            }

            if (work.isDir()) {
              manager.foundDir(url, code, rawResponce, work.getBaseCaseObj());
            } else {
              manager.foundFile(url, code, rawResponce, work.getBaseCaseObj());
            }
          }
        }

        manager.workDone();
        Thread.sleep(20);

      } catch (NoHttpResponseException e) {
        manager.foundError(url, "NoHttpResponseException " + e.getMessage());
        manager.workDone();
      } catch (ConnectTimeoutException e) {
        manager.foundError(url, "ConnectTimeoutException " + e.getMessage());
        manager.workDone();
      } catch (URIException e) {
        manager.foundError(url, "URIException " + e.getMessage());
        manager.workDone();
      } catch (IOException e) {

        manager.foundError(url, "IOException " + e.getMessage());
        manager.workDone();
      } catch (InterruptedException e) {
        // manager.foundError(url, "InterruptedException " + e.getMessage());
        manager.workDone();
        return;
      } catch (IllegalArgumentException e) {

        e.printStackTrace();
        manager.foundError(url, "IllegalArgumentException " + e.getMessage());
        manager.workDone();
      } finally {
        if (httpget != null) {
          httpget.releaseConnection();
        }

        if (httphead != null) {
          httphead.releaseConnection();
        }
      }
    }
  }
Beispiel #28
0
  /**
   * 通过httpclient方法获得网页内容
   *
   * @param url 目标的URL
   * @return 页面内容字符串
   * @throws IOException
   */
  public static String getContentByHttpClient(String url, String encode, int timeout)
      throws IOException {

    HttpClient client = new HttpClient();

    GetMethod method = new GetMethod(url);

    // 设置失败后默认的重试次数为3
    method
        .getParams()
        .setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));

    // 设置失败延时为5000ms,即5秒
    method.getParams().setIntParameter(HttpMethodParams.SO_TIMEOUT, timeout);

    ByteArrayOutputStream outstream = null;

    BufferedInputStream bufferedInputStream = null;

    try {

      int statusCode = client.executeMethod(method);

      if (statusCode != HttpStatus.SC_OK) {

        System.out.println("方法失败: " + method.getStatusLine());
      }
      // 缓冲读取的内容4k,提高性能
      bufferedInputStream = new BufferedInputStream(method.getResponseBodyAsStream());

      outstream = new ByteArrayOutputStream();
      byte[] buffer = new byte[4096];
      int len;
      while ((len = bufferedInputStream.read(buffer)) > 0) {

        outstream.write(buffer, 0, len);
      }

      outstream.flush();

      byte[] responseBody = outstream.toByteArray();

      return new String(responseBody, encode);

    } catch (HttpException e) {

      System.out.println("致命协议错误:" + e.getMessage());
      throw e;
    } catch (IOException e) {

      System.out.println("致命传输错误:" + e.getMessage());
      throw e;
    } finally {
      if (bufferedInputStream != null) {

        bufferedInputStream.close();
      }
      if (outstream != null) {

        outstream.close();
      }
      // 注意,连接在完成后必须释放
      method.releaseConnection();
    }
  }