Ejemplo n.º 1
0
  public String symantic_query(String query) {
    String st = query.split("\\[")[1].split("\\]")[0];

    String res = "";
    try {
      res =
          helpers.getExpand(
              "http://www.kbresearch.nl/tripple.cgi?query=" + helpers.urlEncode(st), log);
    } catch (MalformedURLException e) {
      res = (e.getMessage());
    }
    return (res);
  }
  /**
   * Запрос с указанием параметров.
   *
   * @see MoneyDownloader#login
   * @see MoneyDownloader#password
   * @see MoneyDownloader#dateStart
   * @see MoneyDownloader#dateEnd
   */
  public final void run() {
    File file = new File(directory + "/Results.tsv");
    //        File test = new File(directory + "/tmp/result.tsv");

    try {
      if (!file.exists()) {
        file.createNewFile();
      }
    } catch (IOException e) {
      e.printStackTrace();
    }

    String request = "https://rt.miran.ru?user="******"&pass="******"http://rt.miran.ru/rt/Search/Results.tsv?user="******"&pass="******"&Format=%27__Created__%2FTITLE%3A%D0%94%D0%B0%D1%82%D0%B0%27%2C%0A%27%20%20%20%3Cb%3E%3Ca%20href%3D%22%2Frt%2FTicket%2FDisplay.html%3Fid%3D__id__%22%3E__id__%3C%2Fa%3E%3C%2Fb%3E%2FTITLE%3A%23%27%2C%0A%27__Subject__%2FTITLE%3A%D0%A2%D0%B5%D0%BC%D0%B0%27%2C%0A%27__QueueName__%2FTITLE%3A%D0%9E%D1%87%D0%B5%D1%80%D0%B5%D0%B4%D1%8C%27%2C%0A%27__CustomField.%7Binteraction%20type%7D__%2FTITLE%3A%D0%A2%D0%B8%D0%BF%27%2C%0A%27__CustomField.%7Bbussines%7D__%2FTITLE%3A%D0%91%D0%B8%D0%B7%D0%BD%D0%B5%D1%81%27%2C%0A%27__CustomField.%7Bdirection%7D__%2FTITLE%3A%D0%9D%D0%B0%D0%BF%D1%80%D0%B0%D0%B2%D0%BB%D0%B5%D0%BD%D0%B8%D0%B5%27%2C%0A%27__CustomField.%7Bservice%7D__%2FTITLE%3A%D0%A3%D1%81%D0%BB%D1%83%D0%B3%D0%B0%27%2C%0A%27__CustomField.%7BSD%20Type%7D__%2FTITLE%3A%D0%9A%D0%B0%D1%82%D0%B5%D0%B3%D0%BE%D1%80%D0%B8%D1%8F%27%2C%0A%27__CustomField.%7BSD%20Detail%7D__%2FTITLE%3A%D0%91%D0%BE%D0%BD%D1%83%D1%81%27%2C%0A%27__CustomField.%7BQA%7D__%2FTITLE%3A%D0%92%D1%8B%D0%BF%D0%BE%D0%BB%D0%BD%D0%B8%D0%BB%27&Order=DESC%7CASC%7CASC%7CASC&OrderBy=Created%7C%7C%7C&Page=1&Query=Created%20%3E%20%27"
            + dateStart
            + "%27%20AND%20Created%20%3C%20%27"
            + dateEnd
            + "%27%20AND%20Status%20%3D%20%27closed%27%20AND%20Queue%20!%3D%20%27manager%27%20AND%20Queue%20!%3D%20%27sales%27%20AND%20%27CF.%7BQA%7D%27%20%3D%20%27__CurrentUser__%27&RowsPerPage=0&SavedChartSearchId=new&SavedSearchId=";

    System.out.println(tickets);

    ListCookieHandler cookieHandler = new ListCookieHandler();
    CookieHandler.setDefault(cookieHandler);
    try {
      URL url = new URL(request);

      HttpURLConnection connection = (HttpURLConnection) url.openConnection();
      connection.getResponseCode();

      GetMethod method = new GetMethod(tickets);
      method.addRequestHeader("Cookie", cookieHandler.getCookiez());
      method.addRequestHeader("Referer", "http://rt.miran.ru/");
      HttpClient httpClient = new HttpClient();

      httpClient.executeMethod(method);

      InputStream in = method.getResponseBodyAsStream();
      System.out.println(in.available());

    } catch (MalformedURLException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
Ejemplo n.º 3
0
  public void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    response.setContentType(XML_RESPONSE_HEADER); // Talkback happens in XML form.
    response.setCharacterEncoding("UTF-8"); // Unicode++
    request.setCharacterEncoding("UTF-8");

    PrintWriter out = null; // The talkback buffer.

    // handle startrecord
    Integer startRecord = 0;

    if (!(request.getParameter("startRecord") == null)) {
      try {
        startRecord = Integer.parseInt(request.getParameter("startRecord")) - 1;
      } catch (NumberFormatException e) {
        startRecord = 0;
      }
    }

    // maximumrecords
    Integer maximumRecords = Integer.parseInt(this.config.getProperty("default_maximumRecords"));
    if (!(request.getParameter("maximumRecords") == null)) {
      maximumRecords = Integer.parseInt(request.getParameter("maximumRecords"));
    }

    // operation
    String operation = request.getParameter("operation");

    // x_collection
    String x_collection = request.getParameter("x-collection");
    if (x_collection == null) x_collection = this.config.getProperty("default_x_collection");
    if (x_collection == null) operation = null;

    // sortkeys
    String sortKeys = request.getParameter("sortKeys");

    // sortorder
    String sortOrder = request.getParameter("sortOrder");

    // recordschema
    String recordSchema = request.getParameter("recordSchema");
    if (recordSchema == null) recordSchema = "dc";

    if (recordSchema.equalsIgnoreCase("dcx")) {
      recordSchema = "dcx";
    }

    if (recordSchema.equalsIgnoreCase("solr")) {
      recordSchema = "solr";
    }

    // query request
    String query = request.getParameter("query");
    String q = request.getParameter("q");

    // who is requestor ?
    String remote_ip = request.getHeader("X-FORWARDED-FOR");

    if (remote_ip == null) {
      remote_ip = request.getRemoteAddr().trim();
    } else {
      remote_ip = request.getHeader("X-FORWARDED-FOR");
    }

    // handle debug
    Boolean debug = Boolean.parseBoolean(request.getParameter("debug"));
    if (!debug) {
      out = new PrintWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), true);
    }

    // handle query
    if ((query == null) && (q != null)) {
      query = q;
    } else {
      if ((query != null) && (q == null)) {
        q = query;
      } else {
        operation = null;
      }
    }

    // handle operation
    if (operation == null) {
      if (query != null) {
        operation = "searchRetrieve";
      } else {
        operation = "explain";
      }
    }

    //  searchRetrieve
    if (operation.equalsIgnoreCase("searchRetrieve")) {
      if (query == null) {
        operation = "explain";
        log.debug(operation + ":" + query);
      }
    }

    // start talking back.
    String[] sq = {""};
    String solrquery = "";

    // facet

    String facet = null;
    List<FacetField> fct = null;

    if (request.getParameter("facet") != null) {
      facet = request.getParameter("facet");
      log.debug("facet : " + facet);
    }

    if (operation == null) {
      operation = "searchretrieve";
    } else { // explain response
      if (operation.equalsIgnoreCase("explain")) {
        log.debug("operation = explain");
        out.write("<srw:explainResponse xmlns:srw=\"http://www.loc.gov/zing/srw/\">");
        out.write("</srw:explainResponse>");
      } else { // DEBUG routine
        operation = "searchretrieve";

        String triplequery = null;

        if (query.matches(".*?\\[.+?\\].*?")) { // New symantic syntax
          triplequery = symantic_query(query);
          query = query.split("\\[")[0] + " " + triplequery;
          log.fatal(triplequery);

          solrquery = CQLtoLucene.translate(query, log, config);
        } else {
          solrquery = CQLtoLucene.translate(query, log, config);
        }
        log.debug(solrquery);

        if (debug == true) {
          response.setContentType(HTML_RESPONSE_HEADER);
          out = new PrintWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), true);
          out.write("<html><body>\n\n");
          out.write("'" + remote_ip + "'<br>\n");
          out.write("<form action='http://www.kbresearch.nl/kbSRU'>");
          out.write("<input type=text name=q value='" + query + "' size=120>");
          out.write("<input type=hidden name=debug value=True>");
          out.write("<input type=submit>");
          out.write("<table border=1><tr><td>");
          out.write("q</td><td>" + query + "</td></tr><tr>");
          out.write("<td>query out</td><td>" + URLDecoder.decode(solrquery) + "</td></tr>");
          out.write(
              "<tr><td>SOLR_URL</td><td> <a href='"
                  + this.config.getProperty(
                      "collection." + x_collection.toLowerCase() + ".solr_baseurl")
                  + "/?q="
                  + solrquery
                  + "'>"
                  + this.config.getProperty(
                      "collection." + x_collection.toLowerCase() + ".solr_baseurl")
                  + "/select/?q="
                  + solrquery
                  + "</a><br>"
                  + this.config.getProperty("solr_url")
                  + solrquery
                  + "</td></tr>");
          out.write(
              "<b>SOLR_QUERY</b> : <BR> <iframe width=900 height=400 src='"
                  + this.config.getProperty(
                      "collection." + x_collection.toLowerCase() + ".solr_baseurl")
                  + "/../?q="
                  + solrquery
                  + "'></iframe><BR>");
          out.write(
              "<b>SRU_QUERY</b> : <BR> <a href="
                  + this.config.getProperty("baseurl")
                  + "?q="
                  + query
                  + "'>"
                  + this.config.getProperty("baseurl")
                  + "?q="
                  + query
                  + "</a><br><iframe width=901 height=400 src='http://www.kbresearch.nl/kbSRU/?q="
                  + query
                  + "'></iframe><BR>");
          out.write(
              "<br><b>JSRU_QUERY</b> : <BR><a href='http://jsru.kb.nl/sru/?query="
                  + query
                  + "&x-collection="
                  + x_collection
                  + "'>http://jsru.kb.nl/sru/?query="
                  + query
                  + "&x-collection=GGC</a><br><iframe width=900 height=400 src='http://jsru.kb.nl/sru/?query="
                  + query
                  + "&x-collection=GGC'></iframe>");

        } else { // XML SearchRetrieve response
          String url =
              this.config.getProperty("collection." + x_collection.toLowerCase() + ".solr_baseurl");
          String buffer = "";
          CommonsHttpSolrServer server = null;
          server = new CommonsHttpSolrServer(url);
          log.fatal("URSING " + url);
          server.setParser(new XMLResponseParser());
          int numfound = 0;
          try {
            SolrQuery do_query = new SolrQuery();
            do_query.setQuery(solrquery);
            do_query.setRows(maximumRecords);
            do_query.setStart(startRecord);

            if ((sortKeys != null) && (sortKeys.length() > 1)) {
              if (sortOrder != null) {
                if (sortOrder.equals("asc")) {
                  do_query.setSortField(sortKeys, SolrQuery.ORDER.asc);
                }
                if (sortOrder.equals("desc")) {
                  do_query.setSortField(sortKeys, SolrQuery.ORDER.desc);
                }
              } else {
                for (String str : sortKeys.trim().split(",")) {
                  str = str.trim();
                  if (str.length() > 1) {
                    if (str.equals("date")) {
                      do_query.setSortField("date_date", SolrQuery.ORDER.desc);
                      log.debug("SORTORDERDEBUG | DATE! " + str + " | ");
                      break;
                    } else {
                      do_query.setSortField(str + "_str", SolrQuery.ORDER.asc);
                      log.debug("SORTORDERDEBUG | " + str + " | ");
                      break;
                    }
                  }
                }
              }
            }

            if (facet != null) {
              if (facet.indexOf(",") > 1) {
                for (String str : facet.split(",")) {
                  if (str.indexOf("date") > 1) {
                    do_query.addFacetField(str);
                  } else {
                    do_query.addFacetField(str);
                  }
                  // do_query.setParam("facet.method", "enum");
                }
                // q.setFacetSort(false);
              } else {
                do_query.addFacetField(facet);
              }
              do_query.setFacet(true);
              do_query.setFacetMinCount(1);
              do_query.setFacetLimit(-1);
            }

            log.fatal(solrquery);

            QueryResponse rsp = null;
            boolean do_err = false;
            boolean do_sugg = false;
            SolrDocumentList sdl = null;
            String diag = "";
            StringBuffer suggest = new StringBuffer("");

            String content = "1";

            SolrQuery spellq = do_query;
            try {
              rsp = server.query(do_query);
            } catch (SolrServerException e) {
              String header = this.SRW_HEADER.replaceAll("\\$numberOfRecords", "0");
              out.write(header);
              diag = this.SRW_DIAG.replaceAll("\\$error", e.getMessage());
              do_err = true;
              rsp = null;
            }

            log.fatal("query done..");
            if (!(do_err)) { // XML dc response

              SolrDocumentList docs = rsp.getResults();
              numfound = (int) docs.getNumFound();
              int count = startRecord;
              String header =
                  this.SRW_HEADER.replaceAll("\\$numberOfRecords", Integer.toString(numfound));
              out.write(header);
              out.write("<srw:records>");

              Iterator<SolrDocument> iter = rsp.getResults().iterator();

              while (iter.hasNext()) {
                count += 1;
                if (recordSchema.equalsIgnoreCase("dc")) {
                  SolrDocument resultDoc = iter.next();
                  content = (String) resultDoc.getFieldValue("id");
                  out.write("<srw:record>");
                  out.write("<srw:recordPacking>xml</srw:recordPacking>");
                  out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>");
                  out.write(
                      "<srw:recordData xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/loc.terms/relators/OTH\" xmlns:facets=\"info:srw/extension/4/facets\" >");
                  StringBuffer result = new StringBuffer("");

                  construct_lucene_dc(result, resultDoc);

                  out.write(result.toString());
                  out.write("</srw:recordData>");
                  out.write(
                      "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
                  out.write("</srw:record>");
                }

                if (recordSchema.equalsIgnoreCase("solr")) {
                  SolrDocument resultDoc = iter.next();
                  content = (String) resultDoc.getFieldValue("id");
                  out.write("<srw:record>");
                  out.write("<srw:recordPacking>xml</srw:recordPacking>");
                  out.write("<srw:recordSchema>info:srw/schema/1/solr</srw:recordSchema>");
                  out.write("<srw:recordData xmlns:expand=\"http://www.kbresearch.nl/expand\">");
                  StringBuffer result = new StringBuffer("");
                  construct_lucene_solr(result, resultDoc);
                  out.write(result.toString());

                  out.write("</srw:recordData>");
                  out.write(
                      "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
                  out.write("</srw:record>");
                }

                if (recordSchema.equalsIgnoreCase("dcx")) { // XML dcx response
                  out.write("<srw:record>");
                  out.write("<srw:recordPacking>xml</srw:recordPacking>");
                  out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>");
                  out.write(
                      "<srw:recordData><srw_dc:dc xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/marc.relators/\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >");
                  SolrDocument resultDoc = iter.next();
                  content = (String) resultDoc.getFieldValue("id");

                  String dcx_data =
                      helpers.getOAIdcx(
                          "http://services.kb.nl/mdo/oai?verb=GetRecord&identifier=" + content,
                          log);
                  if (x_collection.equalsIgnoreCase("ggc-thes")) {
                    dcx_data =
                        helpers.getOAIdcx(
                            "http://serviceso.kb.nl/mdo/oai?verb=GetRecord&identifier=" + content,
                            log);
                  }

                  if (!(dcx_data.length() == 0)) {
                    out.write(dcx_data);
                  } else {
                    // Should not do this!!

                    out.write("<srw:record>");
                    out.write("<srw:recordPacking>xml</srw:recordPacking>");
                    out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>");
                    out.write(
                        "<srw:recordData xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/loc.terms/relators/OTH\" >");
                    StringBuffer result = new StringBuffer("");

                    construct_lucene_dc(result, resultDoc);

                    out.write(result.toString());
                    out.write("</srw:recordData>");
                    out.write(
                        "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
                    out.write("</srw:record>");
                  }

                  out.write("</srw_dc:dc>");

                  StringBuffer expand_data;
                  boolean expand = false;

                  if (content.startsWith("GGC-THES:AC:")) {
                    String tmp_content = "";
                    tmp_content = content.replaceFirst("GGC-THES:AC:", "");
                    log.fatal("calling get");
                    expand_data =
                        new StringBuffer(
                            helpers.getExpand(
                                "http://www.kbresearch.nl/general/lod_new/get/"
                                    + tmp_content
                                    + "?format=rdf",
                                log));
                    log.fatal("get finini");

                    if (expand_data.toString().length() > 4) {

                      out.write(
                          "<srw_dc:expand xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >");
                      out.write(expand_data.toString());
                      expand = true;
                    }
                  } else {
                    expand_data =
                        new StringBuffer(
                            helpers.getExpand(
                                "http://www.kbresearch.nl/ANP.cgi?q=" + content, log));
                    if (expand_data.toString().length() > 0) {
                      if (!expand) {
                        out.write(
                            "<srw_dc:expand xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >");
                        expand = true;
                      }
                      out.write(expand_data.toString());
                    }
                  }
                  if (expand) {
                    out.write("</srw_dc:expand>");
                  }

                  out.write("</srw:recordData>");
                  out.write(
                      "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
                  out.write("</srw:record>");
                }
              }
            }

            if ((do_err) || (numfound == 0)) {
              log.fatal("I haz suggestions");

              try {
                spellq.setParam("spellcheck", true);
                spellq.setQueryType("/spell");
                server = new CommonsHttpSolrServer(url);
                rsp = server.query(spellq);
                sdl = rsp.getResults();
                SpellCheckResponse spell;
                spell = rsp.getSpellCheckResponse();
                List<SpellCheckResponse.Suggestion> suggestions = spell.getSuggestions();
                if (suggestions.isEmpty() == false) {
                  suggest.append("<srw:extraResponseData>");
                  suggest.append("<suggestions>");

                  for (SpellCheckResponse.Suggestion sugg : suggestions) {
                    suggest.append("<suggestionfor>" + sugg.getToken() + "</suggestionfor>");
                    for (String item : sugg.getSuggestions()) {
                      suggest.append("<suggestion>" + item + "</suggestion>");
                    }
                    suggest.append("</suggestions>");
                    suggest.append("</srw:extraResponseData>");
                  }
                  do_sugg = true;
                }
              } catch (Exception e) {
                rsp = null;
                // log.fatal(e.toString());
              }
              ;
            }
            ;

            if (!do_err) {
              if (facet != null) {

                try {
                  fct = rsp.getFacetFields();
                  out.write("<srw:facets>");

                  for (String str : facet.split(",")) {
                    out.write("<srw:facet>");
                    out.write("<srw:facetType>");
                    out.write(str);
                    out.write("</srw:facetType>");

                    for (FacetField f : fct) {
                      log.debug(f.getName());
                      // if (f.getName().equals(str+"_str") || (f.getName().equals(str+"_date")) ) {
                      List<FacetField.Count> facetEnties = f.getValues();
                      for (FacetField.Count fcount : facetEnties) {
                        out.write("<srw:facetValue>");
                        out.write("<srw:valueString>");
                        out.write(helpers.xmlEncode(fcount.getName()));
                        out.write("</srw:valueString>");
                        out.write("<srw:count>");
                        out.write(Double.toString(fcount.getCount()));
                        out.write("</srw:count>");
                        out.write("</srw:facetValue>");
                        //   }
                      }
                    }
                    out.write("</srw:facet>");
                  }
                  out.write("</srw:facets>");
                  startRecord += 1;
                } catch (Exception e) {
                }

                // log.fatal(e.toString()); }
              }
            } else {
              out.write(diag);
            }
            out.write("</srw:records>"); // SearchRetrieve response footer
            String footer = this.SRW_FOOTER.replaceAll("\\$query", helpers.xmlEncode(query));
            footer = footer.replaceAll("\\$startRecord", (startRecord).toString());
            footer = footer.replaceAll("\\$maximumRecords", maximumRecords.toString());
            footer = footer.replaceAll("\\$recordSchema", recordSchema);
            if (do_sugg) {
              out.write(suggest.toString());
            }
            out.write(footer);
          } catch (MalformedURLException e) {
            out.write(e.getMessage());
          } catch (IOException e) {
            out.write("TO ERR is Human");
          }
        }
      }
    }
    out.close();
  }