// implements the filter-method which gives you access to important Objects // like NutchDocument public NutchDocument filter( NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) { LOG.info("-------->>>>> WE ARE IN THE INDExer-------------------"); String containsSem = "false"; containsSem = parse.getData().getMeta(WdcParser.META_CONTAINS_SEM); // we don't have to add the triples in a separate field as they are // already in the content field // String triples = ""; // triples = parse.getText(); // doc.add("triples", triples); // // check if the father contains sem data // boolean semFather = false; // try { // semFather = // Boolean.parseBoolean(datum.getMetaData().get(WdcParser.META_CONTAINS_SEM_FATHER).toString()); // // } catch (Exception e) { // LOG.error("CANNOT PROCESS THE FATHER SEM FIELD" + e.getMessage()); // } // adds the new field to the document doc.add("containsSem", containsSem); return doc; }
@Override public void write(NutchDocument doc) throws IOException { final SolrInputDocument inputDoc = new SolrInputDocument(); for (final Entry<String, List<String>> e : doc) { for (final String val : e.getValue()) { Object val2 = val; if (e.getKey().equals("content") || e.getKey().equals("title")) { val2 = SolrUtils.stripNonCharCodepoints(val); } inputDoc.addField(solrMapping.mapKey(e.getKey()), val2); String sCopy = solrMapping.mapCopyKey(e.getKey()); if (sCopy != e.getKey()) { inputDoc.addField(sCopy, val2); } } } inputDoc.setDocumentBoost(doc.getScore()); inputDocs.add(inputDoc); if (inputDocs.size() >= commitSize) { try { LOG.info("Adding " + Integer.toString(inputDocs.size()) + " documents"); solr.add(inputDocs); } catch (final SolrServerException e) { throw new IOException(e); } inputDocs.clear(); } }
public NutchDocument filter( NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { if (this.addStaticFields == true) { for (Entry<String, String[]> entry : this.fields.entrySet()) { doc.add(entry.getKey(), entry.getValue()); } } return doc; }
/** * The {@link RelTagIndexingFilter} filter object. * * @param doc The {@link NutchDocument} object * @param url URL to be filtered for rel-tag's * @param page {@link WebPage} object relative to the URL * @return filtered NutchDocument */ @Override public NutchDocument filter(NutchDocument doc, String url, WebPage page) throws IndexingException { // Check if some Rel-Tags found, possibly put there by RelTagParser ByteBuffer bb = page.getFromMetadata(new Utf8(RelTagParser.REL_TAG)); if (bb != null) { String[] tags = Bytes.toString(bb).split("\t"); for (int i = 0; i < tags.length; i++) { doc.add("tag", tags[i]); } } return doc; }
public NutchDocument filter( NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { Text reprUrl = (Text) datum.getMetaData().get(Nutch.WRITABLE_REPR_URL_KEY); String reprUrlString = reprUrl != null ? reprUrl.toString() : null; String urlString = url.toString(); String host = null; try { URL u; if (reprUrlString != null) { u = new URL(reprUrlString); } else { u = new URL(urlString); } host = u.getHost(); } catch (MalformedURLException e) { throw new IndexingException(e); } if (host != null) { doc.add("host", host); } doc.add("url", reprUrlString == null ? urlString : reprUrlString); // content String content = parse.getText(); if (MAX_CONTENT_LENGTH > -1 && content.length() > MAX_CONTENT_LENGTH) { content = content.substring(0, MAX_CONTENT_LENGTH); } doc.add("content", content); // title String title = parse.getData().getTitle(); if (title.length() > MAX_TITLE_LENGTH) { // truncate title if needed title = title.substring(0, MAX_TITLE_LENGTH); } if (title.length() > 0) { // NUTCH-1004 Do not index empty values for title field doc.add("title", title); } // add cached content/summary display policy, if available String caching = parse.getData().getMeta(Nutch.CACHING_FORBIDDEN_KEY); if (caching != null && !caching.equals(Nutch.CACHING_FORBIDDEN_NONE)) { doc.add("cache", caching); } // add timestamp when fetched, for deduplication doc.add("tstamp", new Date(datum.getFetchTime())); return doc; }