@Override
public boolean accept(VirtualFile file) {
try {
TikaConfig tikaConfig = new TikaConfig();
MediaType mimeType =
tikaConfig.getDetector().detect(file.getContent().getStream(), new Metadata());
return mediaTypes.contains(mimeType);
} catch (TikaException | ForbiddenException | ServerException | IOException e) {
return false;
}
}
private void scan(ByteArrayInputStream in, String path, SVNDirEntry dirEntry) {
try {
Metadata metadata = new Metadata();
metadata.set(Metadata.RESOURCE_NAME_KEY, path);
// The following code part is from an proposal of the Authors of
// Tika:
// https://issues.apache.org/jira/browse/TIKA-232
TikaConfig config = TikaConfig.getDefaultConfig(); // without a
// delegate
// parser
Parser parser = new AutoDetectParser(config);
DefaultHandler handler = new BodyContentHandler();
parser.parse(in, handler, metadata);
getDocument().addTokenizedField(FieldNames.CONTENTS, handler.toString());
} catch (Exception e) {
LOGGER.error("We had an exception " + path + " (r" + dirEntry.getRevision() + ")", e);
} finally {
try {
in.close();
} catch (Exception e) {
LOGGER.error("We had an exception " + path + " (r" + dirEntry.getRevision() + ")", e);
}
}
}
@Override
protected ParseContext buildParseContext(
Metadata metadata, String targetMimeType, TransformationOptions options) {
ParseContext context = super.buildParseContext(metadata, targetMimeType, options);
boolean recurse = includeContents;
if (options.getIncludeEmbedded() != null) {
recurse = options.getIncludeEmbedded();
}
if (recurse) {
// Use an auto detect parser to handle the contents
if (tikaConfig == null) {
tikaConfig = TikaConfig.getDefaultConfig();
}
context.set(Parser.class, new AutoDetectParser(tikaConfig));
}
return context;
}
@Override
public void load(
SolrQueryRequest req,
SolrQueryResponse rsp,
ContentStream stream,
UpdateRequestProcessor processor)
throws Exception {
Parser parser = null;
String streamType = req.getParams().get(ExtractingParams.STREAM_TYPE, null);
if (streamType != null) {
// Cache? Parsers are lightweight to construct and thread-safe, so I'm told
MediaType mt = MediaType.parse(streamType.trim().toLowerCase(Locale.ROOT));
parser = new DefaultParser(config.getMediaTypeRegistry()).getParsers().get(mt);
} else {
parser = autoDetectParser;
}
if (parser != null) {
Metadata metadata = new Metadata();
// If you specify the resource name (the filename, roughly) with this parameter,
// then Tika can make use of it in guessing the appropriate MIME type:
String resourceName = req.getParams().get(ExtractingParams.RESOURCE_NAME, null);
if (resourceName != null) {
metadata.add(TikaMetadataKeys.RESOURCE_NAME_KEY, resourceName);
}
// Provide stream's content type as hint for auto detection
if (stream.getContentType() != null) {
metadata.add(HttpHeaders.CONTENT_TYPE, stream.getContentType());
}
InputStream inputStream = null;
try {
inputStream = stream.getStream();
metadata.add(ExtractingMetadataConstants.STREAM_NAME, stream.getName());
metadata.add(ExtractingMetadataConstants.STREAM_SOURCE_INFO, stream.getSourceInfo());
metadata.add(ExtractingMetadataConstants.STREAM_SIZE, String.valueOf(stream.getSize()));
metadata.add(ExtractingMetadataConstants.STREAM_CONTENT_TYPE, stream.getContentType());
// HtmlParser and TXTParser regard Metadata.CONTENT_ENCODING in metadata
String charset = ContentStreamBase.getCharsetFromContentType(stream.getContentType());
if (charset != null) {
metadata.add(HttpHeaders.CONTENT_ENCODING, charset);
}
String xpathExpr = params.get(ExtractingParams.XPATH_EXPRESSION);
boolean extractOnly = params.getBool(ExtractingParams.EXTRACT_ONLY, false);
SolrContentHandler handler =
factory.createSolrContentHandler(metadata, params, req.getSchema());
ContentHandler parsingHandler = handler;
StringWriter writer = null;
BaseMarkupSerializer serializer = null;
if (extractOnly == true) {
String extractFormat = params.get(ExtractingParams.EXTRACT_FORMAT, "xml");
writer = new StringWriter();
if (extractFormat.equals(TEXT_FORMAT)) {
serializer = new TextSerializer();
serializer.setOutputCharStream(writer);
serializer.setOutputFormat(new OutputFormat("Text", "UTF-8", true));
} else {
serializer = new XMLSerializer(writer, new OutputFormat("XML", "UTF-8", true));
}
if (xpathExpr != null) {
Matcher matcher = PARSER.parse(xpathExpr);
serializer
.startDocument(); // The MatchingContentHandler does not invoke startDocument. See
// http://tika.markmail.org/message/kknu3hw7argwiqin
parsingHandler = new MatchingContentHandler(serializer, matcher);
} else {
parsingHandler = serializer;
}
} else if (xpathExpr != null) {
Matcher matcher = PARSER.parse(xpathExpr);
parsingHandler = new MatchingContentHandler(handler, matcher);
} // else leave it as is
try {
// potentially use a wrapper handler for parsing, but we still need the SolrContentHandler
// for getting the document.
ParseContext context = parseContextConfig.create();
context.set(Parser.class, parser);
context.set(HtmlMapper.class, MostlyPassthroughHtmlMapper.INSTANCE);
// Password handling
RegexRulesPasswordProvider epp = new RegexRulesPasswordProvider();
String pwMapFile = params.get(ExtractingParams.PASSWORD_MAP_FILE);
if (pwMapFile != null && pwMapFile.length() > 0) {
InputStream is = req.getCore().getResourceLoader().openResource(pwMapFile);
if (is != null) {
log.debug("Password file supplied: " + pwMapFile);
epp.parse(is);
}
}
context.set(PasswordProvider.class, epp);
String resourcePassword = params.get(ExtractingParams.RESOURCE_PASSWORD);
if (resourcePassword != null) {
epp.setExplicitPassword(resourcePassword);
log.debug("Literal password supplied for file " + resourceName);
}
parser.parse(inputStream, parsingHandler, metadata, context);
} catch (TikaException e) {
if (ignoreTikaException)
log.warn(
new StringBuilder("skip extracting text due to ")
.append(e.getLocalizedMessage())
.append(". metadata=")
.append(metadata.toString())
.toString());
else throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
if (extractOnly == false) {
addDoc(handler);
} else {
// serializer is not null, so we need to call endDoc on it if using xpath
if (xpathExpr != null) {
serializer.endDocument();
}
rsp.add(stream.getName(), writer.toString());
writer.close();
String[] names = metadata.names();
NamedList metadataNL = new NamedList();
for (int i = 0; i < names.length; i++) {
String[] vals = metadata.getValues(names[i]);
metadataNL.add(names[i], vals);
}
rsp.add(stream.getName() + "_metadata", metadataNL);
}
} catch (SAXException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} finally {
IOUtils.closeQuietly(inputStream);
}
} else {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Stream type of "
+ streamType
+ " didn't match any known parsers. Please supply the "
+ ExtractingParams.STREAM_TYPE
+ " parameter.");
}
}
public void parseEmbedded(
InputStream inputStream,
ContentHandler contentHandler,
Metadata metadata,
boolean outputHtml)
throws SAXException, IOException {
String name = metadata.get(Metadata.RESOURCE_NAME_KEY);
if (name == null) {
name = "file" + count++;
}
MediaType contentType = detector.detect(inputStream, metadata);
if (name.indexOf('.') == -1 && contentType != null) {
try {
name += config.getMimeRepository().forName(contentType.toString()).getExtension();
} catch (MimeTypeException e) {
e.printStackTrace();
}
}
String relID = metadata.get(Metadata.EMBEDDED_RELATIONSHIP_ID);
if (relID != null && !name.startsWith(relID)) {
name = relID + "_" + name;
}
File outputFile = new File(extractDir, name);
File parent = outputFile.getParentFile();
if (!parent.exists()) {
if (!parent.mkdirs()) {
throw new IOException("unable to create directory \"" + parent + "\"");
}
}
System.out.println("Extracting '" + name + "' (" + contentType + ") to " + outputFile);
FileOutputStream os = null;
try {
os = new FileOutputStream(outputFile);
if (inputStream instanceof TikaInputStream) {
TikaInputStream tin = (TikaInputStream) inputStream;
if (tin.getOpenContainer() != null && tin.getOpenContainer() instanceof DirectoryEntry) {
POIFSFileSystem fs = new POIFSFileSystem();
copy((DirectoryEntry) tin.getOpenContainer(), fs.getRoot());
fs.writeFilesystem(os);
} else {
IOUtils.copy(inputStream, os);
}
} else {
IOUtils.copy(inputStream, os);
}
} catch (Exception e) {
logger.warn("Ignoring unexpected exception trying to save embedded file " + name, e);
} finally {
if (os != null) {
os.close();
}
}
}
private class FileEmbeddedDocumentExtractor implements EmbeddedDocumentExtractor {
private int count = 0;
private final TikaConfig config = TikaConfig.getDefaultConfig();
public boolean shouldParseEmbedded(Metadata metadata) {
return true;
}
public void parseEmbedded(
InputStream inputStream,
ContentHandler contentHandler,
Metadata metadata,
boolean outputHtml)
throws SAXException, IOException {
String name = metadata.get(Metadata.RESOURCE_NAME_KEY);
if (name == null) {
name = "file" + count++;
}
MediaType contentType = detector.detect(inputStream, metadata);
if (name.indexOf('.') == -1 && contentType != null) {
try {
name += config.getMimeRepository().forName(contentType.toString()).getExtension();
} catch (MimeTypeException e) {
e.printStackTrace();
}
}
String relID = metadata.get(Metadata.EMBEDDED_RELATIONSHIP_ID);
if (relID != null && !name.startsWith(relID)) {
name = relID + "_" + name;
}
File outputFile = new File(extractDir, name);
File parent = outputFile.getParentFile();
if (!parent.exists()) {
if (!parent.mkdirs()) {
throw new IOException("unable to create directory \"" + parent + "\"");
}
}
System.out.println("Extracting '" + name + "' (" + contentType + ") to " + outputFile);
FileOutputStream os = null;
try {
os = new FileOutputStream(outputFile);
if (inputStream instanceof TikaInputStream) {
TikaInputStream tin = (TikaInputStream) inputStream;
if (tin.getOpenContainer() != null && tin.getOpenContainer() instanceof DirectoryEntry) {
POIFSFileSystem fs = new POIFSFileSystem();
copy((DirectoryEntry) tin.getOpenContainer(), fs.getRoot());
fs.writeFilesystem(os);
} else {
IOUtils.copy(inputStream, os);
}
} else {
IOUtils.copy(inputStream, os);
}
} catch (Exception e) {
logger.warn("Ignoring unexpected exception trying to save embedded file " + name, e);
} finally {
if (os != null) {
os.close();
}
}
}
protected void copy(DirectoryEntry sourceDir, DirectoryEntry destDir) throws IOException {
for (org.apache.poi.poifs.filesystem.Entry entry : sourceDir) {
if (entry instanceof DirectoryEntry) {
// Need to recurse
DirectoryEntry newDir = destDir.createDirectory(entry.getName());
copy((DirectoryEntry) entry, newDir);
} else {
// Copy entry
InputStream contents = new DocumentInputStream((DocumentEntry) entry);
try {
destDir.createDocument(entry.getName(), contents);
} finally {
contents.close();
}
}
}
}
}
@Override
protected Parser getParser() {
return new CompositeParser(tikaConfig.getMediaTypeRegistry(), parsers);
}
public ParseResult getParse(Content content) {
String mimeType = content.getContentType();
URL base;
try {
base = new URL(content.getBaseUrl());
} catch (MalformedURLException e) {
return new ParseStatus(e).getEmptyParseResult(content.getUrl(), getConf());
}
// get the right parser using the mime type as a clue
Parser parser = tikaConfig.getParser(MediaType.parse(mimeType));
byte[] raw = content.getContent();
if (parser == null) {
String message = "Can't retrieve Tika parser for mime-type " + mimeType;
LOG.error(message);
return new ParseStatus(ParseStatus.FAILED, message)
.getEmptyParseResult(content.getUrl(), getConf());
}
LOG.debug("Using Tika parser " + parser.getClass().getName() + " for mime-type " + mimeType);
Metadata tikamd = new Metadata();
HTMLDocumentImpl doc = new HTMLDocumentImpl();
doc.setErrorChecking(false);
DocumentFragment root = doc.createDocumentFragment();
DOMBuilder domhandler = new DOMBuilder(doc, root);
ParseContext context = new ParseContext();
try {
parser.parse(new ByteArrayInputStream(raw), domhandler, tikamd, context);
} catch (Exception e) {
LOG.error("Error parsing " + content.getUrl(), e);
return new ParseStatus(ParseStatus.FAILED, e.getMessage())
.getEmptyParseResult(content.getUrl(), getConf());
}
HTMLMetaTags metaTags = new HTMLMetaTags();
String text = "";
String title = "";
Outlink[] outlinks = new Outlink[0];
org.apache.nutch.metadata.Metadata nutchMetadata = new org.apache.nutch.metadata.Metadata();
// we have converted the sax events generated by Tika into a DOM object
// so we can now use the usual HTML resources from Nutch
// get meta directives
HTMLMetaProcessor.getMetaTags(metaTags, root, base);
if (LOG.isTraceEnabled()) {
LOG.trace("Meta tags for " + base + ": " + metaTags.toString());
}
// check meta directives
if (!metaTags.getNoIndex()) { // okay to index
StringBuffer sb = new StringBuffer();
if (LOG.isTraceEnabled()) {
LOG.trace("Getting text...");
}
utils.getText(sb, root); // extract text
text = sb.toString();
sb.setLength(0);
if (LOG.isTraceEnabled()) {
LOG.trace("Getting title...");
}
utils.getTitle(sb, root); // extract title
title = sb.toString().trim();
}
if (!metaTags.getNoFollow()) { // okay to follow links
ArrayList<Outlink> l = new ArrayList<Outlink>(); // extract outlinks
URL baseTag = utils.getBase(root);
if (LOG.isTraceEnabled()) {
LOG.trace("Getting links...");
}
utils.getOutlinks(baseTag != null ? baseTag : base, l, root);
outlinks = l.toArray(new Outlink[l.size()]);
if (LOG.isTraceEnabled()) {
LOG.trace("found " + outlinks.length + " outlinks in " + content.getUrl());
}
}
// populate Nutch metadata with Tika metadata
String[] TikaMDNames = tikamd.names();
for (String tikaMDName : TikaMDNames) {
if (tikaMDName.equalsIgnoreCase(Metadata.TITLE)) continue;
// TODO what if multivalued?
nutchMetadata.add(tikaMDName, tikamd.get(tikaMDName));
}
// no outlinks? try OutlinkExtractor e.g works for mime types where no
// explicit markup for anchors
if (outlinks.length == 0) {
outlinks = OutlinkExtractor.getOutlinks(text, getConf());
}
ParseStatus status = new ParseStatus(ParseStatus.SUCCESS);
if (metaTags.getRefresh()) {
status.setMinorCode(ParseStatus.SUCCESS_REDIRECT);
status.setArgs(
new String[] {
metaTags.getRefreshHref().toString(), Integer.toString(metaTags.getRefreshTime())
});
}
ParseData parseData =
new ParseData(status, title, outlinks, content.getMetadata(), nutchMetadata);
ParseResult parseResult =
ParseResult.createParseResult(content.getUrl(), new ParseImpl(text, parseData));
// run filters on parse
ParseResult filteredParse = this.htmlParseFilters.filter(content, parseResult, metaTags, root);
if (metaTags.getNoCache()) { // not okay to cache
for (Map.Entry<org.apache.hadoop.io.Text, Parse> entry : filteredParse)
entry.getValue().getData().getParseMeta().set(Nutch.CACHING_FORBIDDEN_KEY, cachingPolicy);
}
return filteredParse;
}