private void save_noLock() throws IOException, SearchLibException {
PrintWriter pw = null;
try {
pw = new PrintWriter(authFile, "UTF-8");
XmlWriter xmlWriter = new XmlWriter(pw, "UTF-8");
xmlWriter.startElement(
AUTH_ITEM_ROOT_NODE,
AUTH_ATTR_ENABLED,
Boolean.toString(enabled),
AUTH_ATTR_INDEX,
index,
AUTH_ATTR_USER_ALLOW_FIELD,
userAllowField,
AUTH_ATTR_USER_DENY_FIELD,
userDenyField,
AUTH_ATTR_GROUP_ALLOW_FIELD,
groupAllowField,
AUTH_ATTR_GROUP_DENY_FIELD,
groupDenyField,
AUTH_ATTR_DEFAULT_USER,
defaultUser,
AUTH_ATTR_DEFAULT_GROUP,
defaultGroup);
xmlWriter.endElement();
} catch (TransformerConfigurationException e) {
throw new SearchLibException(e);
} catch (SAXException e) {
throw new SearchLibException(e);
} finally {
IOUtils.close(pw);
}
}
@Override
public User getUser(Renderer renderer, String username, String password) throws IOException {
if (StringUtils.isEmpty(username) || StringUtils.isEmpty(password))
throw new AuthException("Username or password is empty");
if (StringUtils.isEmpty(renderer.getAuthServer()))
throw new AuthException("No auth server given, check the parameters of the renderer");
ActiveDirectory activeDirectory = null;
try {
String domain = renderer.getAuthDomain();
String authServer = renderer.getAuthServer();
User user = AuthUserCache.INSTANCE.get(username, domain);
if (user != null) return user;
NtlmPasswordAuthentication ntlmAuth = getNtlmAuth(renderer, username, password);
UniAddress dc = UniAddress.getByName(authServer, true);
SmbSession.logon(dc, ntlmAuth);
activeDirectory =
new ActiveDirectory(authServer, ntlmAuth.getUsername(), ntlmAuth.getPassword(), domain);
NamingEnumeration<SearchResult> result = activeDirectory.findUser(username);
Attributes attrs = ActiveDirectory.getAttributes(result);
if (attrs == null) throw new AuthException("No user found: " + username);
String userId = ActiveDirectory.getObjectSID(attrs);
List<ADGroup> groups = new ArrayList<ADGroup>();
activeDirectory.findUserGroups(attrs, groups);
String dnUser = ActiveDirectory.getStringAttribute(attrs, "DistinguishedName");
if (!StringUtils.isEmpty(dnUser)) activeDirectory.findUserGroup(dnUser, groups);
Logging.info("USER authenticated: " + user);
user =
new User(
userId.toLowerCase(),
username.toLowerCase(),
password,
ActiveDirectory.toArray(groups, "everyone"),
ActiveDirectory.getDisplayString(domain, username));
AuthUserCache.INSTANCE.add(username, domain, user);
return user;
} catch (SmbAuthException e) {
Logging.warn(e);
throw new AuthException("Authentication error (SmbAuthException) : " + e.getMessage());
} catch (UnknownHostException e) {
Logging.warn(e);
throw new AuthException("Authentication error (UnknownHostException) : " + e.getMessage());
} catch (NamingException e) {
Logging.warn(e);
throw new AuthException("LDAP error (NamingException) : " + e.getMessage());
} finally {
IOUtils.close(activeDirectory);
}
}
public static final boolean receive_file_exists(
Client client, String filePath, long lastModified, long length) throws IOException {
File existsFile = new File(client.getDirectory(), filePath);
if (!existsFile.exists()) return false;
if (existsFile.lastModified() != lastModified) return false;
if (existsFile.length() != length) return false;
File rootDir = getTempReceiveDir(client);
IOUtils.appendLines(new File(rootDir, PATH_TO_MOVE), filePath);
return true;
}
public static final void receive_file(
Client client, String filePath, long lastModified, InputStream is) throws IOException {
File rootDir = getTempReceiveDir(client);
File targetFile = new File(rootDir, filePath);
targetFile.createNewFile();
FileOutputStream fos = null;
try {
fos = new FileOutputStream(targetFile);
int len;
byte[] buffer = new byte[131072];
while ((len = is.read(buffer)) != -1) fos.write(buffer, 0, len);
} catch (IOException e) {
throw e;
} finally {
IOUtils.close(fos);
}
targetFile.setLastModified(lastModified);
}
@Command
public void onExportReport() throws SearchLibException, IOException {
PrintWriter pw = null;
try {
File tempFile = File.createTempFile("OSS_Query_Reports", "csv");
pw = new PrintWriter(tempFile);
for (FacetItem facetItem : reportSet) {
pw.print('"');
pw.print(facetItem.getTerm().replaceAll("\"", "\"\""));
pw.print('"');
pw.print(',');
pw.println(facetItem.getCount());
}
pw.close();
pw = null;
Filedownload.save(
new FileInputStream(tempFile), "text/csv; charset-UTF-8", "OSS_Query_Reports.csv");
} finally {
IOUtils.close(pw);
}
}
@Override
protected void parseContent(StreamLimiter streamLimiter, LanguageEnum lang) throws IOException {
HSSFWorkbook workbook = new HSSFWorkbook(streamLimiter.getNewInputStream());
ExcelExtractor excel = null;
try {
excel = new ExcelExtractor(workbook);
ParserResultItem result = getNewParserResultItem();
SummaryInformation info = excel.getSummaryInformation();
if (info != null) {
result.addField(ParserFieldEnum.title, info.getTitle());
result.addField(ParserFieldEnum.author, info.getAuthor());
result.addField(ParserFieldEnum.subject, info.getSubject());
}
String content = excel.getText();
result.addField(ParserFieldEnum.content, StringUtils.replaceConsecutiveSpaces(content, " "));
result.langDetection(10000, ParserFieldEnum.content);
} finally {
IOUtils.close(excel);
}
}
/**
* Download the file and extract content informations
*
* @param httpDownloader
*/
public DownloadItem download(HttpDownloader httpDownloader) {
synchronized (this) {
InputStream is = null;
DownloadItem downloadItem = null;
try {
URL url = urlItem.getURL();
if (url == null) throw new MalformedURLException("Malformed URL: " + urlItem.getUrl());
// URL normalisation
URI uri = url.toURI();
url = uri.toURL();
credentialItem = credentialManager == null ? null : credentialManager.matchCredential(url);
String externalFormUrl = url.toExternalForm();
downloadItem = crawlCacheManager.loadCache(uri);
boolean fromCache = (downloadItem != null);
if (!fromCache) {
List<CookieItem> cookieList = cookieManager.getItems(externalFormUrl);
List<HeaderItem> headerList = headerManager.getItems(externalFormUrl);
downloadItem = httpDownloader.get(uri, credentialItem, headerList, cookieList);
} else if (Logging.isDebug) Logging.debug("Crawl cache deliver: " + uri);
urlItem.setContentDispositionFilename(downloadItem.getContentDispositionFilename());
urlItem.setContentBaseType(downloadItem.getContentBaseType());
urlItem.setContentTypeCharset(downloadItem.getContentTypeCharset());
urlItem.setContentEncoding(downloadItem.getContentEncoding());
urlItem.setContentLength(downloadItem.getContentLength());
urlItem.setLastModifiedDate(downloadItem.getLastModified());
urlItem.setFetchStatus(FetchStatus.FETCHED);
urlItem.setHeaders(downloadItem.getHeaders());
Integer code = downloadItem.getStatusCode();
if (code == null) throw new IOException("Http status is null");
urlItem.setResponseCode(code);
redirectUrlLocation = downloadItem.getRedirectLocation();
if (redirectUrlLocation != null)
urlItem.setRedirectionUrl(redirectUrlLocation.toURL().toExternalForm());
urlItem.setBacklinkCount(config.getUrlManager().countBackLinks(urlItem.getUrl()));
if (code >= 200 && code < 300) {
if (!fromCache) is = crawlCacheManager.storeCache(downloadItem);
else is = downloadItem.getContentInputStream();
parseContent(is);
} else if (code == 301) {
urlItem.setFetchStatus(FetchStatus.REDIR_PERM);
} else if (code > 301 && code < 400) {
urlItem.setFetchStatus(FetchStatus.REDIR_TEMP);
} else if (code >= 400 && code < 500) {
urlItem.setFetchStatus(FetchStatus.GONE);
} else if (code >= 500 && code < 600) {
urlItem.setFetchStatus(FetchStatus.HTTP_ERROR);
}
} catch (FileNotFoundException e) {
Logging.info("FileNotFound: " + urlItem.getUrl());
urlItem.setFetchStatus(FetchStatus.GONE);
setError("FileNotFound: " + urlItem.getUrl());
} catch (LimitException e) {
Logging.warn(e.toString() + " (" + urlItem.getUrl() + ")");
urlItem.setFetchStatus(FetchStatus.SIZE_EXCEED);
setError(e.getMessage());
} catch (InstantiationException e) {
Logging.error(e.getMessage(), e);
urlItem.setParserStatus(ParserStatus.PARSER_ERROR);
setError(e.getMessage());
} catch (IllegalAccessException e) {
Logging.error(e.getMessage(), e);
urlItem.setParserStatus(ParserStatus.PARSER_ERROR);
setError(e.getMessage());
} catch (ClassNotFoundException e) {
Logging.error(e.getMessage(), e);
urlItem.setParserStatus(ParserStatus.PARSER_ERROR);
setError(e.getMessage());
} catch (URISyntaxException e) {
Logging.warn(e.getMessage(), e);
urlItem.setFetchStatus(FetchStatus.URL_ERROR);
setError(e.getMessage());
} catch (MalformedURLException e) {
Logging.warn(e.getMessage(), e);
urlItem.setFetchStatus(FetchStatus.URL_ERROR);
setError(e.getMessage());
} catch (IOException e) {
Logging.error(e.getMessage(), e);
urlItem.setFetchStatus(FetchStatus.ERROR);
setError(e.getMessage());
} catch (IllegalArgumentException e) {
Logging.error(e.getMessage(), e);
urlItem.setFetchStatus(FetchStatus.ERROR);
setError(e.getMessage());
} catch (Exception e) {
Logging.error(e.getMessage(), e);
urlItem.setFetchStatus(FetchStatus.ERROR);
setError(e.getMessage());
} finally {
IOUtils.close(is);
}
return downloadItem;
}
}