public static String simplifyURL(String url) {
if (Strings.isBlank(url)) {
return "";
}
if (url.indexOf("?") != -1) {
url = url.substring(0, url.indexOf("?"));
}
if (url.indexOf("#") != -1) {
url = url.substring(0, url.indexOf("#"));
}
url = url.trim();
url = url.replaceFirst("^[a-z]+://", "");
url = url.replaceFirst("^www\\.", "");
String[] parts = url.split("/");
StringBuilder simplified = new StringBuilder();
for (int i = 0; i < parts.length; i++) {
if (i == 0) {
simplified.append(parts[i]);
} else if (i == parts.length - 1) {
simplified.append(" : ");
String part = parts[i];
simplified.append(Strings.decodeURL(part).trim());
}
}
return simplified.toString();
}
public Locale getLocale(String text) {
if (Strings.isBlank(text)) {
return null;
}
LanguageIdentifier id = new LanguageIdentifier(text);
String language = id.getLanguage();
if (Strings.isBlank(language)) {
return null;
}
return new Locale(language);
}
public TextAnalysis analyse(String text) throws ModelException {
String[] words = semanticService.getWords(text);
semanticService.lowercaseWords(words);
List<String> uniqueWords = Strings.asList(semanticService.getUniqueWords(words));
WordListPerspectiveQuery query = new WordListPerspectiveQuery().withWords(uniqueWords);
List<WordListPerspective> list = modelService.list(query);
List<String> unknownWords = Lists.newArrayList();
Set<String> knownWords = new HashSet<String>();
Multimap<String, String> wordsByLanguage = HashMultimap.create();
for (WordListPerspective perspective : list) {
String word = perspective.getText().toLowerCase();
knownWords.add(word);
if (perspective.getLanguage() != null) {
wordsByLanguage.put(perspective.getLanguage(), word);
}
}
Multiset<String> languages = wordsByLanguage.keys();
String language = null;
for (String lang : languages) {
if (language == null
|| (wordsByLanguage.get(lang).size() > wordsByLanguage.get(language).size())) {
language = lang;
}
}
for (String word : uniqueWords) {
if (!knownWords.contains(word)) {
unknownWords.add(word);
}
}
Locale possibleLocale = Locale.ENGLISH;
String[] sentences = semanticService.getSentences(text, possibleLocale);
TextAnalysis analysis = new TextAnalysis();
analysis.setLanguage(language);
analysis.setSentences(Strings.asList(sentences));
analysis.setWordsByLanguage(wordsByLanguage.asMap());
analysis.setUniqueWords(uniqueWords);
analysis.setKnownWords(list);
analysis.setUnknownWords(unknownWords);
return analysis;
}
public Language getLanguageForCode(String code) {
if (Strings.isBlank(code)) {
return null;
}
Query<Language> query = Query.of(Language.class).withField(Language.CODE, code);
return modelService.search(query).getFirst();
}
public LexicalCategory getLexcialCategoryForCode(String code) {
if (Strings.isBlank(code)) {
return null;
}
Query<LexicalCategory> query =
Query.of(LexicalCategory.class).withField(LexicalCategory.CODE, code);
return modelService.search(query).getFirst();
}
public void afterPropertiesSet() throws Exception {
key = getRequest().getString("key");
if (Strings.isNotBlank(key)) {
user = passwordRecoveryService.getUserByRecoveryKey(key);
found = user != null;
}
}
/**
* For use when indexing - has no other usage
*
* @param text
* @return
*/
public static String getAlphabethStartLetter(String text) {
if (text == null) {
return "none";
}
text = text.trim();
if (text.length() == 0) {
return "none";
}
String letter = text.substring(0, 1).toLowerCase();
if (Strings.contains(letter, ALPHABETH)) {
return letter;
}
letter = deAccent(letter);
if (Strings.contains(letter, ALPHABETH)) {
return letter;
}
if (StringUtils.isNumeric(letter)) {
return "number";
}
return "other";
}
public static String[] getWords(String query) {
if (Strings.isBlank(query)) {
return new String[] {};
}
return query.trim().split("\\s+");
}
public static String fallback(String str, String defaultValue) {
if (Strings.isBlank(str)) {
return defaultValue;
}
return str;
}
