/**
* Figure out directive on line of text from robots.txt file. We assume the line has been
* lower-cased
*
* @param line
* @return robot command found on line
*/
private static RobotToken tokenize(String line) {
String lowerLine = line.toLowerCase();
for (String prefix : DIRECTIVE_PREFIX.keySet()) {
int prefixLength = prefix.length();
if (lowerLine.startsWith(prefix)) {
RobotDirective directive = DIRECTIVE_PREFIX.get(prefix);
String dataPortion = lowerLine.substring(prefixLength);
// preserve the original case for sitemaps
if (directive.equals(RobotDirective.SITEMAP)) dataPortion = line.substring(prefixLength);
if (directive.isPrefix()) {
Matcher m = DIRECTIVE_SUFFIX_PATTERN.matcher(dataPortion);
if (m.matches()) {
dataPortion = m.group(1);
} else {
continue;
}
}
Matcher m = COLON_DIRECTIVE_DELIMITER.matcher(dataPortion);
if (!m.matches()) {
m = BLANK_DIRECTIVE_DELIMITER.matcher(dataPortion);
}
if (m.matches()) {
return new RobotToken(directive, m.group(1).trim());
}
}
}
Matcher m = COLON_DIRECTIVE_DELIMITER.matcher(lowerLine);
if (m.matches()) {
return new RobotToken(RobotDirective.UNKNOWN, line);
} else {
return new RobotToken(RobotDirective.MISSING, line);
}
}
