@Override
public void parse(String line, ParseState state) throws ParseException {
super.parse(line, state);
final Matcher matcher = match(Constants.EXT_X_VERSION_PATTERN, line);
if (state.getCompatibilityVersion() != ParseState.NONE) {
throw ParseException.create(
ParseExceptionType.MULTIPLE_EXT_TAG_INSTANCES, getTag(), line);
}
final int compatibilityVersion = ParseUtil.parseInt(matcher.group(1), getTag());
if (compatibilityVersion < Playlist.MIN_COMPATIBILITY_VERSION) {
throw ParseException.create(
ParseExceptionType.INVALID_COMPATIBILITY_VERSION, getTag(), line);
}
if (compatibilityVersion > Constants.MAX_COMPATIBILITY_VERSION) {
throw ParseException.create(
ParseExceptionType.UNSUPPORTED_COMPATIBILITY_VERSION, getTag(), line);
}
state.setCompatibilityVersion(compatibilityVersion);
}
/**
* Handle the crawl-delay: directive
*
* @param state current parsing state
* @param token data for directive
* @return true to keep going, false if we're done
*/
private boolean handleCrawlDelay(ParseState state, RobotToken token) {
state.setFinishedAgentFields(true);
if (!state.isAddingRules()) {
return true;
}
String delayString = token.getData();
if (delayString.length() > 0) {
try {
// Some sites use values like 0.5 for the delay.
if (delayString.indexOf('.') != -1) {
double delayValue = Double.parseDouble(delayString) * 1000.0;
state.setCrawlDelay(Math.round(delayValue));
} else {
long delayValue = Integer.parseInt(delayString) * 1000L; // sec to millisec
state.setCrawlDelay(delayValue);
}
} catch (Exception e) {
reportWarning(
"Error parsing robots rules - can't decode crawl delay: " + delayString,
state.getUrl());
}
}
return true;
}
private T parseFirstRow(JsonParser jp, ParseState state)
throws JsonParseException, IOException, JsonProcessingException, JsonMappingException {
skipToField(jp, VALUE_FIELD_NAME, state);
firstId = state.lastId;
firstKey = state.lastKey;
JsonNode value = null;
if (atObjectStart(jp)) {
value = jp.readValueAsTree();
jp.nextToken();
if (isEndOfRow(jp)) {
state.docFieldName = VALUE_FIELD_NAME;
T doc = mapper.readValue(value, type);
endRow(jp, state);
return doc;
}
}
skipToField(jp, INCLUDED_DOC_FIELD_NAME, state);
if (atObjectStart(jp)) {
state.docFieldName = INCLUDED_DOC_FIELD_NAME;
T doc = jp.readValueAs(type);
endRow(jp, state);
return doc;
}
return null;
}
public static java.util.List<RDNEntry> parse(String data) throws ParseException {
java.util.List<RDNEntry> results = new java.util.LinkedList<>();
RDNEntry current = new RDNEntry();
ParseState state = new ParseState();
state.data = data;
state.pos = 0;
while (state.pos < state.data.length()) {
eatWhite(state);
if (state.pos < state.data.length() && state.data.charAt(state.pos) == '!') {
if (!current.rdn.isEmpty()) {
throw new ParseException("negation symbol '!' must appear at start of list");
}
++state.pos;
current.negate = true;
}
current.rdn.add(parseNameComponent(state));
eatWhite(state);
if (state.pos < state.data.length() && state.data.charAt(state.pos) == ',') {
++state.pos;
} else if (state.pos < state.data.length() && state.data.charAt(state.pos) == ';') {
++state.pos;
results.add(current);
current = new RDNEntry();
} else if (state.pos < state.data.length()) {
throw new ParseException(
"expected ',' or ';' at `" + state.data.substring(state.pos) + "'");
}
}
if (!current.rdn.isEmpty()) {
results.add(current);
}
return results;
}
@Override
public void parse(String line, ParseState state) throws ParseException {
if (state.isMaster()) {
state.getMaster().unknownTags.add(line);
} else {
state.getMedia().unknownTags.add(line);
}
};
@Override
public void parse(String line, ParseState state) throws ParseException {
if (state.isExtended()) {
throw ParseException.create(
ParseExceptionType.MULTIPLE_EXT_TAG_INSTANCES, getTag(), line);
}
state.setExtended();
}
public C parse(C commandInstance, String... args) {
Preconditions.checkNotNull(args, "args is null");
Parser parser = new Parser();
ParseState state = parser.parse(metadata, args);
if (state.getCommand() == null) {
if (state.getGroup() != null) {
state = state.withCommand(state.getGroup().getDefaultCommand());
} else {
state = state.withCommand(metadata.getDefaultCommand());
}
}
validate(state);
CommandMetadata command = state.getCommand();
return injectOptions(
commandInstance,
command.getAllOptions(),
state.getParsedOptions(),
command.getArguments(),
state.getParsedArguments(),
command.getMetadataInjections(),
ImmutableMap.<Class<?>, Object>of(GlobalMetadata.class, metadata));
}
protected void parse() {
Vector vl = new Vector(4);
ParseState ps = new ParseState(roff, rlen);
ps.separator = (byte) ',';
ps.spaceIsSep = false;
while (HttpParser.nextItem(raw, ps) >= 0) {
vl.addElement(new HttpAcceptLanguage(this, raw, ps.start, ps.end));
ps.prepare();
}
languages = new HttpAcceptLanguage[vl.size()];
vl.copyInto(languages);
}
@Override
public void parse(String line, ParseState state) throws ParseException {
super.parse(line, state);
match(Constants.EXT_X_I_FRAMES_ONLY_PATTERN, line);
if (state.getCompatibilityVersion() < 4) {
throw ParseException.create(
ParseExceptionType.REQUIRES_PROTOCOL_VERSION_4_OR_HIGHER, getTag());
}
state.setIsIframesOnly();
}
@Override
public void parse(String line, ParseState state) throws ParseException {
super.parse(line, state);
final Matcher matcher = match(Constants.EXT_X_MEDIA_SEQUENCE_PATTERN, line);
if (state.getMedia().mediaSequenceNumber != null) {
throw ParseException.create(
ParseExceptionType.MULTIPLE_EXT_TAG_INSTANCES, getTag(), line);
}
state.getMedia().mediaSequenceNumber = ParseUtil.parseInt(matcher.group(1), getTag());
}
@Override
public void parse(String line, ParseState state) throws ParseException {
super.parse(line, state);
final Matcher matcher = match(Constants.EXT_X_TARGETDURATION_PATTERN, line);
if (state.getMedia().targetDuration != null) {
throw ParseException.create(
ParseExceptionType.MULTIPLE_EXT_TAG_INSTANCES, getTag(), line);
}
state.getMedia().targetDuration = ParseUtil.parseInt(matcher.group(1), getTag());
}
public LDAPExpr parse(String strExpr) throws LDAPParseException {
if (strExpr == null || strExpr.trim().length() == 0) {
return LDAPExpr.ACCEPT_ALL;
}
ParseState ps = new ParseState(strExpr);
LDAPExpr expr = parseExpr(ps);
ps.skipWhitespace();
if (!ps.isEndOfString()) {
error("expected end of expression ", ps);
}
return expr;
}
@Override
public void parse(String line, ParseState state) throws ParseException {
super.parse(line, state);
final Matcher matcher = match(Constants.EXT_X_PLAYLIST_TYPE_PATTERN, line);
if (state.getMedia().playlistType != null) {
throw ParseException.create(
ParseExceptionType.MULTIPLE_EXT_TAG_INSTANCES, getTag(), line);
}
state.getMedia().playlistType =
ParseUtil.parseEnum(matcher.group(1), PlaylistType.class, getTag());
}
private void validate(ParseState state) {
CommandMetadata command = state.getCommand();
if (command == null) {
List<String> unparsedInput = state.getUnparsedInput();
if (unparsedInput.isEmpty()) {
throw new ParseCommandMissingException();
} else {
throw new ParseCommandUnrecognizedException(unparsedInput);
}
}
ArgumentsMetadata arguments = command.getArguments();
if (state.getParsedArguments().isEmpty() && arguments != null && arguments.isRequired()) {
throw new ParseArgumentsMissingException(arguments.getTitle());
}
if (!state.getUnparsedInput().isEmpty()) {
throw new ParseArgumentsUnexpectedException(state.getUnparsedInput());
}
if (state.getLocation() == Context.OPTION) {
throw new ParseOptionMissingValueException(state.getCurrentOption().getTitle());
}
for (OptionMetadata option : command.getAllOptions()) {
if (option.isRequired() && !state.getParsedOptions().containsKey(option)) {
throw new ParseOptionMissingException(option.getOptions().iterator().next());
}
}
}
/**
* Handle the sitemap: directive
*
* @param state current parsing state
* @param token data for directive
* @return true to keep going, false if we're done
*/
private boolean handleSitemap(ParseState state, RobotToken token) {
String sitemap = token.getData();
try {
URL sitemap_url = new URL(new URL(state.getUrl()), sitemap);
String hostname = sitemap_url.getHost();
if ((hostname != null) && (hostname.length() > 0)) {
state.addSitemap(sitemap_url.toExternalForm());
}
} catch (Exception e) {
reportWarning("Invalid URL with sitemap directive: " + sitemap, state.getUrl());
}
return true;
}
@Override
public void parse(String line, ParseState state) throws ParseException {
super.parse(line, state);
match(Constants.EXT_X_ENDLIST_PATTERN, line);
state.getMedia().endOfList = true;
}
private void skipToField(JsonParser jp, String fieldName, ParseState state)
throws JsonParseException, IOException {
String lastFieldName = null;
while (jp.getCurrentToken() != null) {
switch (jp.getCurrentToken()) {
case FIELD_NAME:
lastFieldName = jp.getCurrentName();
jp.nextToken();
break;
case START_OBJECT:
if (!state.inRow) {
state.inRow = true;
jp.nextToken();
} else {
if (isInField(fieldName, lastFieldName)) {
return;
} else {
jp.skipChildren();
}
}
break;
default:
if (isInField(fieldName, lastFieldName)) {
jp.nextToken();
return;
}
jp.nextToken();
break;
}
}
}
public static java.util.List<RDNPair> parseStrict(String data) throws ParseException {
java.util.List<RDNPair> results = new java.util.LinkedList<>();
ParseState state = new ParseState();
state.data = data;
state.pos = 0;
while (state.pos < state.data.length()) {
results.add(parseNameComponent(state));
eatWhite(state);
if (state.pos < state.data.length()
&& (state.data.charAt(state.pos) == ',' || state.data.charAt(state.pos) == ';')) {
++state.pos;
} else if (state.pos < state.data.length()) {
throw new ParseException(
"expected ',' or ';' at `" + state.data.substring(state.pos) + "'");
}
}
return results;
}
@Override
public void parse(String line, ParseState state) throws ParseException {
super.parse(line, state);
final Matcher matcher = match(Constants.EXTINF_PATTERN, line);
state.getMedia().trackInfo =
new TrackInfo(ParseUtil.parseFloat(matcher.group(1), getTag()), matcher.group(2));
}
/**
* Handle a line that starts with http: and contains "sitemap", which we treat as a missing
* sitemap: xxx directive.
*
* @param state current parsing state
* @param token data for directive
* @return true to keep going, false if we're done
*/
private boolean handleHttp(ParseState state, RobotToken token) {
String urlFragment = token.getData();
if (urlFragment.contains("sitemap")) {
RobotToken fixedToken = new RobotToken(RobotDirective.SITEMAP, "http:" + token.getData());
return handleSitemap(state, fixedToken);
} else {
reportWarning("Found raw non-sitemap URL: http:" + urlFragment, state.getUrl());
return true;
}
}
@Override
public void parse(String line, ParseState state) throws ParseException {
super.parse(line, state);
final StartData.Builder builder = new StartData.Builder();
parseAttributes(line, builder, state, HANDLERS);
final StartData startData = builder.build();
state.getMedia().startData = startData;
}
public C parse(C commandInstance, String... args) {
Preconditions.checkNotNull(args, "args is null");
Parser parser = new Parser(metadata);
ParseState state = parser.parse(args);
CommandMetadata command = MetadataLoader.loadCommand(commandInstance.getClass());
state = state.withCommand(command);
validate(state);
ImmutableMap.Builder<Class<?>, Object> bindings =
ImmutableMap.<Class<?>, Object>builder().put(GlobalMetadata.class, metadata);
if (state.getGroup() != null) {
bindings.put(CommandGroupMetadata.class, state.getGroup());
}
bindings.put(CommandMetadata.class, command);
C c =
(C)
ParserUtil.injectOptions(
commandInstance,
command.getAllOptions(),
state.getParsedOptions(),
command.getArguments(),
state.getParsedArguments(),
command.getMetadataInjections(),
bindings.build());
return c;
}
private Object parseFirstRow(JsonParser jp, ParseState state, Class clazz)
throws JsonParseException, IOException, JsonProcessingException, JsonMappingException {
skipToField(jp, VALUE_FIELD_NAME, state);
JsonNode value = null;
if (atObjectStart(jp)) {
value = jp.readValueAsTree();
jp.nextToken();
if (isEndOfRow(jp)) {
state.docFieldName = VALUE_FIELD_NAME;
Object doc = objectMapper.readValue(value, clazz);
endRow(jp, state);
return doc;
}
}
skipToField(jp, INCLUDED_DOC_FIELD_NAME, state);
if (atObjectStart(jp)) {
state.docFieldName = INCLUDED_DOC_FIELD_NAME;
Object doc = jp.readValueAs(clazz);
endRow(jp, state);
return doc;
}
return null;
}
public static JSONValue parseJSONValue(ParseState parseState) {
if (parseState.skipWhite()) {
char c = parseState.current();
switch (c) {
case '{':
return JSONObject.parseJSON(parseState);
case '"':
return JSONString.parseJSON(parseState);
case '[':
return JSONArray.parseJSON(parseState);
case 'n':
return JSONNull.parseJSON(parseState);
case 't':
case 'f':
return JSONBoolean.parseJSON(parseState);
default:
if ((c >= '0' && c <= '9') || c == '-' || c == '.' || c == 'N') // NaN ok
return JSONNumber.parseJSON(parseState);
break;
}
}
return null;
}
private void skipToField(JsonParser jp, String fieldName, ParseState state)
throws JsonParseException, IOException {
String lastFieldName = null;
while (jp.getCurrentToken() != null) {
switch (jp.getCurrentToken()) {
case FIELD_NAME:
lastFieldName = jp.getCurrentName();
jp.nextToken();
break;
case START_OBJECT:
if (!state.inRow) {
state.inRow = true;
jp.nextToken();
} else {
if (isInField(fieldName, lastFieldName)) {
state.fieldIsNull = false;
return;
} else {
jp.skipChildren();
}
}
break;
default:
if (isInField(ID_FIELD_NAME, lastFieldName)) {
state.lastId = jp.readValueAsTree().getTextValue();
} else if (isInField(KEY_FIELD_NAME, lastFieldName)) {
state.lastKey = jp.readValueAsTree();
} else if (isInField(ERROR_FIELD_NAME, lastFieldName)) {
JsonNode error = jp.readValueAsTree();
if (ignoreNotFound && error.asText().equals("not_found")) {
lastFieldName = null;
state.inRow = false;
jp.nextToken();
} else {
throw new ViewResultException(state.lastKey, error.asText());
}
} else if (isInField(fieldName, lastFieldName)) {
if (jp.getCurrentToken() == JsonToken.VALUE_NULL) {
state.fieldIsNull = true;
} else {
state.fieldIsNull = false;
}
jp.nextToken();
return;
}
jp.nextToken();
break;
}
}
}
/**
* Handle the allow: directive
*
* @param state current parsing state
* @param token data for directive
* @return true to keep going, false if we're done
*/
private boolean handleAllow(ParseState state, RobotToken token) {
state.setFinishedAgentFields(true);
if (!state.isAddingRules()) {
return true;
}
String path = token.getData();
try {
path = URLDecoder.decode(path, "UTF-8");
} catch (Exception e) {
reportWarning("Error parsing robots rules - can't decode path: " + path, state.getUrl());
}
if (path.length() == 0) {
// Allow: <nothing> => allow all.
state.clearRules();
} else {
state.addRule(path, true);
}
return true;
}
// cloneMsgArg is used when the split buffer scenario has the pubArg in the existing read buffer,
// but
// we need to hold onto it into the next read.
private void cloneMsgArg() {
ps.argBuf = ByteBuffer.wrap(ps.argBufStore);
// System.err.printf("ps.argBuf 1= %s\n", ps.argBuf);
ps.argBuf.put(ps.ma.subject, 0, ps.ma.subjectLength);
// System.err.printf("ps.argBuf 2= %s\n", ps.argBuf);
if (ps.ma.replyLength != 0) {
ps.argBuf.put(ps.ma.reply, 0, ps.ma.replyLength);
}
// System.err.printf("ps.argBuf 3= %s\n", ps.argBuf);
ps.argBuf.rewind();
ps.argBuf.get(ps.ma.subject, 0, ps.ma.subjectLength);
if (ps.ma.replyLength != 0) {
ps.argBuf.get(ps.ma.reply, 0, ps.ma.replyLength);
}
// ps.argBuf.flip();
// System.err.printf("ps.argBuf 4= %s\n", ps.argBuf);
}
@Override
public void parse(String line, ParseState state) throws ParseException {
super.parse(line, state);
final EncryptionData.Builder builder =
new EncryptionData.Builder()
.withKeyFormat(Constants.DEFAULT_KEY_FORMAT)
.withKeyFormatVersions(Constants.DEFAULT_KEY_FORMAT_VERSIONS);
parseAttributes(line, builder, state, HANDLERS);
final EncryptionData encryptionData = builder.build();
if (encryptionData.getMethod() != EncryptionMethod.NONE
&& encryptionData.getUri() == null) {
throw ParseException.create(ParseExceptionType.MISSING_ENCRYPTION_URI, getTag(), line);
}
state.getMedia().encryptionData = encryptionData;
}
private void endRow(JsonParser jp, ParseState state) throws IOException, JsonParseException {
state.inRow = false;
jp.nextToken();
}
/**
* Handle the user-agent: directive
*
* @param state current parsing state
* @param token data for directive
* @return true to keep going, false if we're done
*/
private boolean handleUserAgent(ParseState state, RobotToken token) {
if (state.isMatchedRealName()) {
if (state.isFinishedAgentFields()) {
// We're all done.
return false;
} else {
// Skip any more of these, once we have a real name match. We're waiting for some
// allow/disallow/crawl delay fields.
return true;
}
}
if (state.isFinishedAgentFields()) {
// We've got a user agent field, so we haven't yet seen anything that tells us
// we're done with this set of agent names.
state.setFinishedAgentFields(false);
state.setAddingRules(false);
}
// Handle the case when there are multiple target names are passed
String[] targetNames = state.getTargetName().split(",");
for (int count = 0; count < targetNames.length; count++) {
// Extract possible match names from our target agent name, since it appears
// to be expected that "Mozilla botname 1.0" matches "botname"
String[] targetNameSplits = targetNames[count].trim().split(" ");
// TODO KKr - catch case of multiple names, log as non-standard.
String[] agentNames = token.getData().split("[ \t,]");
for (String agentName : agentNames) {
agentName = agentName.trim();
if (agentName.isEmpty()) {
// Ignore empty names
} else if (agentName.equals("*") && !state.isMatchedWildcard()) {
state.setMatchedWildcard(true);
state.setAddingRules(true);
} else {
for (String targetName : targetNameSplits) {
if (targetName.startsWith(agentName)) {
state.setMatchedRealName(true);
state.setAddingRules(true);
state.clearRules(); // In case we previously hit a wildcard rule match
break;
}
}
}
}
}
// Keep going
return true;
}
