public OffsetRange getReferenceSpan(Document doc, int lexOffset) {
    TokenHierarchy<Document> th = TokenHierarchy.get(doc);

    // BaseDocument doc = (BaseDocument)document;

    TokenSequence<? extends PythonTokenId> ts = PythonLexerUtils.getPythonSequence(th, lexOffset);

    if (ts == null) {
      return OffsetRange.NONE;
    }

    ts.move(lexOffset);

    if (!ts.moveNext() && !ts.movePrevious()) {
      return OffsetRange.NONE;
    }

    // Determine whether the caret position is right between two tokens
    boolean isBetween = (lexOffset == ts.offset());

    OffsetRange range = getReferenceSpan(ts, th, lexOffset);

    if ((range == OffsetRange.NONE) && isBetween) {
      // The caret is between two tokens, and the token on the right
      // wasn't linkable. Try on the left instead.
      if (ts.movePrevious()) {
        range = getReferenceSpan(ts, th, lexOffset);
      }
    }

    return range;
  }
  private OffsetRange getReferenceSpan(
      TokenSequence<?> ts, TokenHierarchy<Document> th, int lexOffset) {
    Token<?> token = ts.token();
    TokenId id = token.id();

    //        if (id == PythonTokenId.IDENTIFIER) {
    //            if (token.length() == 1 && id == PythonTokenId.IDENTIFIER &&
    // token.text().toString().equals(",")) {
    //                return OffsetRange.NONE;
    //            }
    //        }

    // TODO: Tokens.SUPER, Tokens.THIS, Tokens.SELF ...
    if (id == PythonTokenId.IDENTIFIER) {
      return new OffsetRange(ts.offset(), ts.offset() + token.length());
    }

    // Look for embedded RDoc comments:
    TokenSequence<?> embedded = ts.embedded();

    if (embedded != null) {
      ts = embedded;
      embedded.move(lexOffset);

      if (embedded.moveNext()) {
        Token<?> embeddedToken = embedded.token();

        if (embeddedToken.id() == PythonStringTokenId.URL) {
          return new OffsetRange(embedded.offset(), embedded.offset() + embeddedToken.length());
        }
        // Recurse into the range - perhaps there is Ruby code (identifiers

        // etc.) to follow there
        OffsetRange range = getReferenceSpan(embedded, th, lexOffset);

        if (range != OffsetRange.NONE) {
          return range;
        }
      }
    }

    return OffsetRange.NONE;
  }
  @NbBundle.Messages({
    "# {0} - PI target",
    "ERR_invalidProcessingInstruction=Invalid processing instruction: {0}. Expected 'import', 'include' or 'language'",
    "ERR_missingProcessingInstruction=Missing processing intruction."
  })
  private void handleErrorInstruction(String target, String data) {
    int start = contentLocator.getElementOffset();
    int offset = -1;
    int piOffset = -1;

    TokenSequence<XMLTokenId> seq = contentLocator.getTokenSequence();

    // lex up to the invalid target:
    seq.move(start);
    boolean found = false;
    while (!found && seq.moveNext()) {
      Token<XMLTokenId> t = seq.token();
      switch (t.id()) {
        case PI_START:
          piOffset = offset;
          if (target == null) {
            found = true;
          }
        case WS:
          break;

        default:
        case PI_TARGET:
          offset = seq.offset();
          found = true;
          break;
      }
    }
    ErrorMark mark;

    if (target != null) {
      mark =
          new ErrorMark(
              offset,
              seq.token().length(),
              "invalid-processing-instruction",
              ERR_invalidProcessingInstruction(target),
              target);
    } else {
      mark =
          new ErrorMark(
              piOffset,
              seq.token().length(),
              "missing-processing-instruction",
              ERR_missingProcessingInstruction());
    }
    addError(mark);
  }
 public boolean verifyState(Document doc, int offset) {
   TokenHierarchy hi = TokenHierarchy.get(doc);
   TokenSequence<HTMLTokenId> ts = hi.tokenSequence(HTMLTokenId.language());
   if (ts != null) {
     ts.move(offset);
     ts.moveNext();
     Token<HTMLTokenId> tok = ts.token();
     int newOffset = ts.offset();
     String matcherText = tok.text().toString();
     Matcher m = MY_SPECIAL_PATTERN.matcher(matcherText);
     if (m.matches()) {
       target = m.group(1);
       int idx = matcherText.indexOf(target);
       targetStart = newOffset + idx;
       targetEnd = targetStart + target.length();
       return true;
     }
   }
   return false;
 }
  private DeclarationLocation findImport(PythonParserResult info, int lexOffset, BaseDocument doc) {
    TokenSequence<? extends PythonTokenId> ts =
        PythonLexerUtils.getPositionedSequence(doc, lexOffset);
    if (ts == null) {
      return DeclarationLocation.NONE;
    }
    if (ts.offset() == lexOffset) {
      // We're looking at the offset to the RIGHT of the caret
      // and here I care about what's on the left
      if (!ts.movePrevious()) {
        return DeclarationLocation.NONE;
      }
    }

    Token<? extends PythonTokenId> token = ts.token();
    if (token == null) {
      return DeclarationLocation.NONE;
    }

    TokenId id = token.id();

    String moduleName = null;
    while (true) {
      if (id == PythonTokenId.IDENTIFIER || id.primaryCategory().equals(PythonLexer.KEYWORD_CAT)) {
        // Possibly inside the import string
        String tokenText = token.text().toString();
        if (moduleName == null) {
          moduleName = tokenText;
        } else {
          moduleName = tokenText + "." + moduleName;
        }
      } else if (id != PythonTokenId.DOT) {
        break;
      }
      if (!ts.movePrevious()) {
        return DeclarationLocation.NONE;
      }
      token = ts.token();
      id = token.id();
    }

    if (id != PythonTokenId.ERROR
        && id != PythonTokenId.NEWLINE
        && id != PythonTokenId.WHITESPACE) {
      return DeclarationLocation.NONE;
    }

    if (!ts.movePrevious()) {
      return DeclarationLocation.NONE;
    }
    token = ts.token();
    id = token.id();
    if (id != PythonTokenId.IMPORT) {
      return DeclarationLocation.NONE;
    }
    if (moduleName == null) {
      return DeclarationLocation.NONE;
    }

    if (id == PythonTokenId.IMPORT || id == PythonTokenId.FROM) {
      if (id == PythonTokenId.IMPORT
          && ts.movePrevious()
          && ts.token().id() == PythonTokenId.WHITESPACE
          && ts.movePrevious()) {
        // See if this was "from foo import bar" such that we really should
        // be listing symbols inside the foo library
        token = ts.token();
        id = token.id();
        String library = null;
        while (true) {
          if (id == PythonTokenId.IDENTIFIER
              || id.primaryCategory().equals(PythonLexer.KEYWORD_CAT)) {
            // Possibly inside the import string
            String tokenText = token.text().toString();
            if (library == null) {
              library = tokenText;
            } else {
              library = tokenText + "." + library;
            }
          } else if (id != PythonTokenId.DOT) {
            break;
          }
          if (!ts.movePrevious()) {
            return DeclarationLocation.NONE;
          }
          token = ts.token();
          id = token.id();
        }
        if (library != null) {
          if (id == PythonTokenId.WHITESPACE
              && ts.movePrevious()
              && ts.token().id() == PythonTokenId.FROM) {
            return findImport(info, library, moduleName);
          }
        }
      }

      return findImport(info, moduleName, null);
    }

    return DeclarationLocation.NONE;
  }
  public void run(RubyRuleContext context, List<Hint> result) {
    Node node = context.node;
    ParserResult info = context.parserResult;

    IfNode ifNode = (IfNode) node;
    if (ifNode.getCondition() == null) {
      // Can happen for this code:
      //   if ()
      //   end
      // (typically while editing)
      return;
    }
    Node body = ifNode.getThenBody();
    Node elseNode = ifNode.getElseBody();

    if (body != null && elseNode != null) {
      // Can't convert if-then-else conditionals
      return;
    }

    if (body == null && elseNode == null) {
      // Can't convert empty conditions
      return;
    }

    // Can't convert if !x/elseif blocks
    if (ifNode.getElseBody() != null && ifNode.getElseBody().getNodeType() == NodeType.IFNODE) {
      return;
    }

    int start = ifNode.getPosition().getStartOffset();
    if (!RubyHints.isNullOrInvisible(body)
        && (
        // Can't convert blocks with multiple statements
        body.getNodeType() == NodeType.BLOCKNODE
            ||
            // Already a statement modifier?
            body.getPosition().getStartOffset() <= start)) {
      return;
    } else if (!RubyHints.isNullOrInvisible(elseNode)
        && (elseNode.getNodeType() == NodeType.BLOCKNODE
            || elseNode.getPosition().getStartOffset() <= start)) {
      return;
    }

    BaseDocument doc = context.doc;
    try {
      int keywordOffset = ConvertIfToUnless.findKeywordOffset(context, ifNode);
      if (keywordOffset == -1 || keywordOffset > doc.getLength() - 1) {
        return;
      }

      char k = doc.getText(keywordOffset, 1).charAt(0);
      if (!(k == 'i' || k == 'u')) {
        return; // Probably ternary operator, ?:
      }
    } catch (BadLocationException ble) {
      Exceptions.printStackTrace(ble);
    }

    // If statement that is not already a statement modifier
    OffsetRange range = AstUtilities.getRange(node);

    if (RubyUtils.isRhtmlDocument(doc) || RubyUtils.isYamlDocument(doc)) {
      // Make sure that we're in a single contiguous Ruby section; if not, this won't work
      range = LexUtilities.getLexerOffsets(info, range);
      if (range == OffsetRange.NONE) {
        return;
      }

      try {
        doc.readLock();
        TokenHierarchy th = TokenHierarchy.get(doc);
        TokenSequence ts = th.tokenSequence();
        ts.move(range.getStart());
        if (!ts.moveNext() && !ts.movePrevious()) {
          return;
        }

        if (ts.offset() + ts.token().length() < range.getEnd()) {
          return;
        }
      } finally {
        doc.readUnlock();
      }
    }

    ConvertToModifier fix = new ConvertToModifier(context, ifNode);

    if (fix.getEditList() == null) {
      return;
    }

    List<HintFix> fixes = Collections.<HintFix>singletonList(fix);

    String displayName = NbBundle.getMessage(ConvertConditionals.class, "ConvertConditionals");
    Hint desc = new Hint(this, displayName, RubyUtils.getFileObject(info), range, fixes, 500);
    result.add(desc);
  }