예제 #1
0
  @Override
  public Item item(final QueryContext qc, final InputInfo ii) throws QueryException {
    checkCreate(qc);

    final Path path = toPath(0, qc);
    final B64 archive = toB64(exprs[1], qc, false);
    final TokenSet hs = entries(2, qc);

    try (ArchiveIn in = ArchiveIn.get(archive.input(info), info)) {
      while (in.more()) {
        final ZipEntry ze = in.entry();
        final String name = ze.getName();
        if (hs == null || hs.delete(token(name)) != 0) {
          final Path file = path.resolve(name);
          if (ze.isDirectory()) {
            Files.createDirectories(file);
          } else {
            Files.createDirectories(file.getParent());
            Files.write(file, in.read());
          }
        }
      }
    } catch (final IOException ex) {
      throw ARCH_FAIL_X.get(info, ex);
    }
    return null;
  }
예제 #2
0
  /**
   * Extracts entries from the archive.
   *
   * @param ctx query context
   * @return text entries
   * @throws QueryException query exception
   */
  private TokenList extract(final QueryContext ctx) throws QueryException {
    final B64 archive = (B64) checkType(checkItem(expr[0], ctx), AtomType.B64);
    TokenSet hs = null;
    if (expr.length > 1) {
      // filter result to specified entries
      hs = new TokenSet();
      final Iter names = ctx.iter(expr[1]);
      for (Item en; (en = names.next()) != null; ) {
        hs.add(checkElmStr(en).string(info));
      }
    }

    final TokenList tl = new TokenList();
    final ArchiveIn in = ArchiveIn.get(archive.input(info), info);
    try {
      while (in.more()) {
        final ZipEntry ze = in.entry();
        if (ze.isDirectory()) continue;
        if (hs == null || hs.delete(token(ze.getName())) != 0) tl.add(in.read());
      }
    } catch (final IOException ex) {
      Util.debug(ex);
      ARCH_FAIL.thrw(info, ex);
    } finally {
      in.close();
    }
    return tl;
  }
예제 #3
0
  @Override
  public byte[] info(final MainOptions options) {
    final TokenBuilder tb = new TokenBuilder();
    tb.add(LI_STRUCTURE).add(HASH).add(NL);
    tb.add(LI_NAMES).add(data.meta.names(type)).add(NL);

    final IndexStats stats = new IndexStats(options.get(MainOptions.MAXSTAT));
    final int s = values.size();
    for (int p = 1; p <= s; p++) {
      final int oc = lenList.get(p);
      if (oc > 0 && stats.adding(oc)) stats.add(values.key(p), oc);
    }
    stats.print(tb);
    return tb.finish();
  }
예제 #4
0
  @Override
  public IndexIterator iter(final IndexToken token) {
    final int id = values.id(token.get());
    if (id == 0) return IndexIterator.EMPTY;

    final int len = lenList.get(id);
    final int[] ids = idsList.get(id), pres;
    if (data.meta.updindex) {
      final IntList tmp = new IntList();
      for (int i = 0; i < len; ++i) tmp.add(data.pre(ids[i]));
      pres = tmp.sort().finish();
    } else {
      pres = ids;
    }

    return new IndexIterator() {
      int p;

      @Override
      public boolean more() {
        return p < len;
      }

      @Override
      public int pre() {
        return pres[p++];
      }

      @Override
      public int size() {
        return len;
      }
    };
  }
예제 #5
0
 @Override
 public void rehash() {
   super.rehash();
   final int s = size << 1;
   ids = Array.copyOf(ids, s);
   len = Arrays.copyOf(len, s);
 }
예제 #6
0
 /**
  * Constructor.
  *
  * @param data data instance
  * @param type index type
  */
 public MemValues(final Data data, final IndexType type) {
   super(data, type);
   // token index: work extra token set instance
   values =
       type == IndexType.TOKEN ? new TokenSet() : ((MemData) data).values(type == IndexType.TEXT);
   final int s = values.size() + 1;
   idsList = new ArrayList<>(s);
   lenList = new IntList(s);
   reorder = new BoolList(s);
 }
예제 #7
0
 /**
  * Removes values from the index.
  *
  * @param key key
  * @param vals sorted values
  */
 void delete(final byte[] key, final int... vals) {
   final int id = values.id(key), vl = vals.length, l = lenList.get(id), s = l - vl;
   final int[] ids = idsList.get(id);
   for (int i = 0, n = 0, v = 0; i < l; i++) {
     if (v == vl || ids[i] != vals[v]) ids[n++] = ids[i];
     else v++;
   }
   lenList.set(id, s);
   if (s == 0) idsList.set(id, null);
 }
예제 #8
0
  /**
   * Adds values to the index.
   *
   * @param key key to be indexed
   * @param vals sorted values
   */
  void add(final byte[] key, final int... vals) {
    // token index: add values. otherwise, reference existing values
    final int id = type == IndexType.TOKEN ? values.put(key) : values.id(key), vl = vals.length;
    // updatable index: if required, resize existing arrays
    while (idsList.size() < id + 1) idsList.add(null);
    if (lenList.size() < id + 1) lenList.set(id, 0);

    final int len = lenList.get(id), size = len + vl;
    int[] ids = idsList.get(id);
    if (ids == null) {
      ids = vals;
    } else {
      if (ids.length < size) ids = Arrays.copyOf(ids, Array.newSize(size));
      System.arraycopy(vals, 0, ids, len, vl);
      if (ids[len - 1] > vals[0]) {
        if (reorder == null) reorder = new BoolList(values.size());
        reorder.set(id, true);
      }
    }
    idsList.set(id, ids);
    lenList.set(id, size);
  }
예제 #9
0
파일: FTWords.java 프로젝트: jefferya/basex
 /**
  * Caches and returns all unique tokens specified in a query.
  *
  * @param list token list
  * @return token set
  */
 private TokenSet unique(final TokenList list) {
   // cache all query tokens in a set (duplicates are removed)
   final TokenSet ts = new TokenSet();
   switch (mode) {
     case ALL:
     case ANY:
       for (final byte[] t : list) ts.add(t);
       break;
     case ALL_WORDS:
     case ANY_WORD:
       final FTLexer l = new FTLexer(ftt.opt);
       for (final byte[] t : list) {
         l.init(t);
         while (l.hasNext()) ts.add(l.nextToken());
       }
       break;
     case PHRASE:
       final TokenBuilder tb = new TokenBuilder();
       for (final byte[] t : list) tb.add(t).add(' ');
       ts.add(tb.trim().finish());
   }
   return ts;
 }
예제 #10
0
 @Override
 protected void rehash() {
   super.rehash();
   funcs = Arrays.copyOf(funcs, size << 1);
 }
예제 #11
0
 @Override
 public int costs(final IndexToken it) {
   return lenList.get(values.id(it.get()));
 }
예제 #12
0
 @Override
 protected void rehash() {
   super.rehash();
   values = Arrays.copyOf(values, size << 1);
 }
public class PsiBuilderQuickTest extends LightPlatformTestCase {
  private static final IFileElementType ROOT = new IFileElementType("ROOT", Language.ANY);

  private static final IElementType LETTER = new IElementType("LETTER", Language.ANY);
  private static final IElementType DIGIT = new IElementType("DIGIT", Language.ANY);
  private static final IElementType OTHER = new IElementType("OTHER", Language.ANY);
  private static final IElementType COLLAPSED = new IElementType("COLLAPSED", Language.ANY);
  private static final IElementType LEFT_BOUND =
      new IElementType("LEFT_BOUND", Language.ANY) {
        @Override
        public boolean isLeftBound() {
          return true;
        }
      };
  private static final IElementType COMMENT = new IElementType("COMMENT", Language.ANY);

  private static final TokenSet WHITESPACE_SET = TokenSet.create(TokenType.WHITE_SPACE);
  private static final TokenSet COMMENT_SET = TokenSet.create(COMMENT);

  @SuppressWarnings("JUnitTestCaseWithNonTrivialConstructors")
  public PsiBuilderQuickTest() {
    PlatformTestCase.initPlatformLangPrefix();
  }

  public void testPlain() {
    doTest(
        "a<<b",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            while (builder.getTokenType() != null) {
              builder.advanceLexer();
            }
          }
        },
        "Element(ROOT)\n"
            + "  PsiElement(LETTER)('a')\n"
            + "  PsiElement(OTHER)('<')\n"
            + "  PsiElement(OTHER)('<')\n"
            + "  PsiElement(LETTER)('b')\n");
  }

  public void testComposites() {
    doTest(
        "1(a(b)c)2(d)3",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            PsiBuilderUtil.advance(builder, 1);
            final PsiBuilder.Marker marker1 = builder.mark();
            PsiBuilderUtil.advance(builder, 2);
            final PsiBuilder.Marker marker2 = builder.mark();
            PsiBuilderUtil.advance(builder, 3);
            marker2.done(OTHER);
            PsiBuilderUtil.advance(builder, 2);
            marker1.done(OTHER);
            PsiBuilderUtil.advance(builder, 1);
            final PsiBuilder.Marker marker3 = builder.mark();
            PsiBuilderUtil.advance(builder, 1);
            builder.mark().done(OTHER);
            PsiBuilderUtil.advance(builder, 2);
            marker3.done(OTHER);
            PsiBuilderUtil.advance(builder, 1);
          }
        },
        "Element(ROOT)\n"
            + "  PsiElement(DIGIT)('1')\n"
            + "  Element(OTHER)\n"
            + "    PsiElement(OTHER)('(')\n"
            + "    PsiElement(LETTER)('a')\n"
            + "    Element(OTHER)\n"
            + "      PsiElement(OTHER)('(')\n"
            + "      PsiElement(LETTER)('b')\n"
            + "      PsiElement(OTHER)(')')\n"
            + "    PsiElement(LETTER)('c')\n"
            + "    PsiElement(OTHER)(')')\n"
            + "  PsiElement(DIGIT)('2')\n"
            + "  Element(OTHER)\n"
            + "    PsiElement(OTHER)('(')\n"
            + "    Element(OTHER)\n"
            + "      <empty list>\n"
            + "    PsiElement(LETTER)('d')\n"
            + "    PsiElement(OTHER)(')')\n"
            + "  PsiElement(DIGIT)('3')\n");
  }

  public void testCollapse() {
    doTest(
        "a<<>>b",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            PsiBuilderUtil.advance(builder, 1);
            final PsiBuilder.Marker marker1 = builder.mark();
            PsiBuilderUtil.advance(builder, 2);
            marker1.collapse(COLLAPSED);
            final PsiBuilder.Marker marker2 = builder.mark();
            PsiBuilderUtil.advance(builder, 2);
            marker2.collapse(COLLAPSED);
            PsiBuilderUtil.advance(builder, 1);
          }
        },
        "Element(ROOT)\n"
            + "  PsiElement(LETTER)('a')\n"
            + "  PsiElement(COLLAPSED)('<<')\n"
            + "  PsiElement(COLLAPSED)('>>')\n"
            + "  PsiElement(LETTER)('b')\n");
  }

  public void testDoneAndError() {
    doTest(
        "a2b",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            IElementType tokenType;
            while ((tokenType = builder.getTokenType()) != null) {
              final PsiBuilder.Marker marker = builder.mark();
              builder.advanceLexer();
              if (tokenType == DIGIT) marker.error("no digits allowed");
              else marker.done(tokenType);
            }
          }
        },
        "Element(ROOT)\n"
            + "  Element(LETTER)\n"
            + "    PsiElement(LETTER)('a')\n"
            + "  PsiErrorElement:no digits allowed\n"
            + "    PsiElement(DIGIT)('2')\n"
            + "  Element(LETTER)\n"
            + "    PsiElement(LETTER)('b')\n");
  }

  public void testPrecedeAndDoneBefore() {
    doTest(
        "ab",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            final PsiBuilder.Marker marker1 = builder.mark();
            builder.advanceLexer();
            final PsiBuilder.Marker marker2 = builder.mark();
            builder.advanceLexer();
            marker2.done(OTHER);
            marker2.precede().doneBefore(COLLAPSED, marker2);
            marker1.doneBefore(COLLAPSED, marker2, "with error");
          }
        },
        "Element(ROOT)\n"
            + "  Element(COLLAPSED)\n"
            + "    PsiElement(LETTER)('a')\n"
            + "    Element(COLLAPSED)\n"
            + "      <empty list>\n"
            + "    PsiErrorElement:with error\n"
            + "      <empty list>\n"
            + "  Element(OTHER)\n"
            + "    PsiElement(LETTER)('b')\n");
  }

  public void testErrorBefore() {
    doTest(
        "a1",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            final PsiBuilder.Marker letter = builder.mark();
            builder.advanceLexer();
            letter.done(LETTER);
            final PsiBuilder.Marker digit = builder.mark();
            builder.advanceLexer();
            digit.done(DIGIT);
            digit.precede().errorBefore("something lost", digit);
          }
        },
        "Element(ROOT)\n"
            + "  Element(LETTER)\n"
            + "    PsiElement(LETTER)('a')\n"
            + "  PsiErrorElement:something lost\n"
            + "    <empty list>\n"
            + "  Element(DIGIT)\n"
            + "    PsiElement(DIGIT)('1')\n");
  }

  public void testValidityChecksOnDone() {
    doFailTest(
        "a",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            final PsiBuilder.Marker first = builder.mark();
            builder.advanceLexer();
            builder.mark();
            first.done(LETTER);
          }
        },
        "Another not done marker added after this one. Must be done before this.");
  }

  public void testValidityChecksOnDoneBefore1() {
    doFailTest(
        "a",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            final PsiBuilder.Marker first = builder.mark();
            builder.advanceLexer();
            final PsiBuilder.Marker second = builder.mark();
            second.precede();
            first.doneBefore(LETTER, second);
          }
        },
        "Another not done marker added after this one. Must be done before this.");
  }

  public void testValidityChecksOnDoneBefore2() {
    doFailTest(
        "a",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            final PsiBuilder.Marker first = builder.mark();
            builder.advanceLexer();
            final PsiBuilder.Marker second = builder.mark();
            second.doneBefore(LETTER, first);
          }
        },
        "'Before' marker precedes this one.");
  }

  public void testValidityChecksOnTreeBuild1() {
    doFailTest(
        "aa",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            while (!builder.eof()) builder.advanceLexer();
          }
        },
        "Parser produced no markers. Text:\naa");
  }

  public void testValidityChecksOnTreeBuild2() {
    doFailTest(
        "aa",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            final PsiBuilder.Marker marker = builder.mark();
            builder.advanceLexer();
            marker.done(LETTER);
          }
        },
        "Tokens [LETTER] were not inserted into the tree. Text:\naa");
  }

  public void testValidityChecksOnTreeBuild3() {
    doFailTest(
        "a ",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            final PsiBuilder.Marker marker = builder.mark();
            builder.advanceLexer();
            marker.done(LETTER);
            while (!builder.eof()) builder.advanceLexer();
          }
        },
        "Tokens [WHITE_SPACE] are outside of root element \"LETTER\". Text:\na ");
  }

  public void testWhitespaceTrimming() {
    doTest(
        " a b ",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            PsiBuilder.Marker marker = builder.mark();
            builder.advanceLexer();
            marker.done(OTHER);
            marker = builder.mark();
            builder.advanceLexer();
            marker.done(OTHER);
            builder.advanceLexer();
          }
        },
        "Element(ROOT)\n"
            + "  PsiWhiteSpace(' ')\n"
            + "  Element(OTHER)\n"
            + "    PsiElement(LETTER)('a')\n"
            + "  PsiWhiteSpace(' ')\n"
            + "  Element(OTHER)\n"
            + "    PsiElement(LETTER)('b')\n"
            + "  PsiWhiteSpace(' ')\n");
  }

  public void testWhitespaceBalancingByErrors() {
    doTest(
        "a b c",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            PsiBuilder.Marker marker = builder.mark();
            builder.advanceLexer();
            builder.error("error 1");
            marker.done(OTHER);
            marker = builder.mark();
            builder.advanceLexer();
            builder.mark().error("error 2");
            marker.done(OTHER);
            marker = builder.mark();
            builder.advanceLexer();
            marker.error("error 3");
          }
        },
        "Element(ROOT)\n"
            + "  Element(OTHER)\n"
            + "    PsiElement(LETTER)('a')\n"
            + "    PsiErrorElement:error 1\n"
            + "      <empty list>\n"
            + "  PsiWhiteSpace(' ')\n"
            + "  Element(OTHER)\n"
            + "    PsiElement(LETTER)('b')\n"
            + "    PsiErrorElement:error 2\n"
            + "      <empty list>\n"
            + "  PsiWhiteSpace(' ')\n"
            + "  PsiErrorElement:error 3\n"
            + "    PsiElement(LETTER)('c')\n");
  }

  public void testWhitespaceBalancingByEmptyComposites() {
    doTest(
        "a b c",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            PsiBuilder.Marker marker = builder.mark();
            builder.advanceLexer();
            builder.mark().done(OTHER);
            marker.done(OTHER);
            marker = builder.mark();
            builder.advanceLexer();
            builder.mark().done(LEFT_BOUND);
            marker.done(OTHER);
            builder.advanceLexer();
          }
        },
        "Element(ROOT)\n"
            + "  Element(OTHER)\n"
            + "    PsiElement(LETTER)('a')\n"
            + "    PsiWhiteSpace(' ')\n"
            + "    Element(OTHER)\n"
            + "      <empty list>\n"
            + "  Element(OTHER)\n"
            + "    PsiElement(LETTER)('b')\n"
            + "    Element(LEFT_BOUND)\n"
            + "      <empty list>\n"
            + "  PsiWhiteSpace(' ')\n"
            + "  PsiElement(LETTER)('c')\n");
  }

  public void testCustomEdgeProcessors() {
    final WhitespacesAndCommentsBinder leftEdgeProcessor =
        new WhitespacesAndCommentsBinder() {
          @Override
          public int getEdgePosition(
              List<IElementType> tokens, boolean atStreamEdge, TokenTextGetter getter) {
            int pos = tokens.size() - 1;
            while (tokens.get(pos) != COMMENT && pos > 0) pos--;
            return pos;
          }
        };
    final WhitespacesAndCommentsBinder rightEdgeProcessor =
        new WhitespacesAndCommentsBinder() {
          @Override
          public int getEdgePosition(
              List<IElementType> tokens, boolean atStreamEdge, TokenTextGetter getter) {
            int pos = 0;
            while (tokens.get(pos) != COMMENT && pos < tokens.size() - 1) pos++;
            return pos + 1;
          }
        };

    doTest(
        "{ # i # }",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            while (builder.getTokenType() != LETTER) builder.advanceLexer();
            final PsiBuilder.Marker marker = builder.mark();
            builder.advanceLexer();
            marker.done(OTHER);
            marker.setCustomEdgeTokenBinders(leftEdgeProcessor, rightEdgeProcessor);
            while (builder.getTokenType() != null) builder.advanceLexer();
          }
        },
        "Element(ROOT)\n"
            + "  PsiElement(OTHER)('{')\n"
            + "  PsiWhiteSpace(' ')\n"
            + "  Element(OTHER)\n"
            + "    PsiElement(COMMENT)('#')\n"
            + "    PsiWhiteSpace(' ')\n"
            + "    PsiElement(LETTER)('i')\n"
            + "    PsiWhiteSpace(' ')\n"
            + "    PsiElement(COMMENT)('#')\n"
            + "  PsiWhiteSpace(' ')\n"
            + "  PsiElement(OTHER)('}')\n");
  }

  private abstract static class MyLazyElementType extends ILazyParseableElementType
      implements ILightLazyParseableElementType {
    protected MyLazyElementType(@NonNls String debugName) {
      super(debugName, Language.ANY);
    }
  }

  public void testLightChameleon() {
    final IElementType CHAMELEON_2 = new MyChameleon2Type();

    final IElementType CHAMELEON_1 = new MyChameleon1Type(CHAMELEON_2);

    doTest(
        "ab{12[.?]}cd{x}",
        new Parser() {
          @Override
          public void parse(PsiBuilder builder) {
            PsiBuilderUtil.advance(builder, 2);
            PsiBuilder.Marker chameleon = builder.mark();
            PsiBuilderUtil.advance(builder, 8);
            chameleon.collapse(CHAMELEON_1);
            PsiBuilderUtil.advance(builder, 2);
            chameleon = builder.mark();
            PsiBuilderUtil.advance(builder, 3);
            chameleon.collapse(CHAMELEON_1);
          }
        },
        "Element(ROOT)\n"
            + "  PsiElement(LETTER)('a')\n"
            + "  PsiElement(LETTER)('b')\n"
            + "  Element(CHAMELEON_1)\n"
            + "    PsiElement(OTHER)('{')\n"
            + "    PsiElement(DIGIT)('1')\n"
            + "    PsiElement(DIGIT)('2')\n"
            + "    Element(OTHER)\n"
            + "      Element(CHAMELEON_2)\n"
            + "        PsiElement(OTHER)('[')\n"
            + "        PsiElement(OTHER)('.')\n"
            + "        PsiErrorElement:test error 2\n"
            + "          PsiElement(OTHER)('?')\n"
            + "        PsiElement(OTHER)(']')\n"
            + "    PsiErrorElement:test error 1\n"
            + "      <empty list>\n"
            + "    PsiElement(OTHER)('}')\n"
            + "  PsiElement(LETTER)('c')\n"
            + "  PsiElement(LETTER)('d')\n"
            + "  Element(CHAMELEON_1)\n"
            + "    PsiElement(OTHER)('{')\n"
            + "    PsiElement(LETTER)('x')\n"
            + "    PsiElement(OTHER)('}')\n");
  }

  @SuppressWarnings("ConstantConditions")
  private static PsiBuilderImpl createBuilder(CharSequence text) {
    ParserDefinition parserDefinition =
        new ParserDefinition() {
          @NotNull
          @Override
          public Lexer createLexer(Project project) {
            return new MyTestLexer();
          }

          @Override
          public PsiParser createParser(Project project) {
            return null;
          }

          @Override
          public IFileElementType getFileNodeType() {
            return null;
          }

          @NotNull
          @Override
          public TokenSet getWhitespaceTokens() {
            return WHITESPACE_SET;
          }

          @NotNull
          @Override
          public TokenSet getCommentTokens() {
            return COMMENT_SET;
          }

          @NotNull
          @Override
          public TokenSet getStringLiteralElements() {
            return null;
          }

          @NotNull
          @Override
          public PsiElement createElement(ASTNode node) {
            return null;
          }

          @Override
          public PsiFile createFile(FileViewProvider viewProvider) {
            return null;
          }

          @Override
          public SpaceRequirements spaceExistanceTypeBetweenTokens(ASTNode left, ASTNode right) {
            return null;
          }
        };
    return new PsiBuilderImpl(
        getProject(),
        null,
        parserDefinition,
        parserDefinition.createLexer(getProject()),
        null,
        text,
        null,
        null);
  }

  private interface Parser {
    void parse(PsiBuilder builder);
  }

  private static void doTest(
      @NonNls final String text, final Parser parser, @NonNls final String expected) {
    final PsiBuilder builder = createBuilder(text);
    final PsiBuilder.Marker rootMarker = builder.mark();
    parser.parse(builder);
    rootMarker.done(ROOT);

    // check light tree composition
    final FlyweightCapableTreeStructure<LighterASTNode> lightTree = builder.getLightTree();
    assertEquals(expected, DebugUtil.lightTreeToString(lightTree, false));
    // verify that light tree can be taken multiple times
    final FlyweightCapableTreeStructure<LighterASTNode> lightTree2 = builder.getLightTree();
    assertEquals(expected, DebugUtil.lightTreeToString(lightTree2, false));

    // check heavy tree composition
    final ASTNode root = builder.getTreeBuilt();
    assertEquals(expected, DebugUtil.nodeTreeToString(root, false));

    // check heavy vs. light tree merging
    final PsiBuilder builder2 = createBuilder(text);
    final PsiBuilder.Marker rootMarker2 = builder2.mark();
    parser.parse(builder2);
    rootMarker2.done(ROOT);
    DiffTree.diff(
        new ASTStructure(root),
        builder2.getLightTree(),
        new ShallowNodeComparator<ASTNode, LighterASTNode>() {
          @Override
          public ThreeState deepEqual(ASTNode oldNode, LighterASTNode newNode) {
            return ThreeState.UNSURE;
          }

          @Override
          public boolean typesEqual(ASTNode oldNode, LighterASTNode newNode) {
            return true;
          }

          @Override
          public boolean hashCodesEqual(ASTNode oldNode, LighterASTNode newNode) {
            return true;
          }
        },
        new DiffTreeChangeBuilder<ASTNode, LighterASTNode>() {
          @Override
          public void nodeReplaced(@NotNull ASTNode oldChild, @NotNull LighterASTNode newChild) {
            fail("replaced(" + oldChild + "," + newChild.getTokenType() + ")");
          }

          @Override
          public void nodeDeleted(@NotNull ASTNode oldParent, @NotNull ASTNode oldNode) {
            fail("deleted(" + oldParent + "," + oldNode + ")");
          }

          @Override
          public void nodeInserted(
              @NotNull ASTNode oldParent, @NotNull LighterASTNode newNode, int pos) {
            fail("inserted(" + oldParent + "," + newNode.getTokenType() + ")");
          }
        });
  }

  private static void doFailTest(
      @NonNls final String text, final Parser parser, @NonNls final String expected) {
    final PrintStream std = System.err;
    //noinspection IOResourceOpenedButNotSafelyClosed
    System.setErr(new PrintStream(new NullStream()));
    try {
      try {
        ParserDefinition parserDefinition =
            new ParserDefinition() {
              @NotNull
              @Override
              public Lexer createLexer(Project project) {
                return null;
              }

              @Override
              public PsiParser createParser(Project project) {
                return null;
              }

              @Override
              public IFileElementType getFileNodeType() {
                return null;
              }

              @NotNull
              @Override
              public TokenSet getWhitespaceTokens() {
                return TokenSet.EMPTY;
              }

              @NotNull
              @Override
              public TokenSet getCommentTokens() {
                return TokenSet.EMPTY;
              }

              @NotNull
              @Override
              public TokenSet getStringLiteralElements() {
                return null;
              }

              @NotNull
              @Override
              public PsiElement createElement(ASTNode node) {
                return null;
              }

              @Override
              public PsiFile createFile(FileViewProvider viewProvider) {
                return null;
              }

              @Override
              public SpaceRequirements spaceExistanceTypeBetweenTokens(
                  ASTNode left, ASTNode right) {
                return null;
              }
            };
        final PsiBuilder builder =
            PsiBuilderFactory.getInstance()
                .createBuilder(parserDefinition, new MyTestLexer(), text);
        builder.setDebugMode(true);
        parser.parse(builder);
        builder.getLightTree();
        fail("should fail");
      } catch (AssertionError e) {
        assertEquals(expected, e.getMessage());
      }
    } finally {
      System.setErr(std);
    }
  }

  private static class MyTestLexer extends LexerBase {
    private CharSequence myBuffer = "";
    private int myIndex = 0;
    private int myBufferEnd = 1;

    @Override
    public void start(CharSequence buffer, int startOffset, int endOffset, int initialState) {
      myBuffer = buffer.subSequence(startOffset, endOffset);
      myIndex = 0;
      myBufferEnd = myBuffer.length();
    }

    @Override
    public int getState() {
      return 0;
    }

    @Override
    public IElementType getTokenType() {
      if (myIndex >= myBufferEnd) return null;
      else if (Character.isLetter(myBuffer.charAt(myIndex))) return LETTER;
      else if (Character.isDigit(myBuffer.charAt(myIndex))) return DIGIT;
      else if (Character.isWhitespace(myBuffer.charAt(myIndex))) return TokenType.WHITE_SPACE;
      else if (myBuffer.charAt(myIndex) == '#') return COMMENT;
      else return OTHER;
    }

    @Override
    public int getTokenStart() {
      return myIndex;
    }

    @Override
    public int getTokenEnd() {
      return myIndex + 1;
    }

    @Override
    public void advance() {
      if (myIndex < myBufferEnd) myIndex++;
    }

    @Override
    public CharSequence getBufferSequence() {
      return myBuffer;
    }

    @Override
    public int getBufferEnd() {
      return myBufferEnd;
    }
  }

  private static class NullStream extends OutputStream {
    @Override
    public void write(final int b) throws IOException {}
  }

  private static class MyChameleon1Type extends MyLazyElementType {
    private final IElementType myCHAMELEON_2;

    public MyChameleon1Type(IElementType CHAMELEON_2) {
      super("CHAMELEON_1");
      myCHAMELEON_2 = CHAMELEON_2;
    }

    @Override
    public FlyweightCapableTreeStructure<LighterASTNode> parseContents(
        LighterLazyParseableNode chameleon) {
      final PsiBuilder builder = createBuilder(chameleon.getText());
      parse(builder);
      return builder.getLightTree();
    }

    @Override
    public ASTNode parseContents(ASTNode chameleon) {
      final PsiBuilder builder = createBuilder(chameleon.getText());
      parse(builder);
      return builder.getTreeBuilt().getFirstChildNode();
    }

    public void parse(PsiBuilder builder) {
      final PsiBuilder.Marker root = builder.mark();
      PsiBuilder.Marker nested = null;
      while (!builder.eof()) {
        final String token = builder.getTokenText();
        if ("[".equals(token) && nested == null) {
          nested = builder.mark();
        }
        builder.advanceLexer();
        if ("]".equals(token) && nested != null) {
          nested.collapse(myCHAMELEON_2);
          nested.precede().done(OTHER);
          nested = null;
          builder.error("test error 1");
        }
      }
      if (nested != null) nested.drop();
      root.done(this);
    }
  }

  private static class MyChameleon2Type extends MyLazyElementType {
    public MyChameleon2Type() {
      super("CHAMELEON_2");
    }

    @Override
    public FlyweightCapableTreeStructure<LighterASTNode> parseContents(
        LighterLazyParseableNode chameleon) {
      final PsiBuilder builder = createBuilder(chameleon.getText());
      parse(builder);
      return builder.getLightTree();
    }

    @Override
    public ASTNode parseContents(ASTNode chameleon) {
      final PsiBuilder builder = createBuilder(chameleon.getText());
      parse(builder);
      return builder.getTreeBuilt().getFirstChildNode();
    }

    public void parse(PsiBuilder builder) {
      final PsiBuilder.Marker root = builder.mark();
      PsiBuilder.Marker error = null;
      while (!builder.eof()) {
        final String token = builder.getTokenText();
        if ("?".equals(token)) error = builder.mark();
        builder.advanceLexer();
        if (error != null) {
          error.error("test error 2");
          error = null;
        }
      }
      root.done(this);
    }
  }
}