@Test
public void testIgnore() throws Throwable {
JavaTokenizer tokenizer = new JavaTokenizer();
tokenizer.setIgnoreLiterals(true);
tokenizer.setIgnoreIdentifiers(true);
SourceCode sourceCode =
new SourceCode(new SourceCode.StringCodeLoader(getSampleCode(), "Foo.java"));
Tokens tokens = new Tokens();
TokenEntry.clearImages();
tokenizer.tokenize(sourceCode, tokens);
Map<String, SourceCode> codeMap = new HashMap<String, SourceCode>();
codeMap.put("Foo.java", sourceCode);
MatchAlgorithm matchAlgorithm = new MatchAlgorithm(codeMap, tokens, 5);
matchAlgorithm.findMatches();
Iterator matches = matchAlgorithm.matches();
Match match = (Match) matches.next();
assertFalse(matches.hasNext());
Iterator marks = match.iterator();
marks.next();
marks.next();
marks.next();
assertFalse(marks.hasNext());
}
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder stringBuilder = sourceCode.getCodeBuffer();
// Note that Java version is irrelevant for tokenizing
LanguageVersionHandler languageVersionHandler =
LanguageVersion.JAVA_14.getLanguageVersionHandler();
String fileName = sourceCode.getFileName();
TokenManager tokenMgr =
languageVersionHandler
.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(fileName, new StringReader(stringBuilder.toString()));
Token currentToken = (Token) tokenMgr.getNextToken();
TokenDiscarder discarder = new TokenDiscarder(ignoreAnnotations);
while (currentToken.image.length() > 0) {
discarder.updateState(currentToken);
if (discarder.isDiscarding()) {
currentToken = (Token) tokenMgr.getNextToken();
continue;
}
processToken(tokenEntries, fileName, currentToken);
currentToken = (Token) tokenMgr.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
}
@Test
public void testSimple() throws Throwable {
JavaTokenizer tokenizer = new JavaTokenizer();
SourceCode sourceCode =
new SourceCode(new SourceCode.StringCodeLoader(getSampleCode(), "Foo.java"));
Tokens tokens = new Tokens();
TokenEntry.clearImages();
tokenizer.tokenize(sourceCode, tokens);
assertEquals(41, tokens.size());
Map<String, SourceCode> codeMap = new HashMap<String, SourceCode>();
codeMap.put("Foo.java", sourceCode);
MatchAlgorithm matchAlgorithm = new MatchAlgorithm(codeMap, tokens, 5);
matchAlgorithm.findMatches();
Iterator matches = matchAlgorithm.matches();
Match match = (Match) matches.next();
assertFalse(matches.hasNext());
Iterator marks = match.iterator();
TokenEntry mark1 = (TokenEntry) marks.next();
TokenEntry mark2 = (TokenEntry) marks.next();
assertFalse(marks.hasNext());
assertEquals(3, mark1.getBeginLine());
assertEquals(4, mark2.getBeginLine());
assertTrue("Foo.java" == mark1.getTokenSrcID() && "Foo.java" == mark2.getTokenSrcID());
assertEquals(LINE_3, match.getSourceCodeSlice());
}
@Test
public void testDuplicationOnSameFile() throws IOException {
TokenEntry.clearImages();
AbstractLanguage cpdLanguage = new AbstractLanguage(new WebCpdTokenizer()) {};
CPD cpd = new CPD(30, cpdLanguage);
cpd.setEncoding(Charset.defaultCharset().name());
cpd.setLoadSourceCodeSlices(false);
cpd.add(new File("src/test/resources/duplications/fileWithDuplications.jsp"));
cpd.go();
List<Match> matches = getMatches(cpd);
assertThat(matches.size(), is(1));
org.sonar.duplications.cpd.Match match = matches.get(0);
assertThat(match.getLineCount(), is(16));
assertThat(match.getFirstMark().getBeginLine(), is(227));
assertThat(match.getSourceCodeSlice(), is(nullValue()));
}
@Override
public final void tokenize(SourceCode source, Tokens cpdTokens) {
String fileName = source.getFileName();
Token token;
TokenStream tokenStream;
try {
tokenStream = new GroovyLexer(new FileReader(new File(fileName))).plumb();
token = tokenStream.nextToken();
while (token.getType() != Token.EOF_TYPE) {
cpdTokens.add(new TokenEntry(token.getText(), fileName, token.getLine()));
token = tokenStream.nextToken();
}
} catch (TokenStreamException tse) {
LOG.error("Unexpected token when lexing file : " + fileName, tse);
} catch (FileNotFoundException fnfe) {
LOG.error("Could not find : " + fileName, fnfe);
}
cpdTokens.add(TokenEntry.getEOF());
}
public final void tokenize(SourceCode source, Tokens cpdTokens) {
String fileName = source.getFileName();
Token token;
ES3Lexer lexer;
try {
lexer = new ES3Lexer(new ANTLRFileStream(fileName));
token = lexer.nextToken();
while (token.getType() != Token.EOF) {
cpdTokens.add(new TokenEntry(token.getText(), fileName, token.getLine()));
token = lexer.nextToken();
}
} catch (FileNotFoundException fnfe) {
LOG.error("File not found", fnfe);
} catch (IOException e) {
LOG.error("Cannot read file", e);
}
cpdTokens.add(TokenEntry.getEOF());
}
public int compareTo(TokenEntry other) {
return getIndex() - other.getIndex();
}