From 24a1bedf765fd2dab8384073c9ac930563fa0110 Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Fri, 24 Feb 2017 17:45:16 +0000 Subject: [Preprocessor] Fix incorrect token caching that occurs when lexing _Pragma in macro argument pre-expansion mode when skipping a function body This commit fixes a token caching problem that currently occurs when clang is skipping a function body (e.g. when looking for a code completion token) and at the same time caching the tokens for _Pragma when lexing it in macro argument pre-expansion mode. When _Pragma is being lexed in macro argument pre-expansion mode, it caches the tokens so that it can avoid interpreting the pragma immediately (as the macro argument may not be used in the macro body), and then either backtracks over or commits these tokens. The problem is that, when we're backtracking/committing in such a scenario, there's already a previous backtracking position stored in BacktrackPositions (as we're skipping the function body), and this leads to a situation where the cached tokens from the pragma (like '(' 'string_literal' and ')') will remain in the cached tokens array incorrectly even after they're consumed (in the case of backtracking) or just ignored (in the case when they're committed). Furthermore, what makes it even worse, is that because of a previous backtracking position, the logic that deals with when should we call ExitCachingLexMode in CachingLex no longer works for us in this situation, and more tokens in the macro argument get cached, to the point where the EOF token that corresponds to the macro argument EOF is cached. This problem leads to all sorts of issues in code completion mode, where incorrect errors get presented and code completion completely fails to produce completion results. rdar://28523863 Differential Revision: https://reviews.llvm.org/D28772 llvm-svn: 296140 --- clang/lib/Lex/Pragma.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'clang/lib/Lex/Pragma.cpp') diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp index 100da514144..f81eaa31e9e 100644 --- a/clang/lib/Lex/Pragma.cpp +++ b/clang/lib/Lex/Pragma.cpp @@ -160,12 +160,23 @@ public: ~LexingFor_PragmaRAII() { if (InMacroArgPreExpansion) { + // When committing/backtracking the cached pragma tokens in a macro + // argument pre-expansion we want to ensure that either the tokens which + // have been committed will be removed from the cache or that the tokens + // over which we just backtracked won't remain in the cache after they're + // consumed and that the caching will stop after consuming them. + // Otherwise the caching will interfere with the way macro expansion + // works, because we will continue to cache tokens after consuming the + // backtracked tokens, which shouldn't happen when we're dealing with + // macro argument pre-expansion. + auto CachedTokenRange = PP.LastCachedTokenRange(); if (Failed) { PP.CommitBacktrackedTokens(); } else { PP.Backtrack(); OutTok = PragmaTok; } + PP.EraseCachedTokens(CachedTokenRange); } } -- cgit v1.2.3