ac7a9ef0ae
Precommit builds cover Linux and Windows, but this ambiguity would only show up on Mac OS: there we have int32_t = int, int64_t = long long and size_t = unsigned long. So printing a size_t, while successful on the other two architectures, cannot be unambiguously resolved on Mac OS. This is not really meant to support printing arguments of type long or size_t, but more as a way to prevent build breakage that would not be detected in precommit builds, as happened in D125429. Technically we have no guarantee that one of these types has the 64 bits that afdac5fbcb6a3 wanted to provide, so proposals are welcome. We do have a guarantee though that these three types are different, so we should be fine with overload resolution. Reviewed By: aeubanks Differential Revision: https://reviews.llvm.org/D125580
770 lines
23 KiB
C++
770 lines
23 KiB
C++
//===--- CommentParser.cpp - Doxygen comment parser -----------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "clang/AST/CommentParser.h"
|
|
#include "clang/AST/CommentCommandTraits.h"
|
|
#include "clang/AST/CommentDiagnostic.h"
|
|
#include "clang/AST/CommentSema.h"
|
|
#include "clang/Basic/CharInfo.h"
|
|
#include "clang/Basic/SourceManager.h"
|
|
#include "llvm/Support/ErrorHandling.h"
|
|
|
|
namespace clang {
|
|
|
|
static inline bool isWhitespace(llvm::StringRef S) {
|
|
for (StringRef::const_iterator I = S.begin(), E = S.end(); I != E; ++I) {
|
|
if (!isWhitespace(*I))
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
namespace comments {
|
|
|
|
/// Re-lexes a sequence of tok::text tokens.
|
|
class TextTokenRetokenizer {
|
|
llvm::BumpPtrAllocator &Allocator;
|
|
Parser &P;
|
|
|
|
/// This flag is set when there are no more tokens we can fetch from lexer.
|
|
bool NoMoreInterestingTokens;
|
|
|
|
/// Token buffer: tokens we have processed and lookahead.
|
|
SmallVector<Token, 16> Toks;
|
|
|
|
/// A position in \c Toks.
|
|
struct Position {
|
|
const char *BufferStart;
|
|
const char *BufferEnd;
|
|
const char *BufferPtr;
|
|
SourceLocation BufferStartLoc;
|
|
unsigned CurToken;
|
|
};
|
|
|
|
/// Current position in Toks.
|
|
Position Pos;
|
|
|
|
bool isEnd() const {
|
|
return Pos.CurToken >= Toks.size();
|
|
}
|
|
|
|
/// Sets up the buffer pointers to point to current token.
|
|
void setupBuffer() {
|
|
assert(!isEnd());
|
|
const Token &Tok = Toks[Pos.CurToken];
|
|
|
|
Pos.BufferStart = Tok.getText().begin();
|
|
Pos.BufferEnd = Tok.getText().end();
|
|
Pos.BufferPtr = Pos.BufferStart;
|
|
Pos.BufferStartLoc = Tok.getLocation();
|
|
}
|
|
|
|
SourceLocation getSourceLocation() const {
|
|
const unsigned CharNo = Pos.BufferPtr - Pos.BufferStart;
|
|
return Pos.BufferStartLoc.getLocWithOffset(CharNo);
|
|
}
|
|
|
|
char peek() const {
|
|
assert(!isEnd());
|
|
assert(Pos.BufferPtr != Pos.BufferEnd);
|
|
return *Pos.BufferPtr;
|
|
}
|
|
|
|
void consumeChar() {
|
|
assert(!isEnd());
|
|
assert(Pos.BufferPtr != Pos.BufferEnd);
|
|
Pos.BufferPtr++;
|
|
if (Pos.BufferPtr == Pos.BufferEnd) {
|
|
Pos.CurToken++;
|
|
if (isEnd() && !addToken())
|
|
return;
|
|
|
|
assert(!isEnd());
|
|
setupBuffer();
|
|
}
|
|
}
|
|
|
|
/// Add a token.
|
|
/// Returns true on success, false if there are no interesting tokens to
|
|
/// fetch from lexer.
|
|
bool addToken() {
|
|
if (NoMoreInterestingTokens)
|
|
return false;
|
|
|
|
if (P.Tok.is(tok::newline)) {
|
|
// If we see a single newline token between text tokens, skip it.
|
|
Token Newline = P.Tok;
|
|
P.consumeToken();
|
|
if (P.Tok.isNot(tok::text)) {
|
|
P.putBack(Newline);
|
|
NoMoreInterestingTokens = true;
|
|
return false;
|
|
}
|
|
}
|
|
if (P.Tok.isNot(tok::text)) {
|
|
NoMoreInterestingTokens = true;
|
|
return false;
|
|
}
|
|
|
|
Toks.push_back(P.Tok);
|
|
P.consumeToken();
|
|
if (Toks.size() == 1)
|
|
setupBuffer();
|
|
return true;
|
|
}
|
|
|
|
void consumeWhitespace() {
|
|
while (!isEnd()) {
|
|
if (isWhitespace(peek()))
|
|
consumeChar();
|
|
else
|
|
break;
|
|
}
|
|
}
|
|
|
|
void formTokenWithChars(Token &Result,
|
|
SourceLocation Loc,
|
|
const char *TokBegin,
|
|
unsigned TokLength,
|
|
StringRef Text) {
|
|
Result.setLocation(Loc);
|
|
Result.setKind(tok::text);
|
|
Result.setLength(TokLength);
|
|
#ifndef NDEBUG
|
|
Result.TextPtr = "<UNSET>";
|
|
Result.IntVal = 7;
|
|
#endif
|
|
Result.setText(Text);
|
|
}
|
|
|
|
public:
|
|
TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P):
|
|
Allocator(Allocator), P(P), NoMoreInterestingTokens(false) {
|
|
Pos.CurToken = 0;
|
|
addToken();
|
|
}
|
|
|
|
/// Extract a word -- sequence of non-whitespace characters.
|
|
bool lexWord(Token &Tok) {
|
|
if (isEnd())
|
|
return false;
|
|
|
|
Position SavedPos = Pos;
|
|
|
|
consumeWhitespace();
|
|
SmallString<32> WordText;
|
|
const char *WordBegin = Pos.BufferPtr;
|
|
SourceLocation Loc = getSourceLocation();
|
|
while (!isEnd()) {
|
|
const char C = peek();
|
|
if (!isWhitespace(C)) {
|
|
WordText.push_back(C);
|
|
consumeChar();
|
|
} else
|
|
break;
|
|
}
|
|
const unsigned Length = WordText.size();
|
|
if (Length == 0) {
|
|
Pos = SavedPos;
|
|
return false;
|
|
}
|
|
|
|
char *TextPtr = Allocator.Allocate<char>(Length + 1);
|
|
|
|
memcpy(TextPtr, WordText.c_str(), Length + 1);
|
|
StringRef Text = StringRef(TextPtr, Length);
|
|
|
|
formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
|
|
return true;
|
|
}
|
|
|
|
bool lexDelimitedSeq(Token &Tok, char OpenDelim, char CloseDelim) {
|
|
if (isEnd())
|
|
return false;
|
|
|
|
Position SavedPos = Pos;
|
|
|
|
consumeWhitespace();
|
|
SmallString<32> WordText;
|
|
const char *WordBegin = Pos.BufferPtr;
|
|
SourceLocation Loc = getSourceLocation();
|
|
bool Error = false;
|
|
if (!isEnd()) {
|
|
const char C = peek();
|
|
if (C == OpenDelim) {
|
|
WordText.push_back(C);
|
|
consumeChar();
|
|
} else
|
|
Error = true;
|
|
}
|
|
char C = '\0';
|
|
while (!Error && !isEnd()) {
|
|
C = peek();
|
|
WordText.push_back(C);
|
|
consumeChar();
|
|
if (C == CloseDelim)
|
|
break;
|
|
}
|
|
if (!Error && C != CloseDelim)
|
|
Error = true;
|
|
|
|
if (Error) {
|
|
Pos = SavedPos;
|
|
return false;
|
|
}
|
|
|
|
const unsigned Length = WordText.size();
|
|
char *TextPtr = Allocator.Allocate<char>(Length + 1);
|
|
|
|
memcpy(TextPtr, WordText.c_str(), Length + 1);
|
|
StringRef Text = StringRef(TextPtr, Length);
|
|
|
|
formTokenWithChars(Tok, Loc, WordBegin,
|
|
Pos.BufferPtr - WordBegin, Text);
|
|
return true;
|
|
}
|
|
|
|
/// Put back tokens that we didn't consume.
|
|
void putBackLeftoverTokens() {
|
|
if (isEnd())
|
|
return;
|
|
|
|
bool HavePartialTok = false;
|
|
Token PartialTok;
|
|
if (Pos.BufferPtr != Pos.BufferStart) {
|
|
formTokenWithChars(PartialTok, getSourceLocation(),
|
|
Pos.BufferPtr, Pos.BufferEnd - Pos.BufferPtr,
|
|
StringRef(Pos.BufferPtr,
|
|
Pos.BufferEnd - Pos.BufferPtr));
|
|
HavePartialTok = true;
|
|
Pos.CurToken++;
|
|
}
|
|
|
|
P.putBack(llvm::makeArrayRef(Toks.begin() + Pos.CurToken, Toks.end()));
|
|
Pos.CurToken = Toks.size();
|
|
|
|
if (HavePartialTok)
|
|
P.putBack(PartialTok);
|
|
}
|
|
};
|
|
|
|
Parser::Parser(Lexer &L, Sema &S, llvm::BumpPtrAllocator &Allocator,
|
|
const SourceManager &SourceMgr, DiagnosticsEngine &Diags,
|
|
const CommandTraits &Traits):
|
|
L(L), S(S), Allocator(Allocator), SourceMgr(SourceMgr), Diags(Diags),
|
|
Traits(Traits) {
|
|
consumeToken();
|
|
}
|
|
|
|
void Parser::parseParamCommandArgs(ParamCommandComment *PC,
|
|
TextTokenRetokenizer &Retokenizer) {
|
|
Token Arg;
|
|
// Check if argument looks like direction specification: [dir]
|
|
// e.g., [in], [out], [in,out]
|
|
if (Retokenizer.lexDelimitedSeq(Arg, '[', ']'))
|
|
S.actOnParamCommandDirectionArg(PC,
|
|
Arg.getLocation(),
|
|
Arg.getEndLocation(),
|
|
Arg.getText());
|
|
|
|
if (Retokenizer.lexWord(Arg))
|
|
S.actOnParamCommandParamNameArg(PC,
|
|
Arg.getLocation(),
|
|
Arg.getEndLocation(),
|
|
Arg.getText());
|
|
}
|
|
|
|
void Parser::parseTParamCommandArgs(TParamCommandComment *TPC,
|
|
TextTokenRetokenizer &Retokenizer) {
|
|
Token Arg;
|
|
if (Retokenizer.lexWord(Arg))
|
|
S.actOnTParamCommandParamNameArg(TPC,
|
|
Arg.getLocation(),
|
|
Arg.getEndLocation(),
|
|
Arg.getText());
|
|
}
|
|
|
|
ArrayRef<Comment::Argument>
|
|
Parser::parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs) {
|
|
auto *Args = new (Allocator.Allocate<Comment::Argument>(NumArgs))
|
|
Comment::Argument[NumArgs];
|
|
unsigned ParsedArgs = 0;
|
|
Token Arg;
|
|
while (ParsedArgs < NumArgs && Retokenizer.lexWord(Arg)) {
|
|
Args[ParsedArgs] = Comment::Argument{
|
|
SourceRange(Arg.getLocation(), Arg.getEndLocation()), Arg.getText()};
|
|
ParsedArgs++;
|
|
}
|
|
|
|
return llvm::makeArrayRef(Args, ParsedArgs);
|
|
}
|
|
|
|
BlockCommandComment *Parser::parseBlockCommand() {
|
|
assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
|
|
|
|
ParamCommandComment *PC = nullptr;
|
|
TParamCommandComment *TPC = nullptr;
|
|
BlockCommandComment *BC = nullptr;
|
|
const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
|
|
CommandMarkerKind CommandMarker =
|
|
Tok.is(tok::backslash_command) ? CMK_Backslash : CMK_At;
|
|
if (Info->IsParamCommand) {
|
|
PC = S.actOnParamCommandStart(Tok.getLocation(),
|
|
Tok.getEndLocation(),
|
|
Tok.getCommandID(),
|
|
CommandMarker);
|
|
} else if (Info->IsTParamCommand) {
|
|
TPC = S.actOnTParamCommandStart(Tok.getLocation(),
|
|
Tok.getEndLocation(),
|
|
Tok.getCommandID(),
|
|
CommandMarker);
|
|
} else {
|
|
BC = S.actOnBlockCommandStart(Tok.getLocation(),
|
|
Tok.getEndLocation(),
|
|
Tok.getCommandID(),
|
|
CommandMarker);
|
|
}
|
|
consumeToken();
|
|
|
|
if (isTokBlockCommand()) {
|
|
// Block command ahead. We can't nest block commands, so pretend that this
|
|
// command has an empty argument.
|
|
ParagraphComment *Paragraph = S.actOnParagraphComment(None);
|
|
if (PC) {
|
|
S.actOnParamCommandFinish(PC, Paragraph);
|
|
return PC;
|
|
} else if (TPC) {
|
|
S.actOnTParamCommandFinish(TPC, Paragraph);
|
|
return TPC;
|
|
} else {
|
|
S.actOnBlockCommandFinish(BC, Paragraph);
|
|
return BC;
|
|
}
|
|
}
|
|
|
|
if (PC || TPC || Info->NumArgs > 0) {
|
|
// In order to parse command arguments we need to retokenize a few
|
|
// following text tokens.
|
|
TextTokenRetokenizer Retokenizer(Allocator, *this);
|
|
|
|
if (PC)
|
|
parseParamCommandArgs(PC, Retokenizer);
|
|
else if (TPC)
|
|
parseTParamCommandArgs(TPC, Retokenizer);
|
|
else
|
|
S.actOnBlockCommandArgs(BC, parseCommandArgs(Retokenizer, Info->NumArgs));
|
|
|
|
Retokenizer.putBackLeftoverTokens();
|
|
}
|
|
|
|
// If there's a block command ahead, we will attach an empty paragraph to
|
|
// this command.
|
|
bool EmptyParagraph = false;
|
|
if (isTokBlockCommand())
|
|
EmptyParagraph = true;
|
|
else if (Tok.is(tok::newline)) {
|
|
Token PrevTok = Tok;
|
|
consumeToken();
|
|
EmptyParagraph = isTokBlockCommand();
|
|
putBack(PrevTok);
|
|
}
|
|
|
|
ParagraphComment *Paragraph;
|
|
if (EmptyParagraph)
|
|
Paragraph = S.actOnParagraphComment(None);
|
|
else {
|
|
BlockContentComment *Block = parseParagraphOrBlockCommand();
|
|
// Since we have checked for a block command, we should have parsed a
|
|
// paragraph.
|
|
Paragraph = cast<ParagraphComment>(Block);
|
|
}
|
|
|
|
if (PC) {
|
|
S.actOnParamCommandFinish(PC, Paragraph);
|
|
return PC;
|
|
} else if (TPC) {
|
|
S.actOnTParamCommandFinish(TPC, Paragraph);
|
|
return TPC;
|
|
} else {
|
|
S.actOnBlockCommandFinish(BC, Paragraph);
|
|
return BC;
|
|
}
|
|
}
|
|
|
|
InlineCommandComment *Parser::parseInlineCommand() {
|
|
assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
|
|
const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
|
|
|
|
const Token CommandTok = Tok;
|
|
consumeToken();
|
|
|
|
TextTokenRetokenizer Retokenizer(Allocator, *this);
|
|
ArrayRef<Comment::Argument> Args =
|
|
parseCommandArgs(Retokenizer, Info->NumArgs);
|
|
|
|
InlineCommandComment *IC = S.actOnInlineCommand(
|
|
CommandTok.getLocation(), CommandTok.getEndLocation(),
|
|
CommandTok.getCommandID(), Args);
|
|
|
|
if (Args.size() < Info->NumArgs) {
|
|
Diag(CommandTok.getEndLocation().getLocWithOffset(1),
|
|
diag::warn_doc_inline_command_not_enough_arguments)
|
|
<< CommandTok.is(tok::at_command) << Info->Name << Args.size()
|
|
<< Info->NumArgs
|
|
<< SourceRange(CommandTok.getLocation(), CommandTok.getEndLocation());
|
|
}
|
|
|
|
Retokenizer.putBackLeftoverTokens();
|
|
|
|
return IC;
|
|
}
|
|
|
|
HTMLStartTagComment *Parser::parseHTMLStartTag() {
|
|
assert(Tok.is(tok::html_start_tag));
|
|
HTMLStartTagComment *HST =
|
|
S.actOnHTMLStartTagStart(Tok.getLocation(),
|
|
Tok.getHTMLTagStartName());
|
|
consumeToken();
|
|
|
|
SmallVector<HTMLStartTagComment::Attribute, 2> Attrs;
|
|
while (true) {
|
|
switch (Tok.getKind()) {
|
|
case tok::html_ident: {
|
|
Token Ident = Tok;
|
|
consumeToken();
|
|
if (Tok.isNot(tok::html_equals)) {
|
|
Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
|
|
Ident.getHTMLIdent()));
|
|
continue;
|
|
}
|
|
Token Equals = Tok;
|
|
consumeToken();
|
|
if (Tok.isNot(tok::html_quoted_string)) {
|
|
Diag(Tok.getLocation(),
|
|
diag::warn_doc_html_start_tag_expected_quoted_string)
|
|
<< SourceRange(Equals.getLocation());
|
|
Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
|
|
Ident.getHTMLIdent()));
|
|
while (Tok.is(tok::html_equals) ||
|
|
Tok.is(tok::html_quoted_string))
|
|
consumeToken();
|
|
continue;
|
|
}
|
|
Attrs.push_back(HTMLStartTagComment::Attribute(
|
|
Ident.getLocation(),
|
|
Ident.getHTMLIdent(),
|
|
Equals.getLocation(),
|
|
SourceRange(Tok.getLocation(),
|
|
Tok.getEndLocation()),
|
|
Tok.getHTMLQuotedString()));
|
|
consumeToken();
|
|
continue;
|
|
}
|
|
|
|
case tok::html_greater:
|
|
S.actOnHTMLStartTagFinish(HST,
|
|
S.copyArray(llvm::makeArrayRef(Attrs)),
|
|
Tok.getLocation(),
|
|
/* IsSelfClosing = */ false);
|
|
consumeToken();
|
|
return HST;
|
|
|
|
case tok::html_slash_greater:
|
|
S.actOnHTMLStartTagFinish(HST,
|
|
S.copyArray(llvm::makeArrayRef(Attrs)),
|
|
Tok.getLocation(),
|
|
/* IsSelfClosing = */ true);
|
|
consumeToken();
|
|
return HST;
|
|
|
|
case tok::html_equals:
|
|
case tok::html_quoted_string:
|
|
Diag(Tok.getLocation(),
|
|
diag::warn_doc_html_start_tag_expected_ident_or_greater);
|
|
while (Tok.is(tok::html_equals) ||
|
|
Tok.is(tok::html_quoted_string))
|
|
consumeToken();
|
|
if (Tok.is(tok::html_ident) ||
|
|
Tok.is(tok::html_greater) ||
|
|
Tok.is(tok::html_slash_greater))
|
|
continue;
|
|
|
|
S.actOnHTMLStartTagFinish(HST,
|
|
S.copyArray(llvm::makeArrayRef(Attrs)),
|
|
SourceLocation(),
|
|
/* IsSelfClosing = */ false);
|
|
return HST;
|
|
|
|
default:
|
|
// Not a token from an HTML start tag. Thus HTML tag prematurely ended.
|
|
S.actOnHTMLStartTagFinish(HST,
|
|
S.copyArray(llvm::makeArrayRef(Attrs)),
|
|
SourceLocation(),
|
|
/* IsSelfClosing = */ false);
|
|
bool StartLineInvalid;
|
|
const unsigned StartLine = SourceMgr.getPresumedLineNumber(
|
|
HST->getLocation(),
|
|
&StartLineInvalid);
|
|
bool EndLineInvalid;
|
|
const unsigned EndLine = SourceMgr.getPresumedLineNumber(
|
|
Tok.getLocation(),
|
|
&EndLineInvalid);
|
|
if (StartLineInvalid || EndLineInvalid || StartLine == EndLine)
|
|
Diag(Tok.getLocation(),
|
|
diag::warn_doc_html_start_tag_expected_ident_or_greater)
|
|
<< HST->getSourceRange();
|
|
else {
|
|
Diag(Tok.getLocation(),
|
|
diag::warn_doc_html_start_tag_expected_ident_or_greater);
|
|
Diag(HST->getLocation(), diag::note_doc_html_tag_started_here)
|
|
<< HST->getSourceRange();
|
|
}
|
|
return HST;
|
|
}
|
|
}
|
|
}
|
|
|
|
HTMLEndTagComment *Parser::parseHTMLEndTag() {
|
|
assert(Tok.is(tok::html_end_tag));
|
|
Token TokEndTag = Tok;
|
|
consumeToken();
|
|
SourceLocation Loc;
|
|
if (Tok.is(tok::html_greater)) {
|
|
Loc = Tok.getLocation();
|
|
consumeToken();
|
|
}
|
|
|
|
return S.actOnHTMLEndTag(TokEndTag.getLocation(),
|
|
Loc,
|
|
TokEndTag.getHTMLTagEndName());
|
|
}
|
|
|
|
BlockContentComment *Parser::parseParagraphOrBlockCommand() {
|
|
SmallVector<InlineContentComment *, 8> Content;
|
|
|
|
while (true) {
|
|
switch (Tok.getKind()) {
|
|
case tok::verbatim_block_begin:
|
|
case tok::verbatim_line_name:
|
|
case tok::eof:
|
|
break; // Block content or EOF ahead, finish this parapgaph.
|
|
|
|
case tok::unknown_command:
|
|
Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
|
|
Tok.getEndLocation(),
|
|
Tok.getUnknownCommandName()));
|
|
consumeToken();
|
|
continue;
|
|
|
|
case tok::backslash_command:
|
|
case tok::at_command: {
|
|
const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
|
|
if (Info->IsBlockCommand) {
|
|
if (Content.size() == 0)
|
|
return parseBlockCommand();
|
|
break; // Block command ahead, finish this parapgaph.
|
|
}
|
|
if (Info->IsVerbatimBlockEndCommand) {
|
|
Diag(Tok.getLocation(),
|
|
diag::warn_verbatim_block_end_without_start)
|
|
<< Tok.is(tok::at_command)
|
|
<< Info->Name
|
|
<< SourceRange(Tok.getLocation(), Tok.getEndLocation());
|
|
consumeToken();
|
|
continue;
|
|
}
|
|
if (Info->IsUnknownCommand) {
|
|
Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
|
|
Tok.getEndLocation(),
|
|
Info->getID()));
|
|
consumeToken();
|
|
continue;
|
|
}
|
|
assert(Info->IsInlineCommand);
|
|
Content.push_back(parseInlineCommand());
|
|
continue;
|
|
}
|
|
|
|
case tok::newline: {
|
|
consumeToken();
|
|
if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
|
|
consumeToken();
|
|
break; // Two newlines -- end of paragraph.
|
|
}
|
|
// Also allow [tok::newline, tok::text, tok::newline] if the middle
|
|
// tok::text is just whitespace.
|
|
if (Tok.is(tok::text) && isWhitespace(Tok.getText())) {
|
|
Token WhitespaceTok = Tok;
|
|
consumeToken();
|
|
if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
|
|
consumeToken();
|
|
break;
|
|
}
|
|
// We have [tok::newline, tok::text, non-newline]. Put back tok::text.
|
|
putBack(WhitespaceTok);
|
|
}
|
|
if (Content.size() > 0)
|
|
Content.back()->addTrailingNewline();
|
|
continue;
|
|
}
|
|
|
|
// Don't deal with HTML tag soup now.
|
|
case tok::html_start_tag:
|
|
Content.push_back(parseHTMLStartTag());
|
|
continue;
|
|
|
|
case tok::html_end_tag:
|
|
Content.push_back(parseHTMLEndTag());
|
|
continue;
|
|
|
|
case tok::text:
|
|
Content.push_back(S.actOnText(Tok.getLocation(),
|
|
Tok.getEndLocation(),
|
|
Tok.getText()));
|
|
consumeToken();
|
|
continue;
|
|
|
|
case tok::verbatim_block_line:
|
|
case tok::verbatim_block_end:
|
|
case tok::verbatim_line_text:
|
|
case tok::html_ident:
|
|
case tok::html_equals:
|
|
case tok::html_quoted_string:
|
|
case tok::html_greater:
|
|
case tok::html_slash_greater:
|
|
llvm_unreachable("should not see this token");
|
|
}
|
|
break;
|
|
}
|
|
|
|
return S.actOnParagraphComment(S.copyArray(llvm::makeArrayRef(Content)));
|
|
}
|
|
|
|
VerbatimBlockComment *Parser::parseVerbatimBlock() {
|
|
assert(Tok.is(tok::verbatim_block_begin));
|
|
|
|
VerbatimBlockComment *VB =
|
|
S.actOnVerbatimBlockStart(Tok.getLocation(),
|
|
Tok.getVerbatimBlockID());
|
|
consumeToken();
|
|
|
|
// Don't create an empty line if verbatim opening command is followed
|
|
// by a newline.
|
|
if (Tok.is(tok::newline))
|
|
consumeToken();
|
|
|
|
SmallVector<VerbatimBlockLineComment *, 8> Lines;
|
|
while (Tok.is(tok::verbatim_block_line) ||
|
|
Tok.is(tok::newline)) {
|
|
VerbatimBlockLineComment *Line;
|
|
if (Tok.is(tok::verbatim_block_line)) {
|
|
Line = S.actOnVerbatimBlockLine(Tok.getLocation(),
|
|
Tok.getVerbatimBlockText());
|
|
consumeToken();
|
|
if (Tok.is(tok::newline)) {
|
|
consumeToken();
|
|
}
|
|
} else {
|
|
// Empty line, just a tok::newline.
|
|
Line = S.actOnVerbatimBlockLine(Tok.getLocation(), "");
|
|
consumeToken();
|
|
}
|
|
Lines.push_back(Line);
|
|
}
|
|
|
|
if (Tok.is(tok::verbatim_block_end)) {
|
|
const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID());
|
|
S.actOnVerbatimBlockFinish(VB, Tok.getLocation(),
|
|
Info->Name,
|
|
S.copyArray(llvm::makeArrayRef(Lines)));
|
|
consumeToken();
|
|
} else {
|
|
// Unterminated \\verbatim block
|
|
S.actOnVerbatimBlockFinish(VB, SourceLocation(), "",
|
|
S.copyArray(llvm::makeArrayRef(Lines)));
|
|
}
|
|
|
|
return VB;
|
|
}
|
|
|
|
VerbatimLineComment *Parser::parseVerbatimLine() {
|
|
assert(Tok.is(tok::verbatim_line_name));
|
|
|
|
Token NameTok = Tok;
|
|
consumeToken();
|
|
|
|
SourceLocation TextBegin;
|
|
StringRef Text;
|
|
// Next token might not be a tok::verbatim_line_text if verbatim line
|
|
// starting command comes just before a newline or comment end.
|
|
if (Tok.is(tok::verbatim_line_text)) {
|
|
TextBegin = Tok.getLocation();
|
|
Text = Tok.getVerbatimLineText();
|
|
} else {
|
|
TextBegin = NameTok.getEndLocation();
|
|
Text = "";
|
|
}
|
|
|
|
VerbatimLineComment *VL = S.actOnVerbatimLine(NameTok.getLocation(),
|
|
NameTok.getVerbatimLineID(),
|
|
TextBegin,
|
|
Text);
|
|
consumeToken();
|
|
return VL;
|
|
}
|
|
|
|
BlockContentComment *Parser::parseBlockContent() {
|
|
switch (Tok.getKind()) {
|
|
case tok::text:
|
|
case tok::unknown_command:
|
|
case tok::backslash_command:
|
|
case tok::at_command:
|
|
case tok::html_start_tag:
|
|
case tok::html_end_tag:
|
|
return parseParagraphOrBlockCommand();
|
|
|
|
case tok::verbatim_block_begin:
|
|
return parseVerbatimBlock();
|
|
|
|
case tok::verbatim_line_name:
|
|
return parseVerbatimLine();
|
|
|
|
case tok::eof:
|
|
case tok::newline:
|
|
case tok::verbatim_block_line:
|
|
case tok::verbatim_block_end:
|
|
case tok::verbatim_line_text:
|
|
case tok::html_ident:
|
|
case tok::html_equals:
|
|
case tok::html_quoted_string:
|
|
case tok::html_greater:
|
|
case tok::html_slash_greater:
|
|
llvm_unreachable("should not see this token");
|
|
}
|
|
llvm_unreachable("bogus token kind");
|
|
}
|
|
|
|
FullComment *Parser::parseFullComment() {
|
|
// Skip newlines at the beginning of the comment.
|
|
while (Tok.is(tok::newline))
|
|
consumeToken();
|
|
|
|
SmallVector<BlockContentComment *, 8> Blocks;
|
|
while (Tok.isNot(tok::eof)) {
|
|
Blocks.push_back(parseBlockContent());
|
|
|
|
// Skip extra newlines after paragraph end.
|
|
while (Tok.is(tok::newline))
|
|
consumeToken();
|
|
}
|
|
return S.actOnFullComment(S.copyArray(llvm::makeArrayRef(Blocks)));
|
|
}
|
|
|
|
} // end namespace comments
|
|
} // end namespace clang
|