[flang] Clean out obsolete parsing code. Handle !cdir$ fixed and free in parser.
Work on scanning compiler directive lines.
Fix glitch in earlier change to parse-state.h.
Add ClassifyLine(), simplify some token sequence data lifetimes and interfaces.
Handle continued directives. Obey !DIR$ FIXED and FREE in prescanner.
Some refactoring of TokenSequence API, then support initial directives (FIXED, FREE, IGNORE_TKR).
Fix !DIR$ IGNORE_TKR syntax, manual was wrong.
Debugging directive scanning & parsing.
Profiling-directed speed-up - do not map cooked source locations to Provenance until a Message is emitted. Turn some non-nullable pointers into references.
Debugging.
Handle !DIR$ IVDEP too, it is in a test.
Accept directives in the execution part.
Original-commit: flang-compiler/f18@fb2ff367ec0609b7307771f927f22fd7bab5e180
Reviewed-on: https://github.com/flang-compiler/f18/pull/34
Tree-same-pre-rewrite: false
diff --git a/flang/lib/parser/basic-parsers.h b/flang/lib/parser/basic-parsers.h
index 72f41b0..b7b1f4d 100644
--- a/flang/lib/parser/basic-parsers.h
+++ b/flang/lib/parser/basic-parsers.h
@@ -1180,26 +1180,6 @@
constexpr FixedParser<true> ok;
constexpr FixedParser<false> cut;
-// guard(bool) returns a parser that succeeds iff its dynamic argument
-// value is true. The state is preserved.
-class GuardParser {
-public:
- using resultType = Success;
- constexpr GuardParser(const GuardParser &) = default;
- constexpr GuardParser(bool ok) : ok_{ok} {}
- constexpr std::optional<Success> Parse(ParseState *) const {
- if (ok_) {
- return {Success{}};
- }
- return {};
- }
-
-private:
- const bool ok_;
-};
-
-inline constexpr auto guard(bool truth) { return GuardParser(truth); }
-
// nextCh is a parser that succeeds if the parsing state is not
// at the end of its input, returning the next character location and
// advancing the parse when it does so.
@@ -1309,25 +1289,6 @@
return {state->userState()};
}
} getUserState;
-
-constexpr struct InFixedForm {
- using resultType = Success;
- constexpr InFixedForm() {}
- static std::optional<Success> Parse(ParseState *state) {
- if (state->inFixedForm()) {
- return {Success{}};
- }
- return {};
- }
-} inFixedForm;
-
-constexpr struct GetColumn {
- using resultType = int;
- constexpr GetColumn() {}
- static std::optional<int> Parse(ParseState *state) {
- return {state->column()};
- }
-} getColumn;
} // namespace parser
} // namespace Fortran
#endif // FORTRAN_PARSER_BASIC_PARSERS_H_
diff --git a/flang/lib/parser/grammar.h b/flang/lib/parser/grammar.h
index f32155d..919b4f7 100644
--- a/flang/lib/parser/grammar.h
+++ b/flang/lib/parser/grammar.h
@@ -148,6 +148,7 @@
constexpr Parser<EndSubroutineStmt> endSubroutineStmt; // R1537
constexpr Parser<EntryStmt> entryStmt; // R1541
constexpr Parser<ContainsStmt> containsStmt; // R1543
+constexpr Parser<CompilerDirective> compilerDirective;
// For a parser p, indirect(p) returns a parser that builds an indirect
// reference to p's return type.
@@ -166,23 +167,15 @@
// R611 label -> digit [digit]...
constexpr auto label = spaces >> digitString;
-static inline bool isColumnOkForFixedFormLabel(int &&column) {
- return column < 6;
-}
-
-constexpr auto isLabelOk = inFixedForm >>
- applyFunction(isColumnOkForFixedFormLabel, getColumn) ||
- pure(true);
-
template<typename PA>
using statementConstructor = construct<Statement<typename PA::resultType>>;
template<typename PA> inline constexpr auto unterminatedStatement(const PA &p) {
- return skipMany("\n"_tok) >>
- sourced(statementConstructor<PA>{}(maybe(label), isLabelOk, spaces >> p));
+ return skipEmptyLines >>
+ sourced(statementConstructor<PA>{}(maybe(label), spaces >> p));
}
-constexpr auto endOfLine = "\n"_ch / skipMany("\n"_tok) ||
+constexpr auto endOfLine = "\n"_ch / skipEmptyLines ||
fail<const char *>("expected end of line"_en_US);
constexpr auto endOfStmt = spaces >>
@@ -192,7 +185,7 @@
return unterminatedStatement(p) / endOfStmt;
}
-constexpr auto ignoredStatementPrefix = skipMany("\n"_tok) >>
+constexpr auto ignoredStatementPrefix = skipEmptyLines >>
maybe(label) >> spaces;
// Error recovery within statements: skip to the end of the line,
@@ -241,7 +234,8 @@
statement(Parser<OtherSpecificationStmt>{})) ||
construct<SpecificationConstruct>{}(
statement(indirect(typeDeclarationStmt))) ||
- construct<SpecificationConstruct>{}(indirect(Parser<StructureDef>{})))
+ construct<SpecificationConstruct>{}(indirect(Parser<StructureDef>{})) ||
+ construct<SpecificationConstruct>{}(indirect(compilerDirective)))
// R513 other-specification-stmt ->
// access-stmt | allocatable-stmt | asynchronous-stmt | bind-stmt |
@@ -393,7 +387,7 @@
TYPE_PARSER(
construct<Program>{}(
// statements consume only trailing noise; consume leading noise here.
- skipMany("\n"_tok) >>
+ skipEmptyLines >>
some(startNewSubprogram >> Parser<ProgramUnit>{} / endOfLine)) /
consumedAllInput)
@@ -555,7 +549,8 @@
construct<ExecutableConstruct>{}(indirect(Parser<SelectRankConstruct>{})) ||
construct<ExecutableConstruct>{}(indirect(Parser<SelectTypeConstruct>{})) ||
construct<ExecutableConstruct>{}(indirect(whereConstruct)) ||
- construct<ExecutableConstruct>{}(indirect(forallConstruct));
+ construct<ExecutableConstruct>{}(indirect(forallConstruct)) ||
+ construct<ExecutableConstruct>{}(indirect(compilerDirective));
// R510 execution-part-construct ->
// executable-construct | format-stmt | entry-stmt | data-stmt
@@ -3643,7 +3638,19 @@
TYPE_PARSER(construct<StmtFunctionStmt>{}(
name, parenthesized(optionalList(name)), "=" >> scalar(expr)))
-// Extension and deprecated statements
+// Directives, extensions, and deprecated statements
+// !DIR$ IVDEP
+// !DIR$ IGNORE_TKR [ [(tkr...)] name ]...
+constexpr auto beginDirective = skipEmptyLines >> spaces >> "!"_ch;
+constexpr auto endDirective = spaces >> endOfLine;
+constexpr auto ivdep = "DIR$ IVDEP" >> construct<CompilerDirective::IVDEP>{};
+constexpr auto ignore_tkr = "DIR$ IGNORE_TKR" >>
+ optionalList(construct<CompilerDirective::IgnoreTKR>{}(
+ defaulted(parenthesized(some("tkr"_ch))), name));
+TYPE_PARSER(beginDirective >> sourced(construct<CompilerDirective>{}(ivdep) ||
+ construct<CompilerDirective>{}(ignore_tkr)) /
+ endDirective)
+
TYPE_PARSER(
extension(construct<BasedPointerStmt>{}("POINTER (" >> objectName / ",",
objectName, maybe(Parser<ArraySpec>{}) / ")")))
diff --git a/flang/lib/parser/idioms.h b/flang/lib/parser/idioms.h
index 0048e6e..00805359 100644
--- a/flang/lib/parser/idioms.h
+++ b/flang/lib/parser/idioms.h
@@ -108,9 +108,10 @@
std::string EnumIndexToString(int index, const char *names);
#define ENUM_CLASS(NAME, ...) \
enum class NAME { __VA_ARGS__ }; \
- static inline std::string EnumToString(NAME e) \
- { return Fortran::parser::EnumIndexToString( \
- static_cast<int>(e), #__VA_ARGS__); }
+ static inline std::string EnumToString(NAME e) { \
+ return Fortran::parser::EnumIndexToString( \
+ static_cast<int>(e), #__VA_ARGS__); \
+ }
} // namespace parser
} // namespace Fortran
diff --git a/flang/lib/parser/message.cc b/flang/lib/parser/message.cc
index 0552753..cd2546d 100644
--- a/flang/lib/parser/message.cc
+++ b/flang/lib/parser/message.cc
@@ -50,9 +50,13 @@
}
Provenance Message::Emit(
- std::ostream &o, const AllSources &sources, bool echoSourceLine) const {
- if (!context_ || context_->Emit(o, sources, false) != provenance_) {
- sources.Identify(o, provenance_, "", echoSourceLine);
+ std::ostream &o, const CookedSource &cooked, bool echoSourceLine) const {
+ Provenance provenance{provenance_};
+ if (cookedSourceLocation_ != nullptr) {
+ provenance = cooked.GetProvenance(cookedSourceLocation_).start();
+ }
+ if (!context_ || context_->Emit(o, cooked, false) != provenance) {
+ cooked.allSources().Identify(o, provenance, "", echoSourceLine);
}
o << " ";
if (string_.empty()) {
@@ -71,10 +75,11 @@
o << string_;
}
o << '\n';
- return provenance_;
+ return provenance;
}
-void Messages::Emit(std::ostream &o, const char *prefix) const {
+void Messages::Emit(
+ std::ostream &o, const char *prefix, bool echoSourceLines) const {
for (const auto &msg : messages_) {
if (prefix) {
o << prefix;
@@ -82,7 +87,7 @@
if (msg.context()) {
o << "In the context ";
}
- msg.Emit(o, allSources_);
+ msg.Emit(o, cooked_, echoSourceLines);
}
}
} // namespace parser
diff --git a/flang/lib/parser/message.h b/flang/lib/parser/message.h
index 4a6fbd0..d55e7d5 100644
--- a/flang/lib/parser/message.h
+++ b/flang/lib/parser/message.h
@@ -73,6 +73,10 @@
public:
Message() {}
Message(const Message &) = default;
+ Message(Message &&) = default;
+ Message &operator=(const Message &that) = default;
+ Message &operator=(Message &&that) = default;
+
Message(Provenance p, MessageFixedText t, MessageContext c = nullptr)
: provenance_{p}, text_{t}, context_{c} {}
Message(Provenance p, MessageFormattedText &&s, MessageContext c = nullptr)
@@ -80,22 +84,35 @@
Message(Provenance p, MessageExpectedText t, MessageContext c = nullptr)
: provenance_{p}, text_{t.AsMessageFixedText()},
isExpectedText_{true}, context_{c} {}
- Message(Message &&) = default;
- Message &operator=(const Message &that) = default;
- Message &operator=(Message &&that) = default;
+
+ Message(const char *csl, MessageFixedText t, MessageContext c = nullptr)
+ : cookedSourceLocation_{csl}, text_{t}, context_{c} {}
+ Message(const char *csl, MessageFormattedText &&s, MessageContext c = nullptr)
+ : cookedSourceLocation_{csl}, string_{s.MoveString()}, context_{c} {}
+ Message(const char *csl, MessageExpectedText t, MessageContext c = nullptr)
+ : cookedSourceLocation_{csl}, text_{t.AsMessageFixedText()},
+ isExpectedText_{true}, context_{c} {}
bool operator<(const Message &that) const {
- return provenance_ < that.provenance_;
+ if (cookedSourceLocation_ != nullptr) {
+ return cookedSourceLocation_ < that.cookedSourceLocation_;
+ } else if (that.cookedSourceLocation_ != nullptr) {
+ return false;
+ } else {
+ return provenance_ < that.provenance_;
+ }
}
Provenance provenance() const { return provenance_; }
+ const char *cookedSourceLocation() const { return cookedSourceLocation_; }
MessageContext context() const { return context_; }
Provenance Emit(
- std::ostream &, const AllSources &, bool echoSourceLine = true) const;
+ std::ostream &, const CookedSource &, bool echoSourceLine = true) const;
private:
Provenance provenance_;
+ const char *cookedSourceLocation_{nullptr};
MessageFixedText text_;
bool isExpectedText_{false}; // implies "expected '%s'"_en_US
std::string string_;
@@ -109,9 +126,9 @@
using iterator = list_type::iterator;
using const_iterator = list_type::const_iterator;
- explicit Messages(const AllSources &sources) : allSources_{sources} {}
+ explicit Messages(const CookedSource &cooked) : cooked_{cooked} {}
Messages(Messages &&that)
- : allSources_{that.allSources_}, messages_{std::move(that.messages_)},
+ : cooked_{that.cooked_}, messages_{std::move(that.messages_)},
last_{that.last_} {}
Messages &operator=(Messages &&that) {
swap(that);
@@ -130,10 +147,18 @@
const_iterator cbegin() const { return messages_.cbegin(); }
const_iterator cend() const { return messages_.cend(); }
- const AllSources &allSources() const { return allSources_; }
+ const CookedSource &cooked() const { return cooked_; }
+
+ bool IsValidLocation(const Message &m) {
+ if (auto p{m.cookedSourceLocation()}) {
+ return cooked_.IsValid(p);
+ } else {
+ return cooked_.IsValid(m.provenance());
+ }
+ }
Message &Put(Message &&m) {
- CHECK(allSources_.IsValid(m.provenance()));
+ CHECK(IsValidLocation(m));
if (messages_.empty()) {
messages_.emplace_front(std::move(m));
last_ = messages_.begin();
@@ -154,10 +179,11 @@
}
}
- void Emit(std::ostream &, const char *prefix = nullptr) const;
+ void Emit(std::ostream &, const char *prefix = nullptr,
+ bool echoSourceLines = true) const;
private:
- const AllSources &allSources_;
+ const CookedSource &cooked_;
list_type messages_;
iterator last_; // valid iff messages_ nonempty
};
diff --git a/flang/lib/parser/parse-state.h b/flang/lib/parser/parse-state.h
index a6e6915..63de742 100644
--- a/flang/lib/parser/parse-state.h
+++ b/flang/lib/parser/parse-state.h
@@ -28,22 +28,21 @@
// TODO: Add a constructor for parsing a normalized module file.
ParseState(const CookedSource &cooked)
: cooked_{cooked}, p_{&cooked[0]}, limit_{p_ + cooked.size()},
- messages_{*cooked.allSources()} {}
+ messages_{cooked} {}
ParseState(const ParseState &that)
: cooked_{that.cooked_}, p_{that.p_}, limit_{that.limit_},
- column_{that.column_}, messages_{*that.cooked_.allSources()},
- userState_{that.userState_}, inFixedForm_{that.inFixedForm_},
- encoding_{that.encoding_}, strictConformance_{that.strictConformance_},
+ messages_{that.cooked_}, userState_{that.userState_},
+ inFixedForm_{that.inFixedForm_}, encoding_{that.encoding_},
+ strictConformance_{that.strictConformance_},
warnOnNonstandardUsage_{that.warnOnNonstandardUsage_},
warnOnDeprecatedUsage_{that.warnOnDeprecatedUsage_},
anyErrorRecovery_{that.anyErrorRecovery_},
anyConformanceViolation_{that.anyConformanceViolation_} {}
ParseState(ParseState &&that)
: cooked_{that.cooked_}, p_{that.p_}, limit_{that.limit_},
- column_{that.column_}, messages_{std::move(that.messages_)},
- context_{std::move(that.context_)}, userState_{that.userState_},
- inFixedForm_{that.inFixedForm_}, encoding_{that.encoding_},
- strictConformance_{that.strictConformance_},
+ messages_{std::move(that.messages_)}, context_{std::move(that.context_)},
+ userState_{that.userState_}, inFixedForm_{that.inFixedForm_},
+ encoding_{that.encoding_}, strictConformance_{that.strictConformance_},
warnOnNonstandardUsage_{that.warnOnNonstandardUsage_},
warnOnDeprecatedUsage_{that.warnOnDeprecatedUsage_},
anyErrorRecovery_{that.anyErrorRecovery_},
@@ -62,7 +61,6 @@
}
const CookedSource &cooked() const { return cooked_; }
- int column() const { return column_; }
Messages *messages() { return &messages_; }
bool anyErrorRecovery() const { return anyErrorRecovery_; }
@@ -134,34 +132,23 @@
Message &PutMessage(MessageExpectedText &&t) {
return PutMessage(p_, std::move(t));
}
+
Message &PutMessage(const char *at, MessageFixedText t) {
- return PutMessage(GetProvenance(at), t);
- }
- Message &PutMessage(const char *at, MessageFormattedText &&t) {
- return PutMessage(GetProvenance(at), std::move(t));
- }
- Message &PutMessage(const char *at, MessageExpectedText &&t) {
- return PutMessage(GetProvenance(at), std::move(t));
- }
- Message &PutMessage(Provenance at, MessageFixedText t) {
return messages_.Put(Message{at, t, context_});
}
- Message &PutMessage(Provenance at, MessageFormattedText &&t) {
+ Message &PutMessage(const char *at, MessageFormattedText &&t) {
return messages_.Put(Message{at, std::move(t), context_});
}
- Message &PutMessage(Provenance at, MessageExpectedText &&t) {
+ Message &PutMessage(const char *at, MessageExpectedText &&t) {
return messages_.Put(Message{at, std::move(t), context_});
}
bool IsAtEnd() const { return p_ >= limit_; }
- char UncheckedAdvance() {
- ++column_;
- char ch{*p_++};
- if (ch == '\n') {
- column_ = 1;
- }
- return ch;
+ char UncheckedAdvance(std::size_t n = 1) {
+ char result{*p_};
+ p_ += n;
+ return result;
}
std::optional<char> GetNextChar() {
@@ -182,7 +169,6 @@
// Text remaining to be parsed
const CookedSource &cooked_;
const char *p_{nullptr}, *limit_{nullptr};
- int column_{1};
// Accumulated messages and current nested context.
Messages messages_;
diff --git a/flang/lib/parser/parse-tree-visitor.h b/flang/lib/parser/parse-tree-visitor.h
index 2760809..fbc2b6b 100644
--- a/flang/lib/parser/parse-tree-visitor.h
+++ b/flang/lib/parser/parse-tree-visitor.h
@@ -5,6 +5,7 @@
#include <cstddef>
#include <optional>
#include <tuple>
+#include <utility>
#include <variant>
/// Parse tree visitor
@@ -67,6 +68,13 @@
visitor.Post(x);
}
}
+template<typename A, typename B, typename V>
+void Walk(const std::pair<A, B> &x, V &visitor) {
+ if (visitor.Pre(x)) {
+ Walk(x.first, visitor);
+ Walk(x.second, visitor);
+ }
+}
// Trait-determined traversal of empty, tuple, union, and wrapper classes.
template<typename A, typename V>
diff --git a/flang/lib/parser/parse-tree.h b/flang/lib/parser/parse-tree.h
index c99f952..9b5e0e4 100644
--- a/flang/lib/parser/parse-tree.h
+++ b/flang/lib/parser/parse-tree.h
@@ -243,7 +243,8 @@
struct ReturnStmt; // R1542
struct StmtFunctionStmt; // R1544
-// Extension and deprecated statements
+// Directives, extensions, and deprecated statements
+struct CompilerDirective;
struct BasedPointerStmt;
struct StructureDef;
struct ArithmeticIfStmt;
@@ -314,12 +315,10 @@
// A wrapper for xzy-stmt productions that are statements, so that
// source provenances and labels have a uniform representation.
template<typename A> struct Statement {
- Statement(std::optional<long> &&lab, bool &&accept, A &&s)
- : label(std::move(lab)), isLabelInAcceptableField{accept},
- statement(std::move(s)) {}
+ Statement(std::optional<long> &&lab, A &&s)
+ : label(std::move(lab)), statement(std::move(s)) {}
CharBlock source;
std::optional<Label> label;
- bool isLabelInAcceptableField{true};
A statement;
};
@@ -361,7 +360,8 @@
Statement<Indirection<OldParameterStmt>>,
Statement<Indirection<ProcedureDeclarationStmt>>,
Statement<OtherSpecificationStmt>,
- Statement<Indirection<TypeDeclarationStmt>>, Indirection<StructureDef>>
+ Statement<Indirection<TypeDeclarationStmt>>, Indirection<StructureDef>,
+ Indirection<CompilerDirective>>
u;
};
@@ -470,7 +470,8 @@
Statement<Indirection<LabelDoStmt>>, Statement<Indirection<EndDoStmt>>,
Indirection<DoConstruct>, Indirection<IfConstruct>,
Indirection<SelectRankConstruct>, Indirection<SelectTypeConstruct>,
- Indirection<WhereConstruct>, Indirection<ForallConstruct>>
+ Indirection<WhereConstruct>, Indirection<ForallConstruct>,
+ Indirection<CompilerDirective>>
u;
};
@@ -3121,6 +3122,20 @@
std::tuple<Name, std::list<Name>, Scalar<Expr>> t;
};
+// Compiler directives
+// !DIR$ IVDEP
+// !DIR$ IGNORE_TKR [ [(tkr...)] name ]...
+struct CompilerDirective {
+ UNION_CLASS_BOILERPLATE(CompilerDirective);
+ struct IgnoreTKR {
+ TUPLE_CLASS_BOILERPLATE(IgnoreTKR);
+ std::tuple<std::list<const char *>, Name> t;
+ };
+ EMPTY_CLASS(IVDEP);
+ CharBlock source;
+ std::variant<std::list<IgnoreTKR>, IVDEP> u;
+};
+
// Legacy extensions
struct BasedPointerStmt {
TUPLE_CLASS_BOILERPLATE(BasedPointerStmt);
diff --git a/flang/lib/parser/parsing.cc b/flang/lib/parser/parsing.cc
index d62d3aa..530f55d 100644
--- a/flang/lib/parser/parsing.cc
+++ b/flang/lib/parser/parsing.cc
@@ -33,7 +33,7 @@
allSources_.PushSearchPathDirectory(path);
}
- Preprocessor preprocessor{&allSources_};
+ Preprocessor preprocessor{allSources_};
for (const auto &predef : options.predefinitions) {
if (predef.second.has_value()) {
preprocessor.Define(predef.first, *predef.second);
@@ -41,13 +41,13 @@
preprocessor.Undefine(predef.first);
}
}
- Prescanner prescanner{&messages_, &cooked_, &preprocessor};
+ Prescanner prescanner{messages_, cooked_, preprocessor};
prescanner.set_fixedForm(options.isFixedForm)
.set_fixedFormColumnLimit(options.fixedFormColumns)
.set_encoding(options.encoding)
.set_enableBackslashEscapesInCharLiterals(options.enableBackslashEscapes)
- .set_enableOldDebugLines(options.enableOldDebugLines);
-// TODO in development .AddCompilerDirectiveSentinel("dir$");
+ .set_enableOldDebugLines(options.enableOldDebugLines)
+ .AddCompilerDirectiveSentinel("dir$");
ProvenanceRange range{
allSources_.AddIncludedFile(*sourceFile, ProvenanceRange{})};
anyFatalError_ = !prescanner.Prescan(range);
diff --git a/flang/lib/parser/parsing.h b/flang/lib/parser/parsing.h
index 8880c72..6214e7b 100644
--- a/flang/lib/parser/parsing.h
+++ b/flang/lib/parser/parsing.h
@@ -51,8 +51,8 @@
private:
Options options_;
AllSources allSources_;
- Messages messages_{allSources_};
- CookedSource cooked_{&allSources_};
+ CookedSource cooked_{allSources_};
+ Messages messages_{cooked_};
bool anyFatalError_{false};
bool consumedWholeFile_{false};
Provenance finalRestingPlace_;
diff --git a/flang/lib/parser/preprocessor.cc b/flang/lib/parser/preprocessor.cc
index ac23bca..30634be 100644
--- a/flang/lib/parser/preprocessor.cc
+++ b/flang/lib/parser/preprocessor.cc
@@ -28,10 +28,10 @@
argumentCount_(argNames.size()), isVariadic_{isVariadic},
replacement_{Tokenize(argNames, repl, firstToken, tokens)} {}
-Definition::Definition(const std::string &predefined, AllSources *sources)
- : isPredefined_{true},
- replacement_{
- predefined, sources->AddCompilerInsertion(predefined).start()} {}
+Definition::Definition(const std::string &predefined, AllSources &sources)
+ : isPredefined_{true}, replacement_{predefined,
+ sources.AddCompilerInsertion(predefined).start()} {
+}
bool Definition::set_isDisabled(bool disable) {
bool was{isDisabled_};
@@ -53,7 +53,7 @@
}
TokenSequence result;
for (std::size_t j{0}; j < tokens; ++j) {
- CharBlock tok{token[firstToken + j]};
+ CharBlock tok{token.TokenAt(firstToken + j)};
if (IsLegalIdentifierStart(tok)) {
auto it = args.find(tok.ToString());
if (it != args.end()) {
@@ -67,8 +67,8 @@
}
static std::size_t AfterLastNonBlank(const TokenSequence &tokens) {
- for (std::size_t j{tokens.size()}; j > 0; --j) {
- if (!tokens[j - 1].IsBlank()) {
+ for (std::size_t j{tokens.SizeInTokens()}; j > 0; --j) {
+ if (!tokens.TokenAt(j - 1).IsBlank()) {
return j;
}
}
@@ -76,12 +76,12 @@
}
static TokenSequence Stringify(
- const TokenSequence &tokens, AllSources *allSources) {
+ const TokenSequence &tokens, AllSources &allSources) {
TokenSequence result;
- Provenance quoteProvenance{allSources->CompilerInsertionProvenance('"')};
+ Provenance quoteProvenance{allSources.CompilerInsertionProvenance('"')};
result.PutNextTokenChar('"', quoteProvenance);
- for (std::size_t j{0}; j < tokens.size(); ++j) {
- const CharBlock &token{tokens[j]};
+ for (std::size_t j{0}; j < tokens.SizeInTokens(); ++j) {
+ const CharBlock &token{tokens.TokenAt(j)};
std::size_t bytes{token.size()};
for (std::size_t k{0}; k < bytes; ++k) {
char ch{token[k]};
@@ -98,14 +98,14 @@
}
TokenSequence Definition::Apply(
- const std::vector<TokenSequence> &args, AllSources *allSources) {
+ const std::vector<TokenSequence> &args, AllSources &allSources) {
TokenSequence result;
bool pasting{false};
bool skipping{false};
int parenthesesNesting{0};
- std::size_t tokens{replacement_.size()};
+ std::size_t tokens{replacement_.SizeInTokens()};
for (std::size_t j{0}; j < tokens; ++j) {
- const CharBlock &token{replacement_[j]};
+ const CharBlock &token{replacement_.TokenAt(j)};
std::size_t bytes{token.size()};
if (skipping) {
if (bytes == 1) {
@@ -124,16 +124,16 @@
}
std::size_t afterLastNonBlank{AfterLastNonBlank(result)};
if (afterLastNonBlank > 0 &&
- result[afterLastNonBlank - 1].ToString() == "#") {
+ result.TokenAt(afterLastNonBlank - 1).ToString() == "#") {
// stringifying
- while (result.size() >= afterLastNonBlank) {
+ while (result.SizeInTokens() >= afterLastNonBlank) {
result.pop_back();
}
result.Put(Stringify(args[index], allSources));
} else {
- std::size_t argTokens{args[index].size()};
+ std::size_t argTokens{args[index].SizeInTokens()};
for (std::size_t k{0}; k < argTokens; ++k) {
- if (!pasting || !args[index][k].IsBlank()) {
+ if (!pasting || !args[index].TokenAt(k).IsBlank()) {
result.Put(args[index], k);
pasting = false;
}
@@ -142,7 +142,8 @@
} else if (bytes == 2 && token[0] == '#' && token[1] == '#') {
// Token pasting operator in body (not expanded argument); discard any
// immediately preceding white space, then reopen the last token.
- while (!result.empty() && result[result.size() - 1].IsBlank()) {
+ while (!result.empty() &&
+ result.TokenAt(result.SizeInTokens() - 1).IsBlank()) {
result.pop_back();
}
if (!result.empty()) {
@@ -153,7 +154,7 @@
// Delete whitespace immediately following ## in the body.
} else if (bytes == 11 && isVariadic_ &&
token.ToString() == "__VA_ARGs__") {
- Provenance commaProvenance{allSources->CompilerInsertionProvenance(',')};
+ Provenance commaProvenance{allSources.CompilerInsertionProvenance(',')};
for (std::size_t k{argumentCount_}; k < args.size(); ++k) {
if (k > argumentCount_) {
result.Put(","s, commaProvenance);
@@ -161,7 +162,7 @@
result.Put(args[k]);
}
} else if (bytes == 10 && isVariadic_ && token.ToString() == "__VA_OPT__" &&
- j + 2 < tokens && replacement_[j + 1].ToString() == "(" &&
+ j + 2 < tokens && replacement_.TokenAt(j + 1).ToString() == "(" &&
parenthesesNesting == 0) {
parenthesesNesting = 1;
skipping = args.size() == argumentCount_;
@@ -187,7 +188,7 @@
std::strftime(buffer, sizeof buffer, format, std::localtime(&now))};
}
-Preprocessor::Preprocessor(AllSources *allSources) : allSources_{allSources} {
+Preprocessor::Preprocessor(AllSources &allSources) : allSources_{allSources} {
// Capture current local date & time once now to avoid having the values
// of __DATE__ or __TIME__ change during compilation.
std::time_t now;
@@ -209,55 +210,55 @@
void Preprocessor::Undefine(std::string macro) { definitions_.erase(macro); }
-bool Preprocessor::MacroReplacement(const TokenSequence &input,
- const Prescanner &prescanner, TokenSequence *result) {
+std::optional<TokenSequence> Preprocessor::MacroReplacement(
+ const TokenSequence &input, const Prescanner &prescanner) {
// Do quick scan for any use of a defined name.
- std::size_t tokens{input.size()};
+ std::size_t tokens{input.SizeInTokens()};
std::size_t j;
for (j = 0; j < tokens; ++j) {
- std::size_t bytes{input[j].size()};
- if (bytes > 0 && IsLegalIdentifierStart(input[j][0]) &&
- IsNameDefined(input[j])) {
+ CharBlock token{input.TokenAt(j)};
+ if (!token.empty() && IsLegalIdentifierStart(token[0]) &&
+ IsNameDefined(token)) {
break;
}
}
if (j == tokens) {
- return false; // contains nothing that would be replaced
+ return {}; // input contains nothing that would be replaced
}
- result->Put(input, 0, j);
+ TokenSequence result{input, 0, j};
for (; j < tokens; ++j) {
- const CharBlock &token{input[j]};
+ const CharBlock &token{input.TokenAt(j)};
if (token.IsBlank() || !IsLegalIdentifierStart(token[0])) {
- result->Put(input, j);
+ result.Put(input, j);
continue;
}
auto it = definitions_.find(token);
if (it == definitions_.end()) {
- result->Put(input, j);
+ result.Put(input, j);
continue;
}
Definition &def{it->second};
if (def.isDisabled()) {
- result->Put(input, j);
+ result.Put(input, j);
continue;
}
if (!def.isFunctionLike()) {
if (def.isPredefined()) {
- std::string name{def.replacement()[0].ToString()};
+ std::string name{def.replacement().TokenAt(0).ToString()};
std::string repl;
if (name == "__FILE__") {
repl = "\""s +
- allSources_->GetPath(prescanner.GetCurrentProvenance()) + '"';
+ allSources_.GetPath(prescanner.GetCurrentProvenance()) + '"';
} else if (name == "__LINE__") {
std::stringstream ss;
- ss << allSources_->GetLineNumber(prescanner.GetCurrentProvenance());
+ ss << allSources_.GetLineNumber(prescanner.GetCurrentProvenance());
repl = ss.str();
}
if (!repl.empty()) {
- ProvenanceRange insert{allSources_->AddCompilerInsertion(repl)};
- ProvenanceRange call{allSources_->AddMacroCall(
+ ProvenanceRange insert{allSources_.AddCompilerInsertion(repl)};
+ ProvenanceRange call{allSources_.AddMacroCall(
insert, input.GetTokenProvenanceRange(j), repl)};
- result->Put(repl, call.start());
+ result.Put(repl, call.start());
continue;
}
}
@@ -268,8 +269,8 @@
ProvenanceRange from{def.replacement().GetProvenanceRange()};
ProvenanceRange use{input.GetTokenProvenanceRange(j)};
ProvenanceRange newRange{
- allSources_->AddMacroCall(from, use, replaced.ToString())};
- result->Put(replaced, newRange);
+ allSources_.AddMacroCall(from, use, replaced.ToString())};
+ result.Put(replaced, newRange);
}
continue;
}
@@ -278,20 +279,21 @@
std::size_t k{j};
bool leftParen{false};
while (++k < tokens) {
- const CharBlock &lookAhead{input[k]};
+ const CharBlock &lookAhead{input.TokenAt(k)};
if (!lookAhead.IsBlank() && lookAhead[0] != '\n') {
leftParen = lookAhead[0] == '(' && lookAhead.size() == 1;
break;
}
}
if (!leftParen) {
- result->Put(input, j);
+ result.Put(input, j);
continue;
}
std::vector<std::size_t> argStart{++k};
for (int nesting{0}; k < tokens; ++k) {
- if (input[k].size() == 1) {
- char ch{input[k][0]};
+ CharBlock token{input.TokenAt(k)};
+ if (token.size() == 1) {
+ char ch{token[0]};
if (ch == '(') {
++nesting;
} else if (ch == ')') {
@@ -306,7 +308,7 @@
}
if (k >= tokens || argStart.size() < def.argumentCount() ||
(argStart.size() > def.argumentCount() && !def.isVariadic())) {
- result->Put(input, j);
+ result.Put(input, j);
continue;
}
std::vector<TokenSequence> args;
@@ -324,24 +326,26 @@
ProvenanceRange from{def.replacement().GetProvenanceRange()};
ProvenanceRange use{input.GetIntervalProvenanceRange(j, k - j)};
ProvenanceRange newRange{
- allSources_->AddMacroCall(from, use, replaced.ToString())};
- result->Put(replaced, newRange);
+ allSources_.AddMacroCall(from, use, replaced.ToString())};
+ result.Put(replaced, newRange);
}
j = k; // advance to the terminal ')'
}
- return true;
+ return {result};
}
TokenSequence Preprocessor::ReplaceMacros(
const TokenSequence &tokens, const Prescanner &prescanner) {
- TokenSequence repl;
- return MacroReplacement(tokens, prescanner, &repl) ? repl : tokens;
+ if (std::optional<TokenSequence> repl{MacroReplacement(tokens, prescanner)}) {
+ return std::move(*repl);
+ }
+ return tokens;
}
static std::size_t SkipBlanks(
const TokenSequence &tokens, std::size_t at, std::size_t lastToken) {
for (; at < lastToken; ++at) {
- if (!tokens[at].IsBlank()) {
+ if (!tokens.TokenAt(at).IsBlank()) {
break;
}
}
@@ -359,12 +363,12 @@
}
void Preprocessor::Directive(const TokenSequence &dir, Prescanner *prescanner) {
- std::size_t tokens{dir.size()};
+ std::size_t tokens{dir.SizeInTokens()};
std::size_t j{SkipBlanks(dir, 0, tokens)};
if (j == tokens) {
return;
}
- if (dir[j].ToString() != "#") {
+ if (dir.TokenAt(j).ToString() != "#") {
prescanner->Error("missing '#'"_en_US, dir.GetTokenProvenance(j));
return;
}
@@ -372,15 +376,15 @@
if (j == tokens) {
return;
}
- if (IsDecimalDigit(dir[j][0]) || dir[j][0] == '"') {
+ if (IsDecimalDigit(dir.TokenAt(j)[0]) || dir.TokenAt(j)[0] == '"') {
return; // TODO: treat as #line
}
std::size_t dirOffset{j};
- std::string dirName{ToLowerCaseLetters(dir[dirOffset].ToString())};
+ std::string dirName{ToLowerCaseLetters(dir.TokenAt(dirOffset).ToString())};
j = SkipBlanks(dir, j + 1, tokens);
CharBlock nameToken;
- if (j < tokens && IsLegalIdentifierStart(dir[j][0])) {
- nameToken = dir[j];
+ if (j < tokens && IsLegalIdentifierStart(dir.TokenAt(j)[0])) {
+ nameToken = dir.TokenAt(j);
}
if (dirName == "line") {
// TODO: implement #line
@@ -392,13 +396,14 @@
}
nameToken = SaveTokenAsName(nameToken);
definitions_.erase(nameToken);
- if (++j < tokens && dir[j].size() == 1 && dir[j][0] == '(') {
+ if (++j < tokens && dir.TokenAt(j).size() == 1 &&
+ dir.TokenAt(j)[0] == '(') {
j = SkipBlanks(dir, j + 1, tokens);
std::vector<std::string> argName;
bool isVariadic{false};
- if (dir[j].ToString() != ")") {
+ if (dir.TokenAt(j).ToString() != ")") {
while (true) {
- std::string an{dir[j].ToString()};
+ std::string an{dir.TokenAt(j).ToString()};
if (an == "...") {
isVariadic = true;
} else {
@@ -416,7 +421,7 @@
dir.GetTokenProvenance(tokens - 1));
return;
}
- std::string punc{dir[j].ToString()};
+ std::string punc{dir.TokenAt(j).ToString()};
if (punc == ")") {
break;
}
@@ -542,8 +547,8 @@
return;
}
std::string include;
- if (dir[j].ToString() == "<") {
- if (dir[tokens - 1].ToString() != ">") {
+ if (dir.TokenAt(j).ToString() == "<") {
+ if (dir.TokenAt(tokens - 1).ToString() != ">") {
prescanner->Error("#include: expected '>' at end of directive"_en_US,
dir.GetTokenProvenance(tokens - 1));
return;
@@ -551,7 +556,7 @@
TokenSequence braced{dir, j + 1, tokens - j - 2};
include = ReplaceMacros(braced, *prescanner).ToString();
} else if (j + 1 == tokens &&
- (include = dir[j].ToString()).substr(0, 1) == "\"" &&
+ (include = dir.TokenAt(j).ToString()).substr(0, 1) == "\"" &&
include.substr(include.size() - 1, 1) == "\"") {
include = include.substr(1, include.size() - 2);
} else {
@@ -565,7 +570,7 @@
return;
}
std::stringstream error;
- const SourceFile *included{allSources_->Open(include, &error)};
+ const SourceFile *included{allSources_.Open(include, &error)};
if (included == nullptr) {
prescanner->Error(
MessageFormattedText("#include: %s"_en_US, error.str().data()),
@@ -576,7 +581,7 @@
return;
}
ProvenanceRange fileRange{
- allSources_->AddIncludedFile(*included, dir.GetProvenanceRange())};
+ allSources_.AddIncludedFile(*included, dir.GetProvenanceRange())};
if (!Prescanner{*prescanner}.Prescan(fileRange)) {
prescanner->set_anyFatalErrors();
}
@@ -599,9 +604,9 @@
static std::string GetDirectiveName(
const TokenSequence &line, std::size_t *rest) {
- std::size_t tokens{line.size()};
+ std::size_t tokens{line.SizeInTokens()};
std::size_t j{SkipBlanks(line, 0, tokens)};
- if (j == tokens || line[j].ToString() != "#") {
+ if (j == tokens || line.TokenAt(j).ToString() != "#") {
*rest = tokens;
return "";
}
@@ -611,7 +616,7 @@
return "";
}
*rest = SkipBlanks(line, j + 1, tokens);
- return ToLowerCaseLetters(line[j].ToString());
+ return ToLowerCaseLetters(line.TokenAt(j).ToString());
}
void Preprocessor::SkipDisabledConditionalCode(const std::string &dirName,
@@ -637,7 +642,8 @@
return;
}
if (dn == "elif" &&
- IsIfPredicateTrue(line, rest, line.size() - rest, prescanner)) {
+ IsIfPredicateTrue(
+ line, rest, line.SizeInTokens() - rest, prescanner)) {
ifStack_.push(CanDeadElseAppear::Yes);
return;
}
@@ -743,7 +749,7 @@
opNameMap[","] = COMMA;
}
- std::size_t tokens{token.size()};
+ std::size_t tokens{token.SizeInTokens()};
if (*atToken >= tokens) {
*error = Message{
token.GetTokenProvenance(tokens - 1), "incomplete expression"_en_US};
@@ -752,7 +758,7 @@
// Parse and evaluate a primary or a unary operator and its operand.
std::size_t opAt{*atToken};
- std::string t{token[opAt].ToString()};
+ std::string t{token.TokenAt(opAt).ToString()};
enum Operator op;
std::int64_t left{0};
if (t == "(") {
@@ -776,8 +782,8 @@
} else if (t == "-") {
op = UMINUS;
} else if (t == "." && *atToken + 2 < tokens &&
- ToLowerCaseLetters(token[*atToken + 1].ToString()) == "not" &&
- token[*atToken + 2].ToString() == ".") {
+ ToLowerCaseLetters(token.TokenAt(*atToken + 1).ToString()) == "not" &&
+ token.TokenAt(*atToken + 2).ToString() == ".") {
op = NOT;
*atToken += 2;
} else {
@@ -803,7 +809,7 @@
}
switch (op) {
case PARENS:
- if (*atToken < tokens && token[*atToken].ToString() == ")") {
+ if (*atToken < tokens && token.TokenAt(*atToken).ToString() == ")") {
++*atToken;
} else {
*error = Message{token.GetTokenProvenance(tokens - 1),
@@ -825,10 +831,10 @@
// Parse and evaluate a binary operator and its second operand, if present.
int advance{1};
- t = token[*atToken].ToString();
+ t = token.TokenAt(*atToken).ToString();
if (t == "." && *atToken + 2 < tokens &&
- token[*atToken + 2].ToString() == ".") {
- t += ToLowerCaseLetters(token[*atToken + 1].ToString()) + '.';
+ token.TokenAt(*atToken + 2).ToString() == ".") {
+ t += ToLowerCaseLetters(token.TokenAt(*atToken + 1).ToString()) + '.';
advance = 3;
}
auto it = opNameMap.find(t);
@@ -931,7 +937,7 @@
case EQV: return -(!left == !right);
case NEQV: return -(!left != !right);
case SELECT:
- if (*atToken >= tokens || token[*atToken].ToString() != ":") {
+ if (*atToken >= tokens || token.TokenAt(*atToken).ToString() != ":") {
*error = Message{token.GetTokenProvenance(opAt),
"':' required in selection expression"_en_US};
return left;
@@ -951,32 +957,34 @@
std::size_t first, std::size_t exprTokens, Prescanner *prescanner) {
TokenSequence expr1{StripBlanks(expr, first, first + exprTokens)};
TokenSequence expr2;
- for (std::size_t j{0}; j < expr1.size(); ++j) {
- if (ToLowerCaseLetters(expr1[j].ToString()) == "defined") {
+ for (std::size_t j{0}; j < expr1.SizeInTokens(); ++j) {
+ if (ToLowerCaseLetters(expr1.TokenAt(j).ToString()) == "defined") {
CharBlock name;
- if (j + 3 < expr1.size() && expr1[j + 1].ToString() == "(" &&
- expr1[j + 3].ToString() == ")") {
- name = expr1[j + 2];
+ if (j + 3 < expr1.SizeInTokens() &&
+ expr1.TokenAt(j + 1).ToString() == "(" &&
+ expr1.TokenAt(j + 3).ToString() == ")") {
+ name = expr1.TokenAt(j + 2);
j += 3;
- } else if (j + 1 < expr1.size() && IsLegalIdentifierStart(expr1[j + 1])) {
- name = expr1[j++];
+ } else if (j + 1 < expr1.SizeInTokens() &&
+ IsLegalIdentifierStart(expr1.TokenAt(j + 1))) {
+ name = expr1.TokenAt(j++);
}
if (!name.empty()) {
char truth{IsNameDefined(name) ? '1' : '0'};
- expr2.Put(&truth, 1, allSources_->CompilerInsertionProvenance(truth));
+ expr2.Put(&truth, 1, allSources_.CompilerInsertionProvenance(truth));
continue;
}
}
expr2.Put(expr1, j);
}
TokenSequence expr3{ReplaceMacros(expr2, *prescanner)};
- TokenSequence expr4{StripBlanks(expr3, 0, expr3.size())};
+ TokenSequence expr4{StripBlanks(expr3, 0, expr3.SizeInTokens())};
std::size_t atToken{0};
std::optional<Message> error;
bool result{ExpressionValue(expr4, 0, &atToken, &error) != 0};
if (error.has_value()) {
prescanner->Error(std::move(*error));
- } else if (atToken < expr4.size()) {
+ } else if (atToken < expr4.SizeInTokens()) {
prescanner->Error(atToken == 0 ? "could not parse any expression"_en_US
: "excess characters after expression"_en_US,
expr4.GetTokenProvenance(atToken));
diff --git a/flang/lib/parser/preprocessor.h b/flang/lib/parser/preprocessor.h
index 2a67944..e0a50c7 100644
--- a/flang/lib/parser/preprocessor.h
+++ b/flang/lib/parser/preprocessor.h
@@ -28,7 +28,7 @@
Definition(const TokenSequence &, std::size_t firstToken, std::size_t tokens);
Definition(const std::vector<std::string> &argNames, const TokenSequence &,
std::size_t firstToken, std::size_t tokens, bool isVariadic = false);
- Definition(const std::string &predefined, AllSources *);
+ Definition(const std::string &predefined, AllSources &);
bool isFunctionLike() const { return isFunctionLike_; }
std::size_t argumentCount() const { return argumentCount_; }
@@ -39,7 +39,7 @@
bool set_isDisabled(bool disable);
- TokenSequence Apply(const std::vector<TokenSequence> &args, AllSources *);
+ TokenSequence Apply(const std::vector<TokenSequence> &args, AllSources &);
private:
static TokenSequence Tokenize(const std::vector<std::string> &argNames,
@@ -56,17 +56,13 @@
// Preprocessing state
class Preprocessor {
public:
- explicit Preprocessor(AllSources *);
+ explicit Preprocessor(AllSources &);
void Define(std::string macro, std::string value);
void Undefine(std::string macro);
- // When the input contains macros to be replaced, the new token sequence
- // is appended to the output and the returned value is true. When
- // no macro replacement is necessary, the output is unmodified and the
- // return value is false.
- bool MacroReplacement(
- const TokenSequence &, const Prescanner &, TokenSequence *);
+ std::optional<TokenSequence> MacroReplacement(
+ const TokenSequence &, const Prescanner &);
// Implements a preprocessor directive.
void Directive(const TokenSequence &, Prescanner *);
@@ -83,7 +79,7 @@
bool IsIfPredicateTrue(const TokenSequence &expr, std::size_t first,
std::size_t exprTokens, Prescanner *);
- AllSources *allSources_;
+ AllSources &allSources_;
std::list<std::string> names_;
std::unordered_map<CharBlock, Definition> definitions_;
std::stack<CanDeadElseAppear> ifStack_;
diff --git a/flang/lib/parser/prescan.cc b/flang/lib/parser/prescan.cc
index 131e35d..8cd94fb 100644
--- a/flang/lib/parser/prescan.cc
+++ b/flang/lib/parser/prescan.cc
@@ -15,7 +15,7 @@
namespace parser {
Prescanner::Prescanner(
- Messages *messages, CookedSource *cooked, Preprocessor *preprocessor)
+ Messages &messages, CookedSource &cooked, Preprocessor &preprocessor)
: messages_{messages}, cooked_{cooked}, preprocessor_{preprocessor} {}
Prescanner::Prescanner(const Prescanner &that)
@@ -28,53 +28,131 @@
compilerDirectiveBloomFilter_{that.compilerDirectiveBloomFilter_},
compilerDirectiveSentinels_{that.compilerDirectiveSentinels_} {}
+static void NormalizeCompilerDirectiveCommentMarker(TokenSequence *dir) {
+ char *p{dir->GetMutableCharData()};
+ char *limit{p + dir->SizeInChars()};
+ for (; p < limit; ++p) {
+ if (*p != ' ') {
+ CHECK(*p == '*' || *p == 'c' || *p == 'C' || *p == '!');
+ *p = '!';
+ return;
+ }
+ }
+ CHECK(!"compiler directive all blank");
+}
+
bool Prescanner::Prescan(ProvenanceRange range) {
- AllSources *allSources{cooked_->allSources()};
- ProvenanceRange around{allSources->GetContiguousRangeAround(range)};
+ AllSources &allSources{cooked_.allSources()};
+ ProvenanceRange around{allSources.GetContiguousRangeAround(range)};
startProvenance_ = range.start();
std::size_t offset{0};
- const SourceFile *source{
- allSources->GetSourceFile(startProvenance_, &offset)};
+ const SourceFile *source{allSources.GetSourceFile(startProvenance_, &offset)};
CHECK(source != nullptr);
start_ = source->content() + offset;
limit_ = start_ + range.size();
lineStart_ = start_;
- BeginSourceLine(lineStart_);
- TokenSequence tokens, preprocessed;
+ const bool beganInFixedForm{inFixedForm_};
while (lineStart_ < limit_) {
- char sentinel[8];
- if (CommentLinesAndPreprocessorDirectives(sentinel) &&
- lineStart_ >= limit_) {
- break;
+ Statement();
+ }
+ if (inFixedForm_ != beganInFixedForm) {
+ std::string dir{"!dir$ "};
+ if (beganInFixedForm) {
+ dir += "fixed";
+ } else {
+ dir += "free";
}
+ dir += '\n';
+ TokenSequence tokens{dir, allSources.AddCompilerInsertion(dir).start()};
+ tokens.Emit(&cooked_);
+ }
+ return !anyFatalErrors_;
+}
+
+void Prescanner::Statement() {
+ TokenSequence tokens;
+ LineClassification line{ClassifyLine(lineStart_)};
+ switch (line.kind) {
+ case LineClassification::Kind::Comment: NextLine(); return;
+ case LineClassification::Kind::Include:
+ FortranInclude(lineStart_ + line.payloadOffset);
+ NextLine();
+ return;
+ case LineClassification::Kind::PreprocessorDirective:
+ if (std::optional<TokenSequence> toks{TokenizePreprocessorDirective()}) {
+ preprocessor_.Directive(*toks, this);
+ }
+ return;
+ case LineClassification::Kind::CompilerDirective:
+ directiveSentinel_ = line.sentinel;
+ CHECK(directiveSentinel_ != nullptr);
+ BeginSourceLineAndAdvance();
+ if (inFixedForm_) {
+ CHECK(*at_ == '!' || *at_ == '*' || *at_ == 'c' || *at_ == 'C');
+ } else {
+ while (*at_ == ' ' || *at_ == '\t') {
+ ++at_;
+ }
+ CHECK(*at_ == '!');
+ }
+ tokens.PutNextTokenChar('!', GetCurrentProvenance());
+ ++at_, ++column_;
+ for (const char *sp{directiveSentinel_}; *sp != '\0';
+ ++sp, ++at_, ++column_) {
+ tokens.PutNextTokenChar(*sp, GetCurrentProvenance());
+ }
+ tokens.CloseToken();
+ break;
+ case LineClassification::Kind::Source:
BeginSourceLineAndAdvance();
if (inFixedForm_) {
LabelField(&tokens);
} else {
SkipSpaces();
}
- while (NextToken(&tokens)) {
- }
- Provenance newlineProvenance{GetCurrentProvenance()};
- if (preprocessor_->MacroReplacement(tokens, *this, &preprocessed)) {
- preprocessed.PutNextTokenChar('\n', newlineProvenance);
- preprocessed.CloseToken();
- const char *ppd{preprocessed.data()};
- if (IsFixedFormCompilerDirectiveLine(ppd, sentinel) ||
- IsFreeFormCompilerDirectiveLine(ppd, sentinel) ||
- !(IsFixedFormCommentLine(ppd) ||
- IsFreeFormComment(ppd))) {
- preprocessed.pop_back(); // clip the newline added above
- preprocessed.EmitLowerCase(cooked_);
- }
- preprocessed.clear();
- } else {
- tokens.EmitLowerCase(cooked_);
- }
- tokens.clear();
- cooked_->Put('\n', newlineProvenance);
+ break;
}
- return !anyFatalErrors_;
+
+ while (NextToken(&tokens)) {
+ }
+
+ Provenance newlineProvenance{GetCurrentProvenance()};
+ if (std::optional<TokenSequence> preprocessed{
+ preprocessor_.MacroReplacement(tokens, *this)}) {
+ // Reprocess the preprocessed line.
+ preprocessed->PutNextTokenChar('\n', newlineProvenance);
+ preprocessed->CloseToken();
+ const char *ppd{preprocessed->ToCharBlock().begin()};
+ LineClassification ppl{ClassifyLine(ppd)};
+ switch (ppl.kind) {
+ case LineClassification::Kind::Comment: break;
+ case LineClassification::Kind::Include:
+ FortranInclude(ppd + ppl.payloadOffset);
+ break;
+ case LineClassification::Kind::PreprocessorDirective:
+ Complain("preprocessed line looks like a preprocessor directive"_en_US,
+ preprocessed->GetProvenanceRange().start());
+ preprocessed->ToLowerCase().Emit(&cooked_);
+ break;
+ case LineClassification::Kind::CompilerDirective:
+ NormalizeCompilerDirectiveCommentMarker(&*preprocessed);
+ preprocessed->ToLowerCase();
+ SourceFormChange(preprocessed->ToString());
+ preprocessed->Emit(&cooked_);
+ break;
+ case LineClassification::Kind::Source:
+ preprocessed->ToLowerCase().Emit(&cooked_);
+ break;
+ }
+ } else {
+ tokens.ToLowerCase();
+ if (line.kind == LineClassification::Kind::CompilerDirective) {
+ SourceFormChange(tokens.ToString());
+ }
+ tokens.Emit(&cooked_);
+ cooked_.Put('\n', newlineProvenance);
+ }
+ directiveSentinel_ = nullptr;
}
TokenSequence Prescanner::TokenizePreprocessorDirective() {
@@ -92,29 +170,29 @@
Message &Prescanner::Error(Message &&message) {
anyFatalErrors_ = true;
- return messages_->Put(std::move(message));
+ return messages_.Put(std::move(message));
}
Message &Prescanner::Error(MessageFixedText text, Provenance p) {
anyFatalErrors_ = true;
- return messages_->Put({p, text});
+ return messages_.Put({p, text});
}
Message &Prescanner::Error(MessageFormattedText &&text, Provenance p) {
anyFatalErrors_ = true;
- return messages_->Put({p, std::move(text)});
+ return messages_.Put({p, std::move(text)});
}
Message &Prescanner::Complain(Message &&message) {
- return messages_->Put(std::move(message));
+ return messages_.Put(std::move(message));
}
Message &Prescanner::Complain(MessageFixedText text, Provenance p) {
- return messages_->Put({p, text});
+ return messages_.Put({p, text});
}
Message &Prescanner::Complain(MessageFormattedText &&text, Provenance p) {
- return messages_->Put({p, std::move(text)});
+ return messages_.Put({p, std::move(text)});
}
void Prescanner::NextLine() {
@@ -180,6 +258,8 @@
if ((inFixedForm_ && column_ > fixedFormColumnLimit_ &&
!tabInCurrentLine_) ||
(*at_ == '!' && !inCharLiteral_)) {
+ // Skip remainder of fixed form line due to '!' comment marker or
+ // hitting the right margin.
while (*at_ != '\n') {
++at_;
}
@@ -421,9 +501,6 @@
}
bool Prescanner::IsFixedFormCommentLine(const char *start) const {
- if (start >= limit_ || !inFixedForm_) {
- return false;
- }
const char *p{start};
char ch{*p};
if (ch == '*' || ch == 'C' || ch == 'c' ||
@@ -455,33 +532,35 @@
}
bool Prescanner::IsFreeFormComment(const char *p) const {
- if (p >= limit_ || inFixedForm_) {
- return false;
- }
while (*p == ' ' || *p == '\t') {
++p;
}
return *p == '!' || *p == '\n';
}
-bool Prescanner::IncludeLine(const char *p) {
- if (p >= limit_) {
- return false;
- }
- const char *start{p};
+std::optional<std::size_t> Prescanner::IsIncludeLine(const char *start) const {
+ const char *p{start};
while (*p == ' ' || *p == '\t') {
++p;
}
for (char ch : "include"s) {
if (ToLowerCaseLetter(*p++) != ch) {
- return false;
+ return {};
}
}
while (*p == ' ' || *p == '\t') {
++p;
}
- if (*p != '"' && *p != '\'') {
- return false;
+ if (*p == '"' || *p == '\'') {
+ return {p - start};
+ }
+ return {};
+}
+
+bool Prescanner::FortranInclude(const char *firstQuote) {
+ const char *p{firstQuote};
+ while (*p != '"' && *p != '\'') {
+ ++p;
}
char quote{*p};
std::string path;
@@ -504,15 +583,15 @@
Complain("excess characters after path name"_en_US, GetProvenance(p));
}
std::stringstream error;
- Provenance provenance{GetProvenance(start)};
- AllSources *allSources{cooked_->allSources()};
- const SourceFile *currentFile{allSources->GetSourceFile(provenance)};
+ Provenance provenance{GetProvenance(lineStart_)};
+ AllSources &allSources{cooked_.allSources()};
+ const SourceFile *currentFile{allSources.GetSourceFile(provenance)};
if (currentFile != nullptr) {
- allSources->PushSearchPathDirectory(DirectoryName(currentFile->path()));
+ allSources.PushSearchPathDirectory(DirectoryName(currentFile->path()));
}
- const SourceFile *included{allSources->Open(path, &error)};
+ const SourceFile *included{allSources.Open(path, &error)};
if (currentFile != nullptr) {
- allSources->PopSearchPathDirectory();
+ allSources.PopSearchPathDirectory();
}
if (included == nullptr) {
Error(MessageFormattedText("INCLUDE: %s"_en_US, error.str().data()),
@@ -523,18 +602,15 @@
return true;
}
ProvenanceRange includeLineRange{
- provenance, static_cast<std::size_t>(p - start)};
+ provenance, static_cast<std::size_t>(p - lineStart_)};
ProvenanceRange fileRange{
- allSources->AddIncludedFile(*included, includeLineRange)};
+ allSources.AddIncludedFile(*included, includeLineRange)};
anyFatalErrors_ |= !Prescanner{*this}.Prescan(fileRange);
return true;
}
bool Prescanner::IsPreprocessorDirectiveLine(const char *start) const {
const char *p{start};
- if (p >= limit_) {
- return false;
- }
for (; *p == ' '; ++p) {
}
if (*p == '#') {
@@ -549,44 +625,14 @@
return IsPreprocessorDirectiveLine(lineStart_);
}
-bool Prescanner::CommentLines() {
- bool any{false};
- char sentinel[8];
+void Prescanner::SkipCommentLines() {
while (lineStart_ < limit_) {
- if (IsFixedFormCompilerDirectiveLine(lineStart_, sentinel) ||
- IsFreeFormCompilerDirectiveLine(lineStart_, sentinel) ||
- !(IsFixedFormCommentLine(lineStart_) ||
- IsFreeFormComment(lineStart_))) {
+ LineClassification line{ClassifyLine(lineStart_)};
+ if (line.kind != LineClassification::Kind::Comment) {
break;
}
NextLine();
- any = true;
}
- return any;
-}
-
-bool Prescanner::CommentLinesAndPreprocessorDirectives(char *sentinel) {
- bool any{false};
- *sentinel = '\0';
- while (lineStart_ < limit_) {
- if (IsFixedFormCompilerDirectiveLine(lineStart_, sentinel) ||
- IsFreeFormCompilerDirectiveLine(lineStart_, sentinel)) {
- break;
- }
- if (IsFixedFormCommentLine(lineStart_) || IsFreeFormComment(lineStart_) ||
- IncludeLine(lineStart_)) {
- NextLine();
- } else if (IsPreprocessorDirectiveLine(lineStart_)) {
- if (std::optional<TokenSequence> tokens{
- TokenizePreprocessorDirective()}) {
- preprocessor_->Directive(*tokens, this);
- }
- } else {
- break;
- }
- any = true;
- }
- return any;
}
const char *Prescanner::FixedFormContinuationLine() {
@@ -595,6 +641,34 @@
return nullptr;
}
tabInCurrentLine_ = false;
+ char col1{*p};
+ if (directiveSentinel_ != nullptr) {
+ // Must be a continued compiler directive.
+ if (col1 != '!' && col1 != '*' && col1 != 'c' && col1 != 'C') {
+ return nullptr;
+ }
+ int j{1};
+ for (; j < 5; ++j) {
+ char ch{directiveSentinel_[j - 1]};
+ if (ch == '\0') {
+ break;
+ }
+ if (ch != ToLowerCaseLetter(p[j])) {
+ return nullptr;
+ }
+ }
+ for (; j < 5; ++j) {
+ if (p[j] != ' ') {
+ return nullptr;
+ }
+ }
+ char col6{p[5]};
+ if (col6 != '\n' && col6 != '\t' && col6 != ' ' && col6 != '0') {
+ return p + 6;
+ }
+ return nullptr;
+ }
+ // Normal case: not in a compiler directive.
if (*p == '&') {
return p + 1; // extension; TODO: emit warning with -Mstandard
}
@@ -616,10 +690,10 @@
bool Prescanner::FixedFormContinuation() {
// N.B. We accept '&' as a continuation indicator (even) in fixed form.
- if (!inFixedForm_ || (*at_ == '&' && inCharLiteral_)) {
+ if (*at_ == '&' && inCharLiteral_) {
return false;
}
- CommentLines();
+ SkipCommentLines();
const char *cont{FixedFormContinuationLine()};
if (cont == nullptr) {
return false;
@@ -631,12 +705,6 @@
}
bool Prescanner::FreeFormContinuation() {
- if (inFixedForm_) {
- return false;
- }
- while (*at_ == ' ' || *at_ == '\t') {
- ++at_;
- }
const char *p{at_};
bool ampersand{*p == '&'};
if (ampersand) {
@@ -646,89 +714,107 @@
if (*p != '\n' && (inCharLiteral_ || *p != '!')) {
return false;
}
- CommentLines();
+ SkipCommentLines();
p = lineStart_;
if (p >= limit_) {
return false;
}
- int column{1};
for (; *p == ' ' || *p == '\t'; ++p) {
- ++column;
}
- if (*p == '&') {
- ++p;
- ++column;
- } else if (ampersand || delimiterNesting_ > 0) {
- if (p > lineStart_) {
- --p;
- --column;
+ if (directiveSentinel_ != nullptr) {
+ // Look for a continued compiler directive.
+ if (*p++ != '!') {
+ return false;
+ }
+ for (const char *s{directiveSentinel_}; *s != '\0'; ++p, ++s) {
+ if (*s != ToLowerCaseLetter(*p)) {
+ return false;
+ }
+ }
+ for (; *p == ' ' || *p == '\t'; ++p) {
+ }
+ if (*p == '&') {
+ ++p;
+ } else if (!ampersand) {
+ return false;
}
} else {
- return false; // not a continuation
+ // Normal case (not a compiler directive)
+ if (*p == '&') {
+ ++p;
+ } else if (ampersand || delimiterNesting_ > 0) {
+ if (p > lineStart_) {
+ --p;
+ }
+ } else {
+ return false; // not a continuation
+ }
}
at_ = p;
- column_ = column;
tabInCurrentLine_ = false;
NextLine();
return true;
}
-bool Prescanner::IsFixedFormCompilerDirectiveLine(const char *start,
- char *sentinel) const {
- *sentinel = '\0';
- if (start >= limit_ || !inFixedForm_) {
- return false;
- }
+std::optional<Prescanner::LineClassification>
+Prescanner::IsFixedFormCompilerDirectiveLine(const char *start) const {
const char *p{start};
- char c1{*p};
- if (!(c1 == '*' || c1 == 'C' || c1 == 'c' || c1 == '!')) {
- return false;
+ char col1{*p};
+ if (col1 != '*' && col1 != 'C' && col1 != 'c' && col1 != '!') {
+ return {};
}
- char *sp{sentinel};
- ++p;
+ char sentinel[5], *sp{sentinel};
for (int col{2}; col < 6; ++col) {
char ch{*++p};
if (ch == '\n' || ch == '\t') {
- return false;
+ return {};
}
if (ch != ' ') {
*sp++ = ToLowerCaseLetter(ch);
}
}
- if (*p != ' ' && *p != '0') {
- return false; // continuation card for directive
- }
*sp = '\0';
- return IsCompilerDirectiveSentinel(sentinel);
+ if (const char *sp{IsCompilerDirectiveSentinel(sentinel)}) {
+ return {
+ LineClassification{LineClassification::Kind::CompilerDirective, 6, sp}};
+ }
+ return {};
}
-bool Prescanner::IsFreeFormCompilerDirectiveLine(const char *start,
- char *sentinel) const {
- *sentinel = '\0';
- if (start >= limit_ || inFixedForm_) {
- return false;
- }
+std::optional<Prescanner::LineClassification>
+Prescanner::IsFreeFormCompilerDirectiveLine(const char *start) const {
+ char sentinel[8];
const char *p{start};
while (*p == ' ' || *p == '\t') {
++p;
}
if (*p++ != '!') {
- return false;
+ return {};
}
- for (int j{0}; j < 5; ++p, ++j) {
- if (*p == '\n' || *p == '&') {
+ for (std::size_t j{0}; j + 1 < sizeof sentinel; ++p, ++j) {
+ if (*p == '\n') {
break;
}
- if (*p == ' ' || *p == '\t') {
+ if (*p == ' ' || *p == '\t' || *p == '&') {
if (j == 0) {
break;
}
sentinel[j] = '\0';
- return IsCompilerDirectiveSentinel(sentinel);
+ for (++p; *p == ' ' || *p == '\t'; ++p) {
+ }
+ if (*p == '!') {
+ break;
+ }
+ if (const char *sp{IsCompilerDirectiveSentinel(sentinel)}) {
+ std::size_t offset = p - start;
+ return {LineClassification{
+ LineClassification::Kind::CompilerDirective, offset, sp}};
+ }
+ break;
}
sentinel[j] = ToLowerCaseLetter(*p);
}
- return false;
+ return {};
}
Prescanner &Prescanner::AddCompilerDirectiveSentinel(const std::string &dir) {
@@ -742,16 +828,54 @@
return *this;
}
-bool Prescanner::IsCompilerDirectiveSentinel(const char *s) const {
+const char *Prescanner::IsCompilerDirectiveSentinel(const char *s) const {
std::uint64_t packed{0};
std::size_t n{0};
for (; s[n] != '\0'; ++n) {
packed = (packed << 8) | (s[n] & 0xff);
}
- return n > 0 && compilerDirectiveBloomFilter_.test(packed % prime1) &&
- compilerDirectiveBloomFilter_.test(packed % prime2) &&
- compilerDirectiveSentinels_.find(std::string(s, n)) !=
- compilerDirectiveSentinels_.end();
+ if (n == 0 || !compilerDirectiveBloomFilter_.test(packed % prime1) ||
+ !compilerDirectiveBloomFilter_.test(packed % prime2)) {
+ return nullptr;
+ }
+ const auto iter = compilerDirectiveSentinels_.find(std::string(s, n));
+ return iter == compilerDirectiveSentinels_.end() ? nullptr : iter->data();
+}
+
+Prescanner::LineClassification Prescanner::ClassifyLine(
+ const char *start) const {
+ if (inFixedForm_) {
+ if (std::optional<LineClassification> lc{
+ IsFixedFormCompilerDirectiveLine(start)}) {
+ return std::move(*lc);
+ }
+ if (IsFixedFormCommentLine(start)) {
+ return {LineClassification::Kind::Comment};
+ }
+ } else {
+ if (std::optional<LineClassification> lc{
+ IsFreeFormCompilerDirectiveLine(start)}) {
+ return std::move(*lc);
+ }
+ if (IsFreeFormComment(start)) {
+ return {LineClassification::Kind::Comment};
+ }
+ }
+ if (std::optional<std::size_t> quoteOffset{IsIncludeLine(start)}) {
+ return {LineClassification::Kind::Include, *quoteOffset};
+ }
+ if (IsPreprocessorDirectiveLine(start)) {
+ return {LineClassification::Kind::PreprocessorDirective};
+ }
+ return {LineClassification::Kind::Source};
+}
+
+void Prescanner::SourceFormChange(std::string &&dir) {
+ if (dir == "!dir$ free") {
+ inFixedForm_ = false;
+ } else if (dir == "!dir$ fixed") {
+ inFixedForm_ = true;
+ }
}
} // namespace parser
} // namespace Fortran
diff --git a/flang/lib/parser/prescan.h b/flang/lib/parser/prescan.h
index 2b3175d..ee978b1 100644
--- a/flang/lib/parser/prescan.h
+++ b/flang/lib/parser/prescan.h
@@ -25,12 +25,12 @@
class Prescanner {
public:
- Prescanner(Messages *, CookedSource *, Preprocessor *);
+ Prescanner(Messages &, CookedSource &, Preprocessor &);
Prescanner(const Prescanner &);
bool anyFatalErrors() const { return anyFatalErrors_; }
void set_anyFatalErrors() { anyFatalErrors_ = true; }
- Messages *messages() const { return messages_; }
+ Messages &messages() const { return messages_; }
Prescanner &set_fixedForm(bool yes) {
inFixedForm_ = yes;
@@ -56,6 +56,7 @@
Prescanner &AddCompilerDirectiveSentinel(const std::string &);
bool Prescan(ProvenanceRange);
+ void Statement();
void NextLine();
// Callbacks for use by Preprocessor.
@@ -72,6 +73,22 @@
Message &Complain(MessageFormattedText &&, Provenance);
private:
+ struct LineClassification {
+ enum class Kind {
+ Comment,
+ PreprocessorDirective,
+ Include,
+ CompilerDirective,
+ Source
+ };
+ LineClassification(Kind k, std::size_t po = 0, const char *s = nullptr)
+ : kind{k}, payloadOffset{po}, sentinel{s} {}
+ LineClassification(LineClassification &&) = default;
+ Kind kind;
+ std::size_t payloadOffset; // byte offset of content
+ const char *sentinel; // if it's a compiler directive
+ };
+
void BeginSourceLine(const char *at) {
at_ = at;
column_ = 1;
@@ -94,8 +111,7 @@
}
void EmitInsertedChar(TokenSequence *tokens, char ch) {
- Provenance provenance{
- cooked_->allSources()->CompilerInsertionProvenance(ch)};
+ Provenance provenance{cooked_.allSources().CompilerInsertionProvenance(ch)};
tokens->PutNextTokenChar(ch, provenance);
}
@@ -113,46 +129,55 @@
void QuotedCharacterLiteral(TokenSequence *);
void Hollerith(TokenSequence *, int);
bool PadOutCharacterLiteral(TokenSequence *);
- bool CommentLines();
- bool CommentLinesAndPreprocessorDirectives(char *sentinel);
+ void SkipCommentLines();
bool IsFixedFormCommentLine(const char *) const;
bool IsFreeFormComment(const char *) const;
- bool IncludeLine(const char *);
+ std::optional<std::size_t> IsIncludeLine(const char *) const;
+ bool FortranInclude(const char *quote);
bool IsPreprocessorDirectiveLine(const char *) const;
const char *FixedFormContinuationLine();
bool FixedFormContinuation();
bool FreeFormContinuation();
- bool IsFixedFormCompilerDirectiveLine(const char *, char *sentinel) const;
- bool IsFreeFormCompilerDirectiveLine(const char *, char *sentinel) const;
- bool IsCompilerDirectiveSentinel(const char *) const;
+ std::optional<LineClassification> IsFixedFormCompilerDirectiveLine(
+ const char *) const;
+ std::optional<LineClassification> IsFreeFormCompilerDirectiveLine(
+ const char *) const;
+ const char *IsCompilerDirectiveSentinel(const char *) const;
+ LineClassification ClassifyLine(const char *) const;
+ void SourceFormChange(std::string &&);
- Messages *messages_;
- CookedSource *cooked_;
- Preprocessor *preprocessor_;
-
- Provenance startProvenance_;
- const char *start_{nullptr}; // beginning of current source file content
- const char *limit_{nullptr}; // first address after end of current source
- const char *at_{nullptr}; // next character to process; < lineStart_
- int column_{1}; // card image column position of next character
- const char *lineStart_{nullptr}; // next line to process; <= limit_
- bool tabInCurrentLine_{false};
- bool preventHollerith_{false};
+ Messages &messages_;
+ CookedSource &cooked_;
+ Preprocessor &preprocessor_;
bool anyFatalErrors_{false};
- bool inCharLiteral_{false};
- bool inPreprocessorDirective_{false};
bool inFixedForm_{false};
int fixedFormColumnLimit_{72};
Encoding encoding_{Encoding::UTF8};
bool enableOldDebugLines_{false};
bool enableBackslashEscapesInCharLiterals_{true};
int delimiterNesting_{0};
- Provenance spaceProvenance_{
- cooked_->allSources()->CompilerInsertionProvenance(' ')};
- Provenance backslashProvenance_{
- cooked_->allSources()->CompilerInsertionProvenance('\\')};
- ProvenanceRange sixSpaceProvenance_{
- cooked_->allSources()->AddCompilerInsertion(" "s)};
+
+ Provenance startProvenance_;
+ const char *start_{nullptr}; // beginning of current source file content
+ const char *limit_{nullptr}; // first address after end of current source
+ const char *lineStart_{nullptr}; // next line to process; <= limit_
+ const char *directiveSentinel_{nullptr}; // current compiler directive
+
+ // This data members are state for processing the source line containing
+ // "at_", which goes to up to the newline character before "lineStart_".
+ const char *at_{nullptr}; // next character to process; < lineStart_
+ int column_{1}; // card image column position of next character
+ bool tabInCurrentLine_{false};
+ bool preventHollerith_{false};
+ bool inCharLiteral_{false};
+ bool inPreprocessorDirective_{false};
+
+ const Provenance spaceProvenance_{
+ cooked_.allSources().CompilerInsertionProvenance(' ')};
+ const Provenance backslashProvenance_{
+ cooked_.allSources().CompilerInsertionProvenance('\\')};
+ const ProvenanceRange sixSpaceProvenance_{
+ cooked_.allSources().AddCompilerInsertion(" "s)};
// To avoid probing the set of active compiler directive sentinel strings
// on every comment line, they're checked first with a cheap Bloom filter.
diff --git a/flang/lib/parser/provenance.cc b/flang/lib/parser/provenance.cc
index 3f47eee..76aabee 100644
--- a/flang/lib/parser/provenance.cc
+++ b/flang/lib/parser/provenance.cc
@@ -33,7 +33,7 @@
}
ProvenanceRange OffsetToProvenanceMappings::Map(std::size_t at) const {
- CHECK(!provenanceMap_.empty());
+ // CHECK(!provenanceMap_.empty());
std::size_t low{0}, count{provenanceMap_.size()};
while (count > 1) {
std::size_t mid{low + (count >> 1)};
@@ -273,8 +273,7 @@
void CookedSource::Marshal() {
CHECK(provenanceMap_.size() == buffer_.size());
- provenanceMap_.Put(
- allSources_->AddCompilerInsertion("(after end of source)"));
+ provenanceMap_.Put(allSources_.AddCompilerInsertion("(after end of source)"));
data_.resize(buffer_.size());
char *p{&data_[0]};
for (char ch : buffer_) {
@@ -328,7 +327,7 @@
void CookedSource::Dump(std::ostream &o) const {
o << "CookedSource:\n";
- allSources_->Dump(o);
+ allSources_.Dump(o);
o << "CookedSource::provenanceMap_:\n";
provenanceMap_.Dump(o);
}
diff --git a/flang/lib/parser/provenance.h b/flang/lib/parser/provenance.h
index afbb97f..2b9e6de 100644
--- a/flang/lib/parser/provenance.h
+++ b/flang/lib/parser/provenance.h
@@ -166,13 +166,18 @@
class CookedSource {
public:
- explicit CookedSource(AllSources *sources) : allSources_{sources} {}
+ explicit CookedSource(AllSources &sources) : allSources_{sources} {}
std::size_t size() const { return data_.size(); }
const char &operator[](std::size_t n) const { return data_[n]; }
const char &at(std::size_t n) const { return data_.at(n); }
- AllSources *allSources() const { return allSources_; }
+ AllSources &allSources() const { return allSources_; }
+
+ bool IsValid(const char *p) const {
+ return p >= &data_.front() && p <= &data_.back() + 1;
+ }
+ bool IsValid(Provenance p) const { return allSources_.IsValid(p); }
ProvenanceRange GetProvenance(const char *) const;
void Identify(std::ostream &, const char *) const;
@@ -190,7 +195,7 @@
void Dump(std::ostream &) const;
private:
- AllSources *allSources_;
+ AllSources &allSources_;
CharBuffer buffer_; // before Marshal()
std::vector<char> data_; // all of it, prescanned and preprocessed
OffsetToProvenanceMappings provenanceMap_;
diff --git a/flang/lib/parser/token-parsers.h b/flang/lib/parser/token-parsers.h
index e1a6f0e..88ab79c 100644
--- a/flang/lib/parser/token-parsers.h
+++ b/flang/lib/parser/token-parsers.h
@@ -496,6 +496,27 @@
return "," >> nonemptyList(p) / "::" ||
("::"_tok || !","_tok) >> defaulted(cut >> nonemptyList(p));
}
+
+// Compiler directives can switch the parser between fixed and free form.
+constexpr struct FormDirectivesAndEmptyLines {
+ using resultType = Success;
+ static std::optional<Success> Parse(ParseState *state) {
+ while (!state->IsAtEnd()) {
+ const char *at{state->GetLocation()};
+ static const char fixed[] = "!dir$ fixed\n", free[] = "!dir$ free\n";
+ if (*at == '\n') {
+ state->UncheckedAdvance();
+ } else if (std::memcmp(at, fixed, sizeof fixed - 1) == 0) {
+ state->set_inFixedForm(true).UncheckedAdvance(sizeof fixed - 1);
+ } else if (std::memcmp(at, free, sizeof free - 1) == 0) {
+ state->set_inFixedForm(false).UncheckedAdvance(sizeof free - 1);
+ } else {
+ break;
+ }
+ }
+ return {Success{}};
+ }
+} skipEmptyLines;
} // namespace parser
} // namespace Fortran
#endif // FORTRAN_PARSER_TOKEN_PARSERS_H_
diff --git a/flang/lib/parser/token-sequence.cc b/flang/lib/parser/token-sequence.cc
index fec6f42..ae66166 100644
--- a/flang/lib/parser/token-sequence.cc
+++ b/flang/lib/parser/token-sequence.cc
@@ -40,8 +40,9 @@
void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
std::size_t offset{0};
- for (std::size_t j{0}; j < that.size(); ++j) {
- CharBlock tok{that[j]};
+ std::size_t tokens{that.SizeInTokens()};
+ for (std::size_t j{0}; j < tokens; ++j) {
+ CharBlock tok{that.TokenAt(j)};
Put(tok, range.OffsetMember(offset));
offset += tok.size();
}
@@ -53,12 +54,12 @@
ProvenanceRange provenance;
std::size_t offset{0};
for (; tokens-- > 0; ++at) {
- CharBlock tok{that[at]};
+ CharBlock tok{that.TokenAt(at)};
std::size_t tokBytes{tok.size()};
for (std::size_t j{0}; j < tokBytes; ++j) {
if (offset == provenance.size()) {
- offset = 0;
provenance = that.provenances_.Map(that.start_[at] + j);
+ offset = 0;
}
PutNextTokenChar(tok[j], provenance.OffsetMember(offset++));
}
@@ -86,55 +87,53 @@
Put(ss.str(), provenance);
}
-void TokenSequence::EmitLowerCase(CookedSource *cooked) const {
+TokenSequence &TokenSequence::ToLowerCase() {
std::size_t tokens{start_.size()};
std::size_t chars{char_.size()};
std::size_t atToken{0};
for (std::size_t j{0}; j < chars;) {
std::size_t nextStart{atToken + 1 < tokens ? start_[++atToken] : chars};
- const char *p{&char_[j]}, *limit{&char_[nextStart]};
+ char *p{&char_[j]}, *limit{&char_[nextStart]};
j = nextStart;
if (IsDecimalDigit(*p)) {
while (p < limit && IsDecimalDigit(*p)) {
- cooked->Put(*p++);
+ ++p;
}
if (p < limit && (*p == 'h' || *p == 'H')) {
// Hollerith
- cooked->Put('h');
- cooked->Put(p + 1, limit - (p + 1));
+ *p = 'h';
} else {
// exponent
- while (p < limit) {
- cooked->Put(ToLowerCaseLetter(*p++));
+ for (; p < limit; ++p) {
+ *p = ToLowerCaseLetter(*p);
}
}
} else if (limit[-1] == '\'' || limit[-1] == '"') {
if (*p == limit[-1]) {
// Character literal without prefix
- cooked->Put(p, limit - p);
} else if (p[1] == limit[-1]) {
// BOZX-prefixed constant
- while (p < limit) {
- cooked->Put(ToLowerCaseLetter(*p++));
+ for (; p < limit; ++p) {
+ *p = ToLowerCaseLetter(*p);
}
} else {
// Kanji NC'...' character literal or literal with kind-param prefix.
- while (*p != limit[-1]) {
- cooked->Put(ToLowerCaseLetter(*p++));
+ for (; *p != limit[-1]; ++p) {
+ *p = ToLowerCaseLetter(*p);
}
- cooked->Put(p, limit - p);
}
} else {
- while (p < limit) {
- cooked->Put(ToLowerCaseLetter(*p++));
+ for (; p < limit; ++p) {
+ *p = ToLowerCaseLetter(*p);
}
}
}
- cooked->PutProvenanceMappings(provenances_);
+ return *this;
}
-std::string TokenSequence::ToString() const {
- return {&char_[0], char_.size()};
+void TokenSequence::Emit(CookedSource *cooked) const {
+ cooked->Put(&char_[0], char_.size());
+ cooked->PutProvenanceMappings(provenances_);
}
Provenance TokenSequence::GetTokenProvenance(
diff --git a/flang/lib/parser/token-sequence.h b/flang/lib/parser/token-sequence.h
index 7053bf6..d336b38 100644
--- a/flang/lib/parser/token-sequence.h
+++ b/flang/lib/parser/token-sequence.h
@@ -2,7 +2,7 @@
#define FORTRAN_PARSER_TOKEN_SEQUENCE_H_
// A buffer class capable of holding a contiguous sequence of characters
-// that has been partitioned into preprocessing tokens, along with their
+// and a partitioning thereof into preprocessing tokens, along with their
// associated provenances.
#include "char-block.h"
@@ -44,17 +44,23 @@
return *this;
}
- CharBlock operator[](std::size_t token) const {
- return {&char_[start_[token]], TokenBytes(token)};
- }
-
bool empty() const { return start_.empty(); }
- std::size_t size() const { return start_.size(); }
- const char *data() const { return &char_[0]; }
void clear();
void pop_back();
void shrink_to_fit();
+ std::size_t SizeInTokens() const { return start_.size(); }
+ std::size_t SizeInChars() const { return char_.size(); }
+
+ CharBlock ToCharBlock() const { return {&char_[0], char_.size()}; }
+ std::string ToString() const { return ToCharBlock().ToString(); }
+
+ CharBlock TokenAt(std::size_t token) const {
+ return {&char_[start_.at(token)], TokenBytes(token)};
+ }
+
+ char CharAt(std::size_t j) const { return char_.at(j); }
+
void PutNextTokenChar(char ch, Provenance provenance) {
char_.emplace_back(ch);
provenances_.Put({provenance, 1});
@@ -77,7 +83,6 @@
void Put(const CharBlock &, Provenance);
void Put(const std::string &, Provenance);
void Put(const std::stringstream &, Provenance);
- std::string ToString() const;
Provenance GetTokenProvenance(
std::size_t token, std::size_t offset = 0) const;
ProvenanceRange GetTokenProvenanceRange(
@@ -86,7 +91,9 @@
std::size_t token, std::size_t tokens = 1) const;
ProvenanceRange GetProvenanceRange() const;
- void EmitLowerCase(CookedSource *) const;
+ char *GetMutableCharData() { return &char_[0]; }
+ TokenSequence &ToLowerCase();
+ void Emit(CookedSource *) const;
private:
std::size_t TokenBytes(std::size_t token) const {
diff --git a/flang/lib/parser/unparse.cc b/flang/lib/parser/unparse.cc
index 0d98419..2243d2e 100644
--- a/flang/lib/parser/unparse.cc
+++ b/flang/lib/parser/unparse.cc
@@ -1941,7 +1941,30 @@
return false;
}
- // Extensions and deprecated constructs
+ // Directives, extensions, and deprecated constructs
+ bool Pre(const CompilerDirective &x) {
+ std::visit(
+ visitors{[&](const std::list<CompilerDirective::IgnoreTKR> &tkr) {
+ Word("!DIR$ IGNORE_TKR");
+ Walk(" ", tkr, ", ");
+ },
+ [&](const CompilerDirective::IVDEP &) { Word("!DIR$ IVDEP\n"); }},
+ x.u);
+ Put('\n');
+ return false;
+ }
+ bool Pre(const CompilerDirective::IgnoreTKR &x) {
+ const auto &list = std::get<std::list<const char *>>(x.t);
+ if (!list.empty()) {
+ Put("(");
+ for (const char *tkr : list) {
+ Put(*tkr);
+ }
+ Put(") ");
+ }
+ Walk(std::get<Name>(x.t));
+ return false;
+ }
bool Pre(const BasedPointerStmt &x) {
Word("POINTER ("), Walk(std::get<0>(x.t)), Put(", ");
Walk(std::get<1>(x.t));
@@ -2150,9 +2173,7 @@
}
}
-void UnparseVisitor::Word(const std::string &str) {
- Word(str.c_str());
-}
+void UnparseVisitor::Word(const std::string &str) { Word(str.c_str()); }
void Unparse(std::ostream &out, const Program &program, Encoding encoding,
bool capitalizeKeywords) {
