Warning: This file is not a C or C++ file. It does not have highlighting.

1//===-- lib/Parser/token-sequence.h -----------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef FORTRAN_PARSER_TOKEN_SEQUENCE_H_
10#define FORTRAN_PARSER_TOKEN_SEQUENCE_H_
11
12// A buffer class capable of holding a contiguous sequence of characters
13// and a partitioning thereof into preprocessing tokens, along with their
14// associated provenances.
15
16#include "flang/Parser/char-block.h"
17#include "flang/Parser/provenance.h"
18#include <cstddef>
19#include <cstring>
20#include <string>
21#include <utility>
22#include <vector>
23
24namespace llvm {
25class raw_ostream;
26}
27
28namespace Fortran::parser {
29
30class Messages;
31class Prescanner;
32
33// Buffers a contiguous sequence of characters that has been partitioned into
34// a sequence of preprocessing tokens with provenances.
35class TokenSequence {
36public:
37 TokenSequence() {}
38 TokenSequence(const TokenSequence &that) { Put(that); }
39 TokenSequence(
40 const TokenSequence &that, std::size_t at, std::size_t count = 1) {
41 Put(that, at, count);
42 }
43 TokenSequence(TokenSequence &&that)
44 : start_{std::move(that.start_)}, nextStart_{that.nextStart_},
45 char_{std::move(that.char_)},
46 provenances_{std::move(that.provenances_)} {}
47 TokenSequence(const std::string &s, Provenance p) { Put(s, p); }
48
49 TokenSequence &operator=(const TokenSequence &that) {
50 clear();
51 Put(that);
52 return *this;
53 }
54 TokenSequence &operator=(TokenSequence &&that);
55 bool empty() const { return start_.empty(); }
56 void clear();
57 void pop_back();
58 void shrink_to_fit();
59 void swap(TokenSequence &);
60
61 std::size_t SizeInTokens() const { return start_.size(); }
62 std::size_t SizeInChars() const { return char_.size(); }
63
64 CharBlock ToCharBlock() const {
65 return char_.empty() ? CharBlock{} : CharBlock{&char_[0], char_.size()};
66 }
67 std::string ToString() const { return ToCharBlock().ToString(); }
68
69 CharBlock TokenAt(std::size_t token) const {
70 if (auto bytes{TokenBytes(token)}) {
71 return {&char_[start_.at(token)], bytes};
72 } else { // char_ could be empty
73 return {};
74 }
75 }
76 char CharAt(std::size_t j) const { return char_.at(j); }
77 CharBlock CurrentOpenToken() const {
78 return {&char_[nextStart_], char_.size() - nextStart_};
79 }
80
81 std::size_t SkipBlanks(std::size_t) const;
82
83 // True if anything remains in the sequence at & after the given offset
84 // except blanks and line-ending C++ and Fortran free-form comments.
85 bool IsAnythingLeft(std::size_t) const;
86
87 void PutNextTokenChar(char ch, Provenance provenance) {
88 char_.emplace_back(ch);
89 provenances_.Put({provenance, 1});
90 }
91
92 void CloseToken() {
93 start_.emplace_back(nextStart_);
94 nextStart_ = char_.size();
95 }
96
97 void ReopenLastToken() {
98 nextStart_ = start_.back();
99 start_.pop_back();
100 }
101
102 void Put(const TokenSequence &);
103 void Put(const TokenSequence &, ProvenanceRange);
104 void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
105 void Put(const char *, std::size_t, Provenance);
106 void Put(const CharBlock &, Provenance);
107 void Put(const std::string &, Provenance);
108 void Put(llvm::raw_string_ostream &, Provenance);
109
110 Provenance GetCharProvenance(std::size_t) const;
111 Provenance GetTokenProvenance(
112 std::size_t token, std::size_t offset = 0) const;
113 ProvenanceRange GetTokenProvenanceRange(
114 std::size_t token, std::size_t offset = 0) const;
115 ProvenanceRange GetIntervalProvenanceRange(
116 std::size_t token, std::size_t tokens = 1) const;
117 ProvenanceRange GetProvenanceRange() const;
118
119 char *GetMutableCharData() { return &char_[0]; }
120 TokenSequence &ToLowerCase();
121 bool HasBlanks(std::size_t firstChar = 0) const;
122 bool HasRedundantBlanks(std::size_t firstChar = 0) const;
123 TokenSequence &RemoveBlanks(std::size_t firstChar = 0);
124 TokenSequence &RemoveRedundantBlanks(std::size_t firstChar = 0);
125 TokenSequence &ClipComment(const Prescanner &, bool skipFirst = false);
126 const TokenSequence &CheckBadFortranCharacters(
127 Messages &, const Prescanner &) const;
128 const TokenSequence &CheckBadParentheses(Messages &) const;
129 void Emit(CookedSource &) const;
130 llvm::raw_ostream &Dump(llvm::raw_ostream &) const;
131
132private:
133 std::size_t TokenBytes(std::size_t token) const {
134 return (token + 1 >= start_.size() ? char_.size() : start_[token + 1]) -
135 start_[token];
136 }
137
138 std::vector<std::size_t> start_;
139 std::size_t nextStart_{0};
140 std::vector<char> char_;
141 OffsetToProvenanceMappings provenances_;
142};
143} // namespace Fortran::parser
144#endif // FORTRAN_PARSER_TOKEN_SEQUENCE_H_
145

Warning: This file is not a C or C++ file. It does not have highlighting.

source code of flang/include/flang/Parser/token-sequence.h