Skip to content

[Parser][NFC] Clean up the lexer index/pos API #6553

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tlively merged 2 commits into from
Apr 29, 2024
Merged

[Parser][NFC] Clean up the lexer index/pos API #6553

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
56708b4
[Parser][NFC] Clean up the lexer index/pos API
tlively Apr 26, 2024
c71a0fc
lint
tlively Apr 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 17 additions & 17 deletions src/parser/lexer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -905,12 +905,12 @@ std::optional<LexResult> keyword(std::string_view in) {
void Lexer::skipSpace() {
while (true) {
if (auto ctx = annotation(next())) {
index += ctx->span.size();
pos += ctx->span.size();
annotations.push_back(ctx->annotation);
continue;
}
if (auto ctx = space(next())) {
index += ctx->span.size();
pos += ctx->span.size();
continue;
}
break;
Expand All @@ -919,7 +919,7 @@ void Lexer::skipSpace() {

bool Lexer::takeLParen() {
if (LexCtx(next()).startsWith("("sv)) {
++index;
++pos;
advance();
return true;
}
Expand All @@ -928,7 +928,7 @@ bool Lexer::takeLParen() {

bool Lexer::takeRParen() {
if (LexCtx(next()).startsWith(")"sv)) {
++index;
++pos;
advance();
return true;
}
Expand All @@ -937,7 +937,7 @@ bool Lexer::takeRParen() {

std::optional<std::string> Lexer::takeString() {
if (auto result = str(next())) {
index += result->span.size();
pos += result->span.size();
advance();
if (result->str) {
return result->str;
Expand All @@ -950,7 +950,7 @@ std::optional<std::string> Lexer::takeString() {

std::optional<Name> Lexer::takeID() {
if (auto result = ident(next())) {
index += result->span.size();
pos += result->span.size();
advance();
if (result->str) {
return Name(*result->str);
Expand All @@ -967,7 +967,7 @@ std::optional<Name> Lexer::takeID() {

std::optional<std::string_view> Lexer::takeKeyword() {
if (auto result = keyword(next())) {
index += result->span.size();
pos += result->span.size();
advance();
return result->span;
}
Expand All @@ -976,7 +976,7 @@ std::optional<std::string_view> Lexer::takeKeyword() {

bool Lexer::takeKeyword(std::string_view expected) {
if (auto result = keyword(next()); result && result->span == expected) {
index += expected.size();
pos += expected.size();
advance();
return true;
}
Expand All @@ -990,7 +990,7 @@ std::optional<uint64_t> Lexer::takeOffset() {
}
Lexer subLexer(result->span.substr(7));
if (auto o = subLexer.takeU64()) {
index += result->span.size();
pos += result->span.size();
advance();
return o;
}
Expand All @@ -1005,7 +1005,7 @@ std::optional<uint32_t> Lexer::takeAlign() {
}
Lexer subLexer(result->span.substr(6));
if (auto o = subLexer.takeU32()) {
index += result->span.size();
pos += result->span.size();
advance();
return o;
}
Expand All @@ -1016,7 +1016,7 @@ std::optional<uint32_t> Lexer::takeAlign() {
template<typename T> std::optional<T> Lexer::takeU() {
static_assert(std::is_integral_v<T> && std::is_unsigned_v<T>);
if (auto result = integer(next()); result && result->isUnsigned<T>()) {
index += result->span.size();
pos += result->span.size();
advance();
return T(result->n);
}
Expand All @@ -1027,7 +1027,7 @@ template<typename T> std::optional<T> Lexer::takeU() {
template<typename T> std::optional<T> Lexer::takeS() {
static_assert(std::is_integral_v<T> && std::is_signed_v<T>);
if (auto result = integer(next()); result && result->isSigned<T>()) {
index += result->span.size();
pos += result->span.size();
advance();
return T(result->n);
}
Expand All @@ -1038,7 +1038,7 @@ template<typename T> std::optional<T> Lexer::takeI() {
static_assert(std::is_integral_v<T> && std::is_unsigned_v<T>);
if (auto result = integer(next())) {
if (result->isUnsigned<T>() || result->isSigned<std::make_signed_t<T>>()) {
index += result->span.size();
pos += result->span.size();
advance();
return T(result->n);
}
Expand Down Expand Up @@ -1078,12 +1078,12 @@ std::optional<double> Lexer::takeF64() {
bits = (bits & ~payloadMask) | payload;
memcpy(&d, &bits, sizeof(bits));
}
index += result->span.size();
pos += result->span.size();
advance();
return d;
}
if (auto result = integer(next())) {
index += result->span.size();
pos += result->span.size();
advance();
if (result->sign == Neg) {
if (result->n == 0) {
Expand Down Expand Up @@ -1115,12 +1115,12 @@ std::optional<float> Lexer::takeF32() {
bits = (bits & ~payloadMask) | payload;
memcpy(&f, &bits, sizeof(bits));
}
index += result->span.size();
pos += result->span.size();
advance();
return f;
}
if (auto result = integer(next())) {
index += result->span.size();
pos += result->span.size();
advance();
if (result->sign == Neg) {
if (result->n == 0) {
Expand Down
18 changes: 8 additions & 10 deletions src/parser/lexer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,18 +58,18 @@ extern Name srcAnnotationKind;

struct Lexer {
private:
size_t index = 0;
size_t pos = 0;
std::vector<Annotation> annotations;

public:
std::string_view buffer;

Lexer(std::string_view buffer) : buffer(buffer) { setIndex(0); }
Lexer(std::string_view buffer) : buffer(buffer) { setPos(0); }

size_t getIndex() const { return index; }
size_t getPos() const { return pos; }

void setIndex(size_t i) {
index = i;
void setPos(size_t i) {
pos = i;
advance();
}

Expand All @@ -93,7 +93,7 @@ struct Lexer {
if (takeString()) {
continue;
}
++index;
++pos;
advance();
}
}
Expand Down Expand Up @@ -150,14 +150,14 @@ struct Lexer {
return ret;
}

std::string_view next() const { return buffer.substr(index); }
std::string_view next() const { return buffer.substr(pos); }

void advance() {
annotations.clear();
skipSpace();
}

bool empty() const { return index == buffer.size(); }
bool empty() const { return pos == buffer.size(); }

TextPos position(const char* c) const;
TextPos position(size_t i) const { return position(buffer.data() + i); }
Expand All @@ -166,8 +166,6 @@ struct Lexer {
}
TextPos position() const { return position(getPos()); }

size_t getPos() const { return index; }

[[nodiscard]] Err err(size_t pos, std::string reason) {
std::stringstream msg;
msg << position(pos) << ": error: " << reason;
Expand Down
10 changes: 5 additions & 5 deletions src/parser/parsers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,11 +372,11 @@ template<typename Ctx> struct WithPosition {
WithPosition(Ctx& ctx, Index pos)
: ctx(ctx), original(ctx.in.getPos()),
annotations(ctx.in.takeAnnotations()) {
ctx.in.setIndex(pos);
ctx.in.setPos(pos);
}

~WithPosition() {
ctx.in.setIndex(original);
ctx.in.setPos(original);
ctx.in.setAnnotations(std::move(annotations));
}
};
Expand Down Expand Up @@ -1325,7 +1325,7 @@ trycatch(Ctx& ctx, const std::vector<Annotation>& annotations, bool folded) {
if (id && id != label) {
// Instead of returning an error, retry without the ID.
parseID = false;
ctx.in.setIndex(afterCatchPos);
ctx.in.setPos(afterCatchPos);
continue;
}
}
Expand All @@ -1334,7 +1334,7 @@ trycatch(Ctx& ctx, const std::vector<Annotation>& annotations, bool folded) {
if (parseID && tag.getErr()) {
// Instead of returning an error, retry without the ID.
parseID = false;
ctx.in.setIndex(afterCatchPos);
ctx.in.setPos(afterCatchPos);
continue;
}
CHECK_ERR(tag);
Expand Down Expand Up @@ -3375,7 +3375,7 @@ template<typename Ctx> MaybeResult<> elem(Ctx& ctx) {
offset = *off;
} else {
// This must be the beginning of the elemlist instead.
ctx.in.setIndex(beforeLParen);
ctx.in.setPos(beforeLParen);
}
}
}
Expand Down