/src/serenity/Userland/Libraries/LibWeb/CSS/Parser/TokenStream.h

Source
/*
 * Copyright (c) 2020-2021, the SerenityOS developers.
 * Copyright (c) 2021-2024, Sam Atkins <sam@ladybird.org>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 */

#pragma once

#include <AK/Format.h>
#include <AK/Vector.h>
#include <LibWeb/CSS/Parser/ComponentValue.h>
#include <LibWeb/CSS/Parser/Tokenizer.h>

namespace Web::CSS::Parser {

// https://drafts.csswg.org/css-syntax/#css-token-stream
template<typename T>
class TokenStream {
public:
    class StateTransaction {
    public:
        explicit StateTransaction(TokenStream<T>& token_stream)
            : m_token_stream(token_stream)
            , m_saved_index(token_stream.m_index)
        {
        }

        ~StateTransaction()
        {
            if (!m_commit)
                m_token_stream.m_index = m_saved_index;
        }

        StateTransaction create_child() { return StateTransaction(*this); }

        void commit()
        {
            m_commit = true;
            if (m_parent)
                m_parent->commit();
        }

    private:
        explicit StateTransaction(StateTransaction& parent)
            : m_parent(&parent)
            , m_token_stream(parent.m_token_stream)
            , m_saved_index(parent.m_token_stream.m_index)
        {
        }

        StateTransaction* m_parent { nullptr };
        TokenStream<T>& m_token_stream;
        size_t m_saved_index { 0 };
        bool m_commit { false };
    };

    explicit TokenStream(Span<T const> tokens)
        : m_tokens(tokens)
        , m_eof(make_eof())
    {
    }

    explicit TokenStream(Vector<T> const& tokens)
        : m_tokens(tokens.span())
        , m_eof(make_eof())
    {
    }

    static TokenStream<T> of_single_token(T const& token)
    {
        return TokenStream(Span<T const> { &token, 1 });
    }

    TokenStream(TokenStream<T> const&) = delete;
    TokenStream(TokenStream<T>&&) = default;

    // https://drafts.csswg.org/css-syntax/#token-stream-next-token
    [[nodiscard]] T const& next_token() const
    {
        // The item of tokens at index.
        // If that index would be out-of-bounds past the end of the list, it’s instead an <eof-token>.
        if (m_index < m_tokens.size())
            return m_tokens[m_index];
        return m_eof;
    }

    // https://drafts.csswg.org/css-syntax/#token-stream-empty
    [[nodiscard]] bool is_empty() const
    {
        // A token stream is empty if the next token is an <eof-token>.
        return next_token().is(Token::Type::EndOfFile);
    }

    // https://drafts.csswg.org/css-syntax/#token-stream-consume-a-token
    [[nodiscard]] T const& consume_a_token()
    {
        // Let token be the next token. Increment index, then return token.
        auto& token = next_token();
        ++m_index;
        return token;
    }

    // https://drafts.csswg.org/css-syntax/#token-stream-discard-a-token
    void discard_a_token()
    {
        // If the token stream is not empty, increment index.
        if (!is_empty())
            ++m_index;
    }

    // https://drafts.csswg.org/css-syntax/#token-stream-mark
    void mark()
    {
        // Append index to marked indexes.
        m_marked_indexes.append(m_index);
    }

    // https://drafts.csswg.org/css-syntax/#token-stream-restore-a-mark
    void restore_a_mark()
    {
        // Pop from marked indexes, and set index to the popped value.
        m_index = m_marked_indexes.take_last();
    }

    // https://drafts.csswg.org/css-syntax/#token-stream-discard-a-mark
    void discard_a_mark()
    {
        // Pop from marked indexes, and do nothing with the popped value.
        m_marked_indexes.take_last();
    }

    // https://drafts.csswg.org/css-syntax/#token-stream-discard-whitespace
    void discard_whitespace()
    {
        // While the next token is a <whitespace-token>, discard a token.
        while (next_token().is(Token::Type::Whitespace))
            discard_a_token();
    }

    bool has_next_token()
    {
        return !is_empty();
    }

    // Deprecated, used in older versions of the spec.
    T const& current_token()
    {
        if (m_index < 1 || (m_index - 1) >= m_tokens.size())
            return m_eof;

        return m_tokens.at(m_index - 1);
    }

    // Deprecated
    T const& peek_token(size_t offset = 0)
    {
        if (remaining_token_count() <= offset)
            return m_eof;

        return m_tokens.at(m_index + offset);
    }

    // Deprecated, was used in older versions of the spec.
    void reconsume_current_input_token()
    {
        if (m_index > 0)
            --m_index;
    }

    StateTransaction begin_transaction() { return StateTransaction(*this); }

    size_t remaining_token_count() const
    {
        if (m_tokens.size() > m_index)
            return m_tokens.size() - m_index;
        return 0;
    }

    void dump_all_tokens()
    {
        dbgln("Dumping all tokens:");
        for (size_t i = 0; i < m_tokens.size(); ++i) {
            auto& token = m_tokens[i];
            if (i == m_index)
                dbgln("-> {}", token.to_debug_string());
            else
                dbgln("   {}", token.to_debug_string());
        }
    }

    void copy_state(Badge<Parser>, TokenStream<T> const& other)
    {
        m_index = other.m_index;
    }

private:
    // https://drafts.csswg.org/css-syntax/#token-stream-tokens
    Span<T const> m_tokens;

    // https://drafts.csswg.org/css-syntax/#token-stream-index
    size_t m_index { 0 };

    // https://drafts.csswg.org/css-syntax/#token-stream-marked-indexes
    Vector<size_t> m_marked_indexes;

    T make_eof()
    {
        if constexpr (IsSame<T, Token>) {
            return Tokenizer::create_eof_token();
        }
        if constexpr (IsSame<T, ComponentValue>) {
            return ComponentValue(Tokenizer::create_eof_token());
        }
    }

    T m_eof;
};

}

Coverage Report

Created: 2026-05-16 07:03

Line	Count	Source
1		/*
2		* Copyright (c) 2020-2021, the SerenityOS developers.
3		* Copyright (c) 2021-2024, Sam Atkins <sam@ladybird.org>
4		*
5		* SPDX-License-Identifier: BSD-2-Clause
6		*/
7
8		#pragma once
9
10		#include <AK/Format.h>
11		#include <AK/Vector.h>
12		#include <LibWeb/CSS/Parser/ComponentValue.h>
13		#include <LibWeb/CSS/Parser/Tokenizer.h>
14
15		namespace Web::CSS::Parser {
16
17		// https://drafts.csswg.org/css-syntax/#css-token-stream
18		template<typename T>
19		class TokenStream {
20		public:
21		class StateTransaction {
22		public:
23		explicit StateTransaction(TokenStream<T>& token_stream)
24	0	: m_token_stream(token_stream)
25	0	, m_saved_index(token_stream.m_index)
26	0	{
27	0	}
28
29		~StateTransaction()
30	0	{
31	0	if (!m_commit)
32	0	m_token_stream.m_index = m_saved_index;
33	0	}
34
35	0	StateTransaction create_child() { return StateTransaction(*this); }
36
37		void commit()
38	0	{
39	0	m_commit = true;
40	0	if (m_parent)
41	0	m_parent->commit();
42	0	}
43
44		private:
45		explicit StateTransaction(StateTransaction& parent)
46	0	: m_parent(&parent)
47	0	, m_token_stream(parent.m_token_stream)
48	0	, m_saved_index(parent.m_token_stream.m_index)
49	0	{
50	0	}
51
52		StateTransaction* m_parent { nullptr };
53		TokenStream<T>& m_token_stream;
54		size_t m_saved_index { 0 };
55		bool m_commit { false };
56		};
57
58		explicit TokenStream(Span<T const> tokens)
59	0	: m_tokens(tokens)
60	0	, m_eof(make_eof())
61	0	{
62	0	}
63
64		explicit TokenStream(Vector<T> const& tokens)
65	0	: m_tokens(tokens.span())
66	0	, m_eof(make_eof())
67	0	{
68	0	} Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::ComponentValue>::TokenStream(AK::Vector<Web::CSS::Parser::ComponentValue, 0ul> const&) Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::Token>::TokenStream(AK::Vector<Web::CSS::Parser::Token, 0ul> const&)
69
70		static TokenStream<T> of_single_token(T const& token)
71	0	{
72	0	return TokenStream(Span<T const> { &token, 1 });
73	0	}
74
75		TokenStream(TokenStream<T> const&) = delete;
76		TokenStream(TokenStream<T>&&) = default;
77
78		// https://drafts.csswg.org/css-syntax/#token-stream-next-token
79		[[nodiscard]] T const& next_token() const
80	0	{
81		// The item of tokens at index.
82		// If that index would be out-of-bounds past the end of the list, it’s instead an <eof-token>.
83	0	if (m_index < m_tokens.size())
84	0	return m_tokens[m_index];
85	0	return m_eof;
86	0	} Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::ComponentValue>::next_token() const Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::Token>::next_token() const
87
88		// https://drafts.csswg.org/css-syntax/#token-stream-empty
89		[[nodiscard]] bool is_empty() const
90	0	{
91		// A token stream is empty if the next token is an <eof-token>.
92	0	return next_token().is(Token::Type::EndOfFile);
93	0	} Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::Token>::is_empty() const Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::ComponentValue>::is_empty() const
94
95		// https://drafts.csswg.org/css-syntax/#token-stream-consume-a-token
96		[[nodiscard]] T const& consume_a_token()
97	0	{
98		// Let token be the next token. Increment index, then return token.
99	0	auto& token = next_token();
100	0	++m_index;
101	0	return token;
102	0	} Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::ComponentValue>::consume_a_token() Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::Token>::consume_a_token()
103
104		// https://drafts.csswg.org/css-syntax/#token-stream-discard-a-token
105		void discard_a_token()
106	0	{
107		// If the token stream is not empty, increment index.
108	0	if (!is_empty())
109	0	++m_index;
110	0	} Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::Token>::discard_a_token() Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::ComponentValue>::discard_a_token()
111
112		// https://drafts.csswg.org/css-syntax/#token-stream-mark
113		void mark()
114	0	{
115		// Append index to marked indexes.
116	0	m_marked_indexes.append(m_index);
117	0	}
118
119		// https://drafts.csswg.org/css-syntax/#token-stream-restore-a-mark
120		void restore_a_mark()
121	0	{
122		// Pop from marked indexes, and set index to the popped value.
123	0	m_index = m_marked_indexes.take_last();
124	0	}
125
126		// https://drafts.csswg.org/css-syntax/#token-stream-discard-a-mark
127		void discard_a_mark()
128	0	{
129		// Pop from marked indexes, and do nothing with the popped value.
130	0	m_marked_indexes.take_last();
131	0	}
132
133		// https://drafts.csswg.org/css-syntax/#token-stream-discard-whitespace
134		void discard_whitespace()
135	0	{
136		// While the next token is a <whitespace-token>, discard a token.
137	0	while (next_token().is(Token::Type::Whitespace))
138	0	discard_a_token();
139	0	} Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::Token>::discard_whitespace() Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::ComponentValue>::discard_whitespace()
140
141		bool has_next_token()
142	0	{
143	0	return !is_empty();
144	0	} Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::Token>::has_next_token() Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::ComponentValue>::has_next_token()
145
146		// Deprecated, used in older versions of the spec.
147		T const& current_token()
148		{
149		if (m_index < 1 \|\| (m_index - 1) >= m_tokens.size())
150		return m_eof;
151
152		return m_tokens.at(m_index - 1);
153		}
154
155		// Deprecated
156		T const& peek_token(size_t offset = 0)
157	0	{
158	0	if (remaining_token_count() <= offset)
159	0	return m_eof;
160
161	0	return m_tokens.at(m_index + offset);
162	0	}
163
164		// Deprecated, was used in older versions of the spec.
165		void reconsume_current_input_token()
166	0	{
167	0	if (m_index > 0)
168	0	--m_index;
169	0	}
170
171	0	StateTransaction begin_transaction() { return StateTransaction(*this); }
172
173		size_t remaining_token_count() const
174	0	{
175	0	if (m_tokens.size() > m_index)
176	0	return m_tokens.size() - m_index;
177	0	return 0;
178	0	}
179
180		void dump_all_tokens()
181		{
182		dbgln("Dumping all tokens:");
183		for (size_t i = 0; i < m_tokens.size(); ++i) {
184		auto& token = m_tokens[i];
185		if (i == m_index)
186		dbgln("-> {}", token.to_debug_string());
187		else
188		dbgln(" {}", token.to_debug_string());
189		}
190		}
191
192		void copy_state(Badge<Parser>, TokenStream<T> const& other)
193	0	{
194	0	m_index = other.m_index;
195	0	}
196
197		private:
198		// https://drafts.csswg.org/css-syntax/#token-stream-tokens
199		Span<T const> m_tokens;
200
201		// https://drafts.csswg.org/css-syntax/#token-stream-index
202		size_t m_index { 0 };
203
204		// https://drafts.csswg.org/css-syntax/#token-stream-marked-indexes
205		Vector<size_t> m_marked_indexes;
206
207		T make_eof()
208	0	{
209	0	if constexpr (IsSame<T, Token>) {
210	0	return Tokenizer::create_eof_token();
211	0	}
212	0	if constexpr (IsSame<T, ComponentValue>) {
213	0	return ComponentValue(Tokenizer::create_eof_token());
214	0	}
215	0	} Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::ComponentValue>::make_eof() Unexecuted instantiation: Web::CSS::Parser::TokenStream<Web::CSS::Parser::Token>::make_eof()
216
217		T m_eof;
218		};
219
220		}