Text.h

Go to the documentation of this file.

#pragma once
 
#include <initializer_list>
#include <string_view>
#include <tuple>
#include <unordered_map>
 
#include <sourcepp/String.h>
 
class BufferStream;
 
namespace sourcepp::parser::text {
 
constexpr std::string_view DEFAULT_SINGLE_LINE_COMMENT_START = "//";
constexpr std::string_view DEFAULT_MULTI_LINE_COMMENT_START  = "/*";
constexpr std::string_view DEFAULT_MULTI_LINE_COMMENT_END    = "*/";
constexpr std::pair        DEFAULT_QUOTES                    = {'\"', '\"'};
 
using EscapeSequenceMap = std::unordered_map<char, char>;
 
[[nodiscard]] const EscapeSequenceMap& getDefaultEscapeSequences();
 
[[nodiscard]] const EscapeSequenceMap& getDefaultEscapeSequencesOrNone(bool useEscapes);
 
class Tokenizer {
public:
    class Result {
        friend Tokenizer;
 
    public:
        [[nodiscard]] const std::vector<std::string_view>& getTokens() const;
 
        [[nodiscard]] uint64_t size() const;
 
        [[nodiscard]] uint64_t tell() const;
 
        void seek(uint64_t head_);
 
        [[nodiscard]] std::string_view next();
 
        template<typename T>
        requires std::same_as<T, bool> || std::integral<T> || std::floating_point<T>
        bool next(T& value) {
            if constexpr (std::same_as<T, bool>) {
                return string::toBool(this->next(), value);
            } else if constexpr (std::integral<T>) {
                return string::toInt(this->next(), value);
            } else if constexpr (std::floating_point<T>) {
                return string::toFloat(this->next(), value);
            } else {
                return false;
            }
        }
 
        [[nodiscard]] std::string_view peek() const;
 
        template<typename T>
        requires std::same_as<T, bool> || std::integral<T> || std::floating_point<T>
        bool peek(T& value) {
            if constexpr (std::same_as<T, bool>) {
                return string::toBool(this->peek(), value);
            } else if constexpr (std::integral<T>) {
                return string::toInt(this->peek(), value);
            } else if constexpr (std::floating_point<T>) {
                return string::toFloat(this->peek(), value);
            } else {
                return false;
            }
        }
 
        void skip();
 
    protected:
        Result() = default;
 
        std::vector<std::string_view> tokens;
        uint64_t head = 0;
    };
 
    Tokenizer() = default;
 
    Tokenizer& setSingleLineCommentStart(std::string_view singleLineCommentStart_);
 
    Tokenizer& setMultiLineCommentStart(std::string_view multiLineCommentStart_);
 
    Tokenizer& setMultiLineCommentEnd(std::string_view multiLineCommentEnd_);
 
    //Tokenizer& setQuotes(std::initializer_list<std::pair<char, char>> quotes_);
 
    Tokenizer& setEscapeSequences(const EscapeSequenceMap& escapeSequences_);
 
    [[nodiscard]] Result parse(std::string_view text, std::string& backingString) const;
 
protected:
    std::string_view singleLineCommentStart = DEFAULT_SINGLE_LINE_COMMENT_START;
    std::string_view multiLineCommentStart = DEFAULT_MULTI_LINE_COMMENT_START;
    std::string_view multiLineCommentEnd = DEFAULT_MULTI_LINE_COMMENT_END;
    // Disabled because MSVC is a garbage fire: "list initialization inside member initializer list or non-static data member initializer is not implemented"
    //std::initializer_list<std::pair<char, char>> quotes{DEFAULT_QUOTES};
    EscapeSequenceMap escapeSequences = getDefaultEscapeSequences();
};

[[nodiscard]] std::string convertSpecialCharsToEscapes(std::string_view str, const EscapeSequenceMap& escapeSequences);

[[nodiscard]] std::string convertEscapesToSpecialChars(std::string_view str, const EscapeSequenceMap& escapeSequences);

void eatWhitespace(BufferStream& stream);

void eatSingleLineComment(BufferStream& stream);

void eatMultiLineComment(BufferStream& stream, std::string_view multiLineCommentEnd = DEFAULT_MULTI_LINE_COMMENT_END);

void eatWhitespaceAndSingleLineComments(BufferStream& stream, std::string_view singleLineCommentStart = DEFAULT_SINGLE_LINE_COMMENT_START);

void eatWhitespaceAndMultiLineComments(BufferStream& stream, std::string_view multiLineCommentStart = DEFAULT_MULTI_LINE_COMMENT_START);

void eatWhitespaceAndComments(BufferStream& stream, std::string_view singleLineCommentStart = DEFAULT_SINGLE_LINE_COMMENT_START, std::string_view multiLineCommentStart = DEFAULT_MULTI_LINE_COMMENT_START);

[[nodiscard]] bool tryToEatChar(BufferStream& stream, char c);

[[nodiscard]] std::string readString(BufferStream& stream, std::initializer_list<std::pair<char, char>> quotes = {DEFAULT_QUOTES}, const EscapeSequenceMap& escapeSequences = getDefaultEscapeSequences());

std::string_view readStringToBuffer(BufferStream& stream, BufferStream& backing, std::initializer_list<std::pair<char, char>> quotes = {DEFAULT_QUOTES}, const EscapeSequenceMap& escapeSequences = getDefaultEscapeSequences());

[[nodiscard]] std::string readUnquotedString(BufferStream& stream, const EscapeSequenceMap& escapeSequences = getDefaultEscapeSequences());

std::string_view readUnquotedStringToBuffer(BufferStream& stream, BufferStream& backing, const EscapeSequenceMap& escapeSequences = getDefaultEscapeSequences());

[[nodiscard]] std::string readUnquotedString(BufferStream& stream, std::string_view end, const EscapeSequenceMap& escapeSequences = getDefaultEscapeSequences());

std::string_view readUnquotedStringToBuffer(BufferStream& stream, BufferStream& backing, std::string_view end, const EscapeSequenceMap& escapeSequences = getDefaultEscapeSequences());
 
} // namespace sourcepp::parser::text