src/tokenizer.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

#pragma once

#include <string>
#include <vector>

namespace TehJSON
{
	enum struct TokenType
	{
		LBrace,
		RBrace,
		LSquare,
		RSquare,
		Colon,
		Comma,
		StringLit,
		IntLit,
		FloatLit,
		BoolLit,
	};

	std::string getTokenName(TokenType t);

	struct Token
	{
		TokenType type;
		std::string content;
	};

	class Tokenizer
	{
	public:
		void appendInput(std::string s);
		std::vector<Token> tokenize();
		std::string getInput();
	private:
		std::string input = "";
	};
}