1 files changed, 36 insertions, 0 deletions
diff --git a/src/tokenizer.h b/src/tokenizer.h
new file mode 100644
index 0000000..3322553
--- /dev/null
+++ b/src/tokenizer.h
@@ -0,0 +1,36 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+namespace TehJSON
+{
+	enum struct TokenType
+	{
+		LBrace,
+		RBrace,
+		Colon,
+		Comma,
+		StringLit,
+		IntLit,
+		FloatLit,
+	};
+
+	std::string getTokenName(TokenType t);
+
+	struct Token
+	{
+		TokenType type;
+		std::string content;
+	};
+
+	class Tokenizer
+	{
+	public:
+		void appendInput(std::string s);
+		std::vector<Token> tokenize();
+		std::string getInput();
+	private:
+		std::string input = "";
+	};
+}