kyou/tokens.h at master · Tehsapper/kyou · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
#pragma once

#include <stddef.h>
#include <stdint.h>

#define KANJI_SUN          u8"日"
#define KANJI_MOON         u8"月"
#define KANJI_STARS        u8"星"

#define KANJI_STORAGE_BASE u8"品"
#define KANJI_STORAGE      u8"品台"

#define KANJI_FIRE         u8"火"
#define KANJI_WATER        u8"水"
#define KANJI_TREE         u8"木"
#define KANJI_EARTH        u8"土"
#define KANJI_METAL        u8"金"

#define KANJI_SPRING       u8"春"
#define KANJI_SUMMER       u8"夏"
#define KANJI_AUTUMN       u8"秋"
#define KANJI_WINTER       u8"冬"
#define KANJI_STRING       u8"文"
#define KANJI_CHAR         u8"字"

#define KANJI_MOVE         u8"動"
#define KANJI_PUSH         u8"押"
#define KANJI_CALL         u8"呼"
#define KANJI_RETURN       u8"帰"
#define KANJI_POP          u8"弾"

#define KANJI_ADD          u8"足"
#define KANJI_SUBSTRACT    u8"引"
#define KANJI_MULTIPLY     u8"掛"
#define KANJI_DIVIDE       u8"割"
#define KANJI_MODULO       u8"余"
#define KANJI_OR           u8"或"
#define KANJI_AND          u8"共"
#define KANJI_XOR          u8"排"

#define KANJI_ZERO         u8"霊"
#define KANJI_ONE          u8"一"
#define KANJI_TWO          u8"二"
#define KANJI_THREE        u8"三"
#define KANJI_FOUR         u8"四"
#define KANJI_FIVE         u8"五"
#define KANJI_SIX          u8"六"
#define KANJI_SEVEN        u8"七"
#define KANJI_EIGHT        u8"八"
#define KANJI_NINE         u8"九"
#define KANJI_TEN          u8"十"
#define KANJI_HUNDRED      u8"百"
#define KANJI_THOUSAND     u8"千"
#define KANJI_TEN_THOUSAND u8"万"

#define KANJI_LABEL        u8"札"
#define KANJI_SECTION      u8"句"
#define KANJI_EXPORT       u8"輸出"
#define KANJI_IMPORT       u8"輸入"
#define KANJI_STORE        u8"資"
#define KANJI_TIMES        u8"度"
#define KANJI_EMIT         u8"発"
#define KANJI_BRANCH       u8"別"
#define KANJI_ALWAYS       u8"常"
#define KANJI_EQUALS       u8"等"
#define KANJI_GREATER      u8"大"
#define KANJI_LESS         u8"小"

#define KANJI_OPEN_QUOTE   u8"「"
#define KANJI_CLOSE_QUOTE  u8"」"

typedef enum {
	// three lights: stdout, stdin and virtual memory
	TOKEN_SUN,
	TOKEN_MOON,
	TOKEN_STARS,

	// additional tokens for stack
	TOKEN_STORAGE,
	TOKEN_STORAGE_BASE,

	// five elements
	TOKEN_FIRE,
	TOKEN_WATER,
	TOKEN_TREE,
	TOKEN_METAL,
	TOKEN_EARTH,

	// four seasons
	TOKEN_SPRING,
	TOKEN_SUMMER,
	TOKEN_AUTUMN,
	TOKEN_WINTER,

	TOKEN_STRING_TYPE, // not to be confused with spring and string
	TOKEN_CHAR,

	// operations
	TOKEN_MOVE,
	TOKEN_PUSH,
	TOKEN_CALL,
	TOKEN_RETURN,
	TOKEN_POP,

	TOKEN_ADD,
	TOKEN_SUB,
	TOKEN_MUL,
	TOKEN_DIV,
	TOKEN_MOD,
	TOKEN_OR,
	TOKEN_AND,
	TOKEN_XOR,

	TOKEN_IDENTIFIER,
	TOKEN_NUMBER,
	TOKEN_STRING,
	TOKEN_LABEL,
	TOKEN_BRANCH,
	TOKEN_ALWAYS,
	TOKEN_EQUALS,
	TOKEN_LESS,
	TOKEN_GREATER,

	TOKEN_EOF,
	TOKEN_NONE
}
token_type;

typedef struct
{
	token_type type;
	union {
		int8_t as_int8;
		int16_t as_int16;
		int32_t as_int32;
		int64_t as_int64;
		void* as_ptr;
		const char* as_cstr;
	};
	uint32_t line, col; // hopefully nobody has more than 4 mln lines of code in a file
}
token;

typedef struct
{
	token* data;
	size_t size;
}
tokens;

typedef enum { TOKENIZE_SUCCESS, TOKENIZE_ERROR } tokenize_result_t;

tokenize_result_t tokenize(tokens* output, const char* data, size_t data_size);