1 //
2 // Copyright (c) 2006, Brian Frank and Andy Frank
3 // Licensed under the Academic Free License version 3.0
4 //
5 // History:
6 // 16 Apr 06 Brian Frank Creation
7 //
8
9 **
10 ** Token is the enum for all the token types.
11 **
12 enum Token
13 {
14
15 //////////////////////////////////////////////////////////////////////////
16 // Enum
17 //////////////////////////////////////////////////////////////////////////
18
19 // identifer/literals
20 identifier ("identifier"),
21 strLiteral ("Str literal"),
22 intLiteral ("Int literal"),
23 floatLiteral ("Float literal"),
24 durationLiteral ("Duration literal"),
25 uriLiteral ("Uri literal"),
26
27 // operators
28 dot("."),
29 semicolon (";"),
30 comma (","),
31 colon (":"),
32 doubleColon ("::"),
33 plus ("+"),
34 minus ("-"),
35 star ("*"),
36 slash ("/"),
37 percent ("%"),
38 pound ("#"),
39 increment ("++"),
40 decrement ("--"),
41 bang ("!"),
42 question ("?"),
43 tilde ("~"),
44 pipe ("|"),
45 amp ("&"),
46 caret ("^"),
47 at ("@"),
48 doublePipe ("||"),
49 doubleAmp ("&&"),
50 same ("==="),
51 notSame ("!=="),
52 eq ("=="),
53 notEq ("!="),
54 cmp ("<=>"),
55 lt ("<"),
56 ltEq ("<="),
57 gt (">"),
58 gtEq (">="),
59 lshift ("<<"),
60 rshift (">>"),
61 lbrace ("{"),
62 rbrace ("}"),
63 lparen ("("),
64 rparen (")"),
65 lbracket ("["),
66 rbracket ("]"),
67 dotDot (".."),
68 dotDotDot ("..."),
69 defAssign (":="),
70 assign ("="),
71 assignPlus ("+="),
72 assignMinus ("-="),
73 assignStar ("*="),
74 assignSlash ("/="),
75 assignPercent ("%="),
76 assignAmp ("&="),
77 assignPipe ("|="),
78 assignCaret ("^="),
79 assignLshift ("<<="),
80 assignRshift (">>="),
81 arrow ("->"),
82 docComment ("**"),
83
84 // keywords
85 abstractKeyword,
86 asKeyword,
87 assertKeyword,
88 booleanKeyword,
89 breakKeyword,
90 caseKeyword,
91 catchKeyword,
92 classKeyword,
93 constKeyword,
94 continueKeyword,
95 defaultKeyword,
96 delegateKeyword,
97 doKeyword,
98 elseKeyword,
99 enumKeyword,
100 eventKeyword,
101 explicitKeyword,
102 externKeyword,
103 falseKeyword,
104 finalKeyword,
105 finallyKeyword,
106 fixedKeyword,
107 forKeyword,
108 foreachKeyword,
109 gotoKeyword,
110 ifKeyword,
111 implicitKeyword,
112 implementsKeyword,
113 instanceofKeyword,
114 internalKeyword,
115 interfaceKeyword,
116 isKeyword,
117 lockKeyword,
118 mixinKeyword,
119 nativeKeyword,
120 newKeyword,
121 nullKeyword,
122 objectKeyword,
123 operatorKeyword,
124 overrideKeyword,
125 packageKeyword,
126 privateKeyword,
127 protectedKeyword,
128 publicKeyword,
129 readonlyKeyword,
130 refKeyword,
131 returnKeyword,
132 sealedKeyword,
133 sizeofKeyword,
134 stackallocKeyword,
135 staticKeyword,
136 strictfpKeyword,
137 structKeyword,
138 superKeyword,
139 switchKeyword,
140 synchronizedKeyword,
141 thisKeyword,
142 throwKeyword,
143 throwsKeyword,
144 transientKeyword,
145 trueKeyword,
146 tryKeyword,
147 typeofKeyword,
148 uncheckedKeyword,
149 unsafeKeyword,
150 usingKeyword,
151 virtualKeyword,
152 volatileKeyword,
153 voidKeyword,
154 whileKeyword,
155
156 // misc
157 eof("eof");
158
159 // potential keywords:
160 // async, checked, contract, decimal, duck, def, isnot,
161 // namespace, once, unchecked, unless, when, var, with
162
163 //////////////////////////////////////////////////////////////////////////
164 // Constructor
165 //////////////////////////////////////////////////////////////////////////
166
167 **
168 ** Construct with symbol str, or null symbol for keyword.
169 **
170 private new make(Str symbol := null)
171 {
172 if (symbol == null)
173 {
174 if (!name.endsWith("Keyword")) throw Err.make(name)
175 this.symbol = name[0..-8]
176 this.keyword = true
177 this.isAssign = false
178 }
179 else
180 {
181 this.symbol = symbol
182 this.keyword = false
183 this.isAssign = name.startsWith("assign") ||
184 name == "increment" ||
185 name == "decrement"
186 }
187 }
188
189 //////////////////////////////////////////////////////////////////////////
190 // Methods
191 //////////////////////////////////////////////////////////////////////////
192
193 **
194 ** Get this Token as a ExprId or throw Err.
195 **
196 ExprId toExprId()
197 {
198 switch (this)
199 {
200 // unary
201 case bang: return ExprId.boolNot
202
203 // binary
204 case assign: return ExprId.assign
205 case doubleAmp: return ExprId.boolAnd
206 case doublePipe: return ExprId.boolOr
207 case same: return ExprId.same
208 case notSame: return ExprId.notSame
209 case asKeyword: return ExprId.asExpr
210 case isKeyword: return ExprId.isExpr
211
212 // default
213 default: throw Err.make(toStr)
214 }
215 }
216
217 /**
218
219
220
221 ShortcutOp toShortcutOp(Int degree)
222 {
223 switch (this)
224 {
225 case plus: return ShortcutOp.plus // a + b
226 case minus: return degree == 1 ? ShortcutOp.negate : ShortcutOp.minus // -a; a - b
227 case star: return ShortcutOp.star // a * b
228 case slash: return ShortcutOp.slash // a / b
229 case percent: return ShortcutOp.percent // a % b
230 case lshift: return ShortcutOp.lshift // a << b
231 case rshift: return ShortcutOp.rshift // a >> b
232 case amp: return ShortcutOp.amp // a & b
233 case pipe: return ShortcutOp.pipe // a | b
234 case caret: return ShortcutOp.caret // a ^ b
235 case tilde: return ShortcutOp.tilde // ~a
236 case increment: return ShortcutOp.increment // ++a, a++
237 case decrement: return ShortcutOp.decrement // --a, a--
238 case eq: return ShortcutOp.eq // a == b
239 case notEq: return ShortcutOp.eq // a != b
240 case cmp: return ShortcutOp.cmp // a <=> b
241 case gt: return ShortcutOp.cmp // a > b
242 case gtEq: return ShortcutOp.cmp // a >= b
243 case lt: return ShortcutOp.cmp // a < b
244 case ltEq: return ShortcutOp.cmp // a <= b
245 case assignPlus: return ShortcutOp.plus // a += b
246 case assignMinus: return ShortcutOp.minus // a -= b
247 case assignStar: return ShortcutOp.star // a *= b
248 case assignSlash: return ShortcutOp.slash // a /= b
249 case assignPercent: return ShortcutOp.percent // a %= b
250 case assignAmp: return ShortcutOp.amp // a &= b
251 case assignPipe: return ShortcutOp.pipe // a |= b
252 case assignCaret: return ShortcutOp.caret // a ^= b
253 case assignLshift: return ShortcutOp.lshift // a <<= b
254 case assignRshift: return ShortcutOp.rshift // a >>= b
255 default: throw Err.make(toStr)
256 }
257 }
258
259 //////////////////////////////////////////////////////////////////////////
260 // Keyword Lookup
261 //////////////////////////////////////////////////////////////////////////
262
263 **
264 ** Get a map of the keywords
265 **
266 const static Str:Token keywords
267 static
268 {
269 map := Str:Token[:]
270 values.each |Token t|
271 {
272 if (t.keyword) map[t.symbol] = t
273 }
274 keywords = map.toImmutable
275 }
276
277 //////////////////////////////////////////////////////////////////////////
278 // Test
279 //////////////////////////////////////////////////////////////////////////
280
281 static Void main()
282 {
283 values.each |Token t|
284 {
285 echo(t.name + " '" + t.symbol + "'")
286 }
287
288 echo(keywords)
289 }
290
291 //////////////////////////////////////////////////////////////////////////
292 // Fields
293 //////////////////////////////////////////////////////////////////////////
294
295 ** Get string used to display token to user in error messages
296 const Str symbol
297
298 ** Is this a keyword token such as "null"
299 const Bool keyword
300
301 ** Is this an assignment token such as "=", etc "+=", etc
302 const Bool isAssign
303
304 }
2 // Copyright (c) 2006, Brian Frank and Andy Frank
3 // Licensed under the Academic Free License version 3.0
4 //
5 // History:
6 // 16 Apr 06 Brian Frank Creation
7 //
8
9 **
10 ** Token is the enum for all the token types.
11 **
12 enum Token
13 {
14
15 //////////////////////////////////////////////////////////////////////////
16 // Enum
17 //////////////////////////////////////////////////////////////////////////
18
19 // identifer/literals
20 identifier ("identifier"),
21 strLiteral ("Str literal"),
22 intLiteral ("Int literal"),
23 floatLiteral ("Float literal"),
24 durationLiteral ("Duration literal"),
25 uriLiteral ("Uri literal"),
26
27 // operators
28 dot("."),
29 semicolon (";"),
30 comma (","),
31 colon (":"),
32 doubleColon ("::"),
33 plus ("+"),
34 minus ("-"),
35 star ("*"),
36 slash ("/"),
37 percent ("%"),
38 pound ("#"),
39 increment ("++"),
40 decrement ("--"),
41 bang ("!"),
42 question ("?"),
43 tilde ("~"),
44 pipe ("|"),
45 amp ("&"),
46 caret ("^"),
47 at ("@"),
48 doublePipe ("||"),
49 doubleAmp ("&&"),
50 same ("==="),
51 notSame ("!=="),
52 eq ("=="),
53 notEq ("!="),
54 cmp ("<=>"),
55 lt ("<"),
56 ltEq ("<="),
57 gt (">"),
58 gtEq (">="),
59 lshift ("<<"),
60 rshift (">>"),
61 lbrace ("{"),
62 rbrace ("}"),
63 lparen ("("),
64 rparen (")"),
65 lbracket ("["),
66 rbracket ("]"),
67 dotDot (".."),
68 dotDotDot ("..."),
69 defAssign (":="),
70 assign ("="),
71 assignPlus ("+="),
72 assignMinus ("-="),
73 assignStar ("*="),
74 assignSlash ("/="),
75 assignPercent ("%="),
76 assignAmp ("&="),
77 assignPipe ("|="),
78 assignCaret ("^="),
79 assignLshift ("<<="),
80 assignRshift (">>="),
81 arrow ("->"),
82 docComment ("**"),
83
84 // keywords
85 abstractKeyword,
86 asKeyword,
87 assertKeyword,
88 booleanKeyword,
89 breakKeyword,
90 caseKeyword,
91 catchKeyword,
92 classKeyword,
93 constKeyword,
94 continueKeyword,
95 defaultKeyword,
96 delegateKeyword,
97 doKeyword,
98 elseKeyword,
99 enumKeyword,
100 eventKeyword,
101 explicitKeyword,
102 externKeyword,
103 falseKeyword,
104 finalKeyword,
105 finallyKeyword,
106 fixedKeyword,
107 forKeyword,
108 foreachKeyword,
109 gotoKeyword,
110 ifKeyword,
111 implicitKeyword,
112 implementsKeyword,
113 instanceofKeyword,
114 internalKeyword,
115 interfaceKeyword,
116 isKeyword,
117 lockKeyword,
118 mixinKeyword,
119 nativeKeyword,
120 newKeyword,
121 nullKeyword,
122 objectKeyword,
123 operatorKeyword,
124 overrideKeyword,
125 packageKeyword,
126 privateKeyword,
127 protectedKeyword,
128 publicKeyword,
129 readonlyKeyword,
130 refKeyword,
131 returnKeyword,
132 sealedKeyword,
133 sizeofKeyword,
134 stackallocKeyword,
135 staticKeyword,
136 strictfpKeyword,
137 structKeyword,
138 superKeyword,
139 switchKeyword,
140 synchronizedKeyword,
141 thisKeyword,
142 throwKeyword,
143 throwsKeyword,
144 transientKeyword,
145 trueKeyword,
146 tryKeyword,
147 typeofKeyword,
148 uncheckedKeyword,
149 unsafeKeyword,
150 usingKeyword,
151 virtualKeyword,
152 volatileKeyword,
153 voidKeyword,
154 whileKeyword,
155
156 // misc
157 eof("eof");
158
159 // potential keywords:
160 // async, checked, contract, decimal, duck, def, isnot,
161 // namespace, once, unchecked, unless, when, var, with
162
163 //////////////////////////////////////////////////////////////////////////
164 // Constructor
165 //////////////////////////////////////////////////////////////////////////
166
167 **
168 ** Construct with symbol str, or null symbol for keyword.
169 **
170 private new make(Str symbol := null)
171 {
172 if (symbol == null)
173 {
174 if (!name.endsWith("Keyword")) throw Err.make(name)
175 this.symbol = name[0..-8]
176 this.keyword = true
177 this.isAssign = false
178 }
179 else
180 {
181 this.symbol = symbol
182 this.keyword = false
183 this.isAssign = name.startsWith("assign") ||
184 name == "increment" ||
185 name == "decrement"
186 }
187 }
188
189 //////////////////////////////////////////////////////////////////////////
190 // Methods
191 //////////////////////////////////////////////////////////////////////////
192
193 **
194 ** Get this Token as a ExprId or throw Err.
195 **
196 ExprId toExprId()
197 {
198 switch (this)
199 {
200 // unary
201 case bang: return ExprId.boolNot
202
203 // binary
204 case assign: return ExprId.assign
205 case doubleAmp: return ExprId.boolAnd
206 case doublePipe: return ExprId.boolOr
207 case same: return ExprId.same
208 case notSame: return ExprId.notSame
209 case asKeyword: return ExprId.asExpr
210 case isKeyword: return ExprId.isExpr
211
212 // default
213 default: throw Err.make(toStr)
214 }
215 }
216
217 /**
218
219
220
221 ShortcutOp toShortcutOp(Int degree)
222 {
223 switch (this)
224 {
225 case plus: return ShortcutOp.plus // a + b
226 case minus: return degree == 1 ? ShortcutOp.negate : ShortcutOp.minus // -a; a - b
227 case star: return ShortcutOp.star // a * b
228 case slash: return ShortcutOp.slash // a / b
229 case percent: return ShortcutOp.percent // a % b
230 case lshift: return ShortcutOp.lshift // a << b
231 case rshift: return ShortcutOp.rshift // a >> b
232 case amp: return ShortcutOp.amp // a & b
233 case pipe: return ShortcutOp.pipe // a | b
234 case caret: return ShortcutOp.caret // a ^ b
235 case tilde: return ShortcutOp.tilde // ~a
236 case increment: return ShortcutOp.increment // ++a, a++
237 case decrement: return ShortcutOp.decrement // --a, a--
238 case eq: return ShortcutOp.eq // a == b
239 case notEq: return ShortcutOp.eq // a != b
240 case cmp: return ShortcutOp.cmp // a <=> b
241 case gt: return ShortcutOp.cmp // a > b
242 case gtEq: return ShortcutOp.cmp // a >= b
243 case lt: return ShortcutOp.cmp // a < b
244 case ltEq: return ShortcutOp.cmp // a <= b
245 case assignPlus: return ShortcutOp.plus // a += b
246 case assignMinus: return ShortcutOp.minus // a -= b
247 case assignStar: return ShortcutOp.star // a *= b
248 case assignSlash: return ShortcutOp.slash // a /= b
249 case assignPercent: return ShortcutOp.percent // a %= b
250 case assignAmp: return ShortcutOp.amp // a &= b
251 case assignPipe: return ShortcutOp.pipe // a |= b
252 case assignCaret: return ShortcutOp.caret // a ^= b
253 case assignLshift: return ShortcutOp.lshift // a <<= b
254 case assignRshift: return ShortcutOp.rshift // a >>= b
255 default: throw Err.make(toStr)
256 }
257 }
258
259 //////////////////////////////////////////////////////////////////////////
260 // Keyword Lookup
261 //////////////////////////////////////////////////////////////////////////
262
263 **
264 ** Get a map of the keywords
265 **
266 const static Str:Token keywords
267 static
268 {
269 map := Str:Token[:]
270 values.each |Token t|
271 {
272 if (t.keyword) map[t.symbol] = t
273 }
274 keywords = map.toImmutable
275 }
276
277 //////////////////////////////////////////////////////////////////////////
278 // Test
279 //////////////////////////////////////////////////////////////////////////
280
281 static Void main()
282 {
283 values.each |Token t|
284 {
285 echo(t.name + " '" + t.symbol + "'")
286 }
287
288 echo(keywords)
289 }
290
291 //////////////////////////////////////////////////////////////////////////
292 // Fields
293 //////////////////////////////////////////////////////////////////////////
294
295 ** Get string used to display token to user in error messages
296 const Str symbol
297
298 ** Is this a keyword token such as "null"
299 const Bool keyword
300
301 ** Is this an assignment token such as "=", etc "+=", etc
302 const Bool isAssign
303
304 }
More Info
Slots
- abstractKeyword
- amp
- arrow
- asKeyword
- assertKeyword
- assign
- assignAmp
- assignCaret
- assignLshift
- assignMinus
- assignPercent
- assignPipe
- assignPlus
- assignRshift
- assignSlash
- assignStar
- at
- bang
- booleanKeyword
- breakKeyword
- caret
- caseKeyword
- catchKeyword
- classKeyword
- cmp
- colon
- comma
- constKeyword
- continueKeyword
- decrement
- defAssign
- defaultKeyword
- delegateKeyword
- doKeyword
- docComment
- dot
- dotDot
- dotDotDot
- doubleAmp
- doubleColon
- doublePipe
- durationLiteral
- elseKeyword
- enumKeyword
- eof
- eq
- eventKeyword
- explicitKeyword
- externKeyword
- falseKeyword
- finalKeyword
- finallyKeyword
- fixedKeyword
- floatLiteral
- forKeyword
- foreachKeyword
- fromStr
- gotoKeyword
- gt
- gtEq
- identifier
- ifKeyword
- implementsKeyword
- implicitKeyword
- increment
- instanceofKeyword
- intLiteral
- interfaceKeyword
- internalKeyword
- isAssign
- isKeyword
- keyword
- keywords
- lbrace
- lbracket
- lockKeyword
- lparen
- lshift
- lt
- ltEq
- main
- make
- minus
- mixinKeyword
- nativeKeyword
- newKeyword
- notEq
- notSame
- nullKeyword
- objectKeyword
- operatorKeyword
- overrideKeyword
- packageKeyword
- percent
- pipe
- plus
- pound
- privateKeyword
- protectedKeyword
- publicKeyword
- question
- rbrace
- rbracket
- readonlyKeyword
- refKeyword
- returnKeyword
- rparen
- rshift
- same
- sealedKeyword
- semicolon
- sizeofKeyword
- slash
- stackallocKeyword
- star
- staticKeyword
- strLiteral
- strictfpKeyword
- structKeyword
- superKeyword
- switchKeyword
- symbol
- synchronizedKeyword
- thisKeyword
- throwKeyword
- throwsKeyword
- tilde
- toExprId
- toShortcutOp
- transientKeyword
- trueKeyword
- tryKeyword
- typeofKeyword
- uncheckedKeyword
- unsafeKeyword
- uriLiteral
- usingKeyword
- values
- virtualKeyword
- voidKeyword
- volatileKeyword
- whileKeyword