summaryrefslogtreecommitdiff
path: root/05/tokenize.b
blob: 56838c577337c035b7c8f5ae8e11407acbda38d4 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
global file_list ; initialized in main -- null-separated 255-terminated array of strings

; get the name of the file with the given index
function file_get
	argument idx
	local p
	p = file_list
	:file_get_loop
		if idx == 0 goto file_got
		if *1p == 255 goto file_uhoh
		idx -= 1
		p = memchr(p, 0)
		p += 1
		goto file_get_loop
	:file_got
	return p
	:file_uhoh
	puts(.str_bad_file_index)
	exit(1)
	:str_bad_file_index
		string Bad file index. This shouldn't happen.
		byte 10
		byte 0
	
; get the index of the given file, returns -1 if file does not exist
function file_get_index
	argument filename
	local p
	local b
	local i
	p = file_list
	i = 0
	:file_get_index_loop
		if *1p == 255 goto return_minus1
		b = str_equals(p, filename)
		if b != 0 goto file_found
		i += 1
		p = memchr(p, 0)
		p += 1
		goto file_get_index_loop
	:file_found
	return i
	
; add to list of files if not already there
function file_add
	argument filename
	local p
	p = file_get_index(filename)
	if p != -1 goto return_0
	p = memchr(file_list, 255)
	p = strcpy(p, filename)
	p += 1
	*1p = 255
	return

; return keyword ID associated with str, or 0 if it's not a keyword
function get_keyword_id
	argument keyword_str
	local p
	local c
	local b
	p = .keyword_table
	:keyword_id_loop
		c = *1p
		if c == 255 goto no_such_keyword_str
		p += 1
		b = str_equals(keyword_str, p)
		if b != 0 goto got_keyword_id
		p = memchr(p, 0)
		p += 1
		goto keyword_id_loop
	:no_such_keyword_str
	return 0
	:got_keyword_id
	return c

; get string associated with keyword id, or "@BAD_KEYWORD_ID" if it's not a keyword
function get_keyword_str
	argument keyword_id
	local p
	local c
	local b
	p = .keyword_table
	:keyword_str_loop
		c = *1p
		if c == 255 goto no_such_keyword_id
		if c == keyword_id goto found_keyword_id
		p = memchr(p, 0)
		p += 1
		goto keyword_str_loop
	:found_keyword_id
		return p + 1
	:no_such_keyword_id
		return .str_no_such_keyword_id
	:str_no_such_keyword_id
		string @BAD_KEYWORD_ID
		byte 0

; returns a unique number associated with the line `token` appears on.
function token_get_location
	argument token
	return *8token > 16 ; right shift by 16 to remove type,info, and extract 6 bytes of file,line

; turn pptokens into tokens, written to out.
; This corresponds to translation phases 5-6 and the first half of 7
; IMPORTANT: this function uses pointers to pptokens, so it should NOT be freed!
; Returns a pointer to the end of tokens.
function tokenize
	argument pptokens
	argument out
	; you might think we wouldn't need these arguments because the pptokens array starts with
	; a line directive. but we also use this function to tokenize the expression of a #if,
	; where that isn't the case.
	argument initial_filename
	argument initial_line_number
	local in
	local file
	local line_number
	local b
	local c
	local n
	local p
	local data
	local significand
	local exponent
	local new_exponent
	local pow10
	local integer
	local fraction
	local lower
	local upper
	
	file_add(initial_filename)
	file = file_get_index(initial_filename)
	line_number = initial_line_number
	
	
	in = pptokens
	:tokenize_loop
		c = *1in
		if c == '$ goto tokenize_line_directive
		if c == 32 goto tokenize_skip_pptoken
		if c == 10 goto tokenize_newline
		if c == '' goto tokenize_constant_char
		if c == '" goto tokenize_string_literal
		if c == 0 goto tokenize_loop_end
		
		b = get_keyword_id(in)
		if b != 0 goto tokenize_keyword
		
		b = isdigit_or_dot(c)
		if b != 0 goto tokenize_number
		
		; it's an identifier. we just need to make sure it's made up of identifier characters.
		p = in
		b = isalpha_or_underscore(*1p)
		if b == 0 goto bad_token
		
		:ident_check_loop
			b = isalnum_or_underscore(*1p)
			if b == 0 goto bad_token
			p += 1
			if *1p != 0 goto ident_check_loop
		; all good.
		*1out = TOKEN_IDENTIFIER
		out += 2 ; no info
		data = in ; data will point to the identifier name
		pptoken_skip(&in)
		goto token_output
		
		:tokenize_newline
			line_number += 1
			pptoken_skip(&in)
			goto tokenize_loop
		:tokenize_skip_pptoken
			pptoken_skip(&in)
			goto tokenize_loop
		:tokenize_line_directive
			in += 1
			line_number = stoi(in)
			in = memchr(in, 32)
			in += 1
			file_add(in)
			file = file_get_index(in)
			pptoken_skip(&in)
			goto tokenize_loop
		:token_no_data
			data = 0
			; (fallthrough)
		:token_output ; write token location & data (see local variable data), and continue tokenizing
			*2out = file
			out += 2
			*4out = line_number
			out += 4
			*8out = data
			out += 8
			goto tokenize_loop
		:tokenize_keyword
			pptoken_skip(&in)
			*1out = b ; type
			; no info for keywords
			out += 2
			goto token_no_data
		:tokenize_number
			; first, check if it's a float
			b = strchr(in, '.)
			if b != 0 goto tokenize_float
			b = strchr(in, 'x) ; e may appear in hex integer literals, so we need to check this
			if b != 0 goto tokenize_hex_integer
			b = strchr(in, 'X)
			if b != 0 goto tokenize_hex_integer
			b = strchr(in, 'e) ; exponent
			if b != 0 goto tokenize_float
			b = strchr(in, 'E) ; exponent
			if b != 0 goto tokenize_float
			if *1in == '0 goto tokenize_octal_integer ; fun fact: in the C89 standard, 0 is considered an octal integer
				; plain ol' decimal constant
				n = strtoi(&in, 10)
				goto tokenize_finish_integer
			:tokenize_hex_integer
				if *1in != '0 goto bad_number_token
				in += 1
				c = *1in
				c &= 223 ; 223 = ~32 -- remove case
				if c != 'X goto bad_number_token
				in += 1
				n = strtoi(&in, 16)
				goto tokenize_finish_integer
			:tokenize_octal_integer
				in += 1 ; skip 0
				n = strtoi(&in, 8)
				goto tokenize_finish_integer
			:tokenize_finish_integer
			c = read_number_suffix(file, line_number, &in)
			if c == NUMBER_SUFFIX_F goto f_suffix_on_integer
			in += 1 ; move past null separator
			*1out = TOKEN_CONSTANT_INT
			out += 1
			*1out = c ; info = suffix
			out += 1
			data = n
			goto token_output
		:tokenize_constant_char
			in += 1
			c = read_c_char(&in)
			if *1in != '' goto bad_char_constant
			if c ] 255 goto bad_char_constant
			pptoken_skip(&in)
			*1out = TOKEN_CONSTANT_CHAR
			out += 2 ; no info
			data = c
			goto token_output
		:tokenize_string_literal
			data = rodata_end_addr
			p = output_file_data + rodata_end_addr
			
			:string_literal_loop
				in += 1 ; skip opening "
				:string_literal_char_loop
					if *1in == '" goto string_literal_char_loop_end
					c = read_c_char(&in)
					if c ] 255 goto bad_char_in_string
					*1p = c
					p += 1
					goto string_literal_char_loop					
				:string_literal_char_loop_end
				pptoken_skip(&in) ; skip closing "
				pptoken_skip_whitespace(&in, &line_number)
				if *1in == '" goto string_literal_loop ; string concatenation, e.g. "Hello, " "world!"
			*1p = 0 ; null terminator
			p += 1
			rodata_end_addr = p - output_file_data
			
			*1out = TOKEN_STRING_LITERAL
			out += 2 ; no info
			goto token_output
		:tokenize_float
			; @NONSTANDARD: this doesn't allow for floats whose integral part is >=2^64, e.g. 1000000000000000000000000.0
			significand = 0
			exponent = 0
			pow10 = 0
			integer = strtoi(&in, 10)
			fraction = 0
			if *1in != '. goto float_no_fraction 
			in += 1
			p = in
			fraction = strtoi(&in, 10)
			; e.g. to turn 35 into .35, multiply by 10^-2
			pow10 = p - in
			;putnln_signed(pow10)
			if pow10 < -400 goto bad_float
			:float_no_fraction
			; construct the number integer + fraction*10^pow10
			; first, deal with the fractional part
			significand = fraction
			float_multiply_by_power_of_10(&significand, &exponent, pow10)
			
			if integer == 0 goto float_no_integer
			; now deal with the integer part
			new_exponent = leftmost_1bit(integer)
			new_exponent -= 58
			n = new_exponent - exponent
			significand = right_shift(significand, n)
			exponent = new_exponent
			significand += right_shift(integer, exponent)
			:float_no_integer
			
			if *1in == 'e goto float_exponent
			if *1in == 'E goto float_exponent
			
			:float_have_significand_and_exponent
			if significand == 0 goto float_zero
			normalize_float(&significand, &exponent)
			; make number round to the nearest representable float roughly (this is what gcc does)
			; this fails for 5e-100 probably because of imprecision, but mostly works
			significand += 15
			; reduce to 53-bit significand (top bit is removed to get 52)
			significand >= 5
			exponent += 5
			exponent += 52  ; 1001010111... => 1.001010111... 
			n = leftmost_1bit(significand)
			exponent += n - 52 ; in most cases, this is 0, but sometimes it turns out to be 1.
			b = 1 < n
			significand &= ~b
			data = significand
			
			if exponent <= -1023 goto float_zero ; this number is too small in magnitude to be represented as a double. it becomes 0
			if exponent >= 1024 goto float_infinity ; number too big to be represented as a double.
			exponent += 1023 ; float format
			
			data |= exponent < 52
			
			:float_have_data
			*1out = TOKEN_CONSTANT_FLOAT
			out += 1
			; suffix
			*1out = read_number_suffix(file, line_number, &in)
			pptoken_skip(&in)
			out += 1
			goto token_output
			:float_exponent
				in += 1
				if *1in == '+ goto float_exponent_plus
				if *1in == '- goto float_exponent_minus
				; e.g. 1e100
				pow10 = strtoi(&in, 10)
				:float_have_exponent
				float_multiply_by_power_of_10(&significand, &exponent, pow10)
				goto float_have_significand_and_exponent
			:float_exponent_plus
				; e.g. 1e+100
				in += 1
				pow10 = strtoi(&in, 10)
				goto float_have_exponent
			:float_exponent_minus
				; e.g. 1e-100
				in += 1
				pow10 = strtoi(&in, 10)
				pow10 = 0 - pow10
				goto float_have_exponent
			:float_zero
				data = 0
				goto float_have_data
			:float_infinity
				data = 0x7ff0000000000000 ; double infinity
				goto float_have_data
	:tokenize_loop_end
	; EOF token
	*1out = TOKEN_EOF
	out += 2
	*2out = file
	out += 2
	*4out = line_number
	out += 12
	
	return out
	:f_suffix_on_integer
		compile_error(file, line_number, .str_f_suffix_on_integer)
	:str_f_suffix_on_integer
		string Integer with f suffix.
		byte 0
	:bad_number_token
		compile_error(file, line_number, .str_bad_number_token)
	:str_bad_number_token
		string Bad number literal.
		byte 0
	:bad_char_constant
		compile_error(file, line_number, .str_bad_char_constant)
	:str_bad_char_constant
		string Bad character constant. Note that multibyte constants are not supported.
		byte 0
	:bad_char_in_string
		compile_error(file, line_number, .str_bad_char_in_string)
	:str_bad_char_in_string
		string Bad character in string literal.
		byte 0
	:bad_token
		compile_error(file, line_number, .str_bad_token)
	:str_bad_token
		string Bad token.
		byte 0
	:bad_float
		compile_error(file, line_number, .str_bad_float)
	:str_bad_float
		string Bad floating-point number.
		byte 0

function float_multiply_by_power_of_10
	argument p_significand
	argument p_exponent
	argument pow10
	local significand
	local exponent
	local p
	local lower
	local upper
	local n
	significand = *8p_significand
	exponent = *8p_exponent

	p = powers_of_10
	p += pow10 < 4
	full_multiply_signed(significand, *8p, &lower, &upper)
	if upper == 0 goto fmultiply2_no_upper
		n = leftmost_1bit(upper)
		n += 1
		significand = lower > n
		exponent += n
		n = 64 - n
		significand |= upper < n
		goto fmultiply2_cont
	:fmultiply2_no_upper
		significand = lower
		goto fmultiply2_cont
	:fmultiply2_cont
	p += 8
	exponent += *8p
	
	*8p_significand = significand
	*8p_exponent = exponent
	return 0
	
; return character or escaped character from *p_in, advancing accordingly
; returns -1 on bad character
function read_c_char
	argument p_in
	local in
	local c
	local x
	in = *8p_in
	if *1in == '\ goto escape_sequence
	; no escape sequence; just a normal character
	c = *1in
	in += 1
	goto escape_sequence_return
	
	:escape_sequence
		in += 1
		c = *1in
		in += 1
		if c == 'x goto escape_sequence_hex
		if c == '' goto escape_sequence_single_quote
		if c == '" goto escape_sequence_double_quote
		if c == '? goto escape_sequence_question
		if c == '\ goto escape_sequence_backslash
		if c == 'a goto escape_sequence_bell
		if c == 'b goto escape_sequence_backspace
		if c == 'f goto escape_sequence_form_feed
		if c == 'n goto escape_sequence_newline
		if c == 'r goto escape_sequence_carriage_return
		if c == 't goto escape_sequence_tab
		if c == 'v goto escape_sequence_vertical_tab
		; octal
		in -= 1
		x = isoctdigit(*1in)
		if x == 0 goto return_minus1
		c = *1in - '0
		in += 1
		x = isoctdigit(*1in)
		if x == 0 goto escape_sequence_return
		c <= 3
		c += *1in - '0
		in += 1
		x = isoctdigit(*1in)
		if x == 0 goto escape_sequence_return
		c <= 3
		c += *1in - '0
		in += 1
		if c ] 255 goto return_minus1 ; e.g. '\712'
		goto escape_sequence_return
	:escape_sequence_hex
		x = in
		c = strtoi(&in, 16)
		if in == x goto return_minus1 ; e.g. '\xhello'
		if c ] 255 goto return_minus1 ; e.g. '\xabc'
		goto escape_sequence_return
	:escape_sequence_single_quote
		c = ''
		goto escape_sequence_return
	:escape_sequence_double_quote
		c = '"
		goto escape_sequence_return
	:escape_sequence_question
		c = '?
		goto escape_sequence_return
	:escape_sequence_backslash
		c = '\
		goto escape_sequence_return
	:escape_sequence_bell
		c = 7
		goto escape_sequence_return
	:escape_sequence_backspace
		c = 8
		goto escape_sequence_return
	:escape_sequence_form_feed
		c = 12
		goto escape_sequence_return
	:escape_sequence_newline
		c = 10
		goto escape_sequence_return
	:escape_sequence_carriage_return
		c = 13
		goto escape_sequence_return
	:escape_sequence_tab
		c = 9
		goto escape_sequence_return
	:escape_sequence_vertical_tab
		c = 11
		goto escape_sequence_return
	:escape_sequence_return
	*8p_in = in
	return c


function read_number_suffix
	argument file
	argument line_number
	argument p_s
	local s
	local c
	local suffix
	s = *8p_s
	c = *1s
	suffix = 0
	if c == 0 goto number_suffix_return
	if c == 'u goto number_suffix_u
	if c == 'U goto number_suffix_u
	if c == 'l goto number_suffix_l
	if c == 'L goto number_suffix_l
	if c == 'f goto number_suffix_f
	if c == 'F goto number_suffix_f
	goto bad_number_suffix
	:number_suffix_u
		s += 1
		c = *1s
		if c == 'l goto number_suffix_ul
		if c == 'L goto number_suffix_ul
		if c != 0 goto bad_number_suffix
		suffix = NUMBER_SUFFIX_U
		goto number_suffix_return
	:number_suffix_l
		s += 1
		c = *1s
		if c == 'u goto number_suffix_ul
		if c == 'U goto number_suffix_ul
		if c == 'l goto number_suffix_l  ; handle ll suffix (even though it's C99)
		if c == 'L goto number_suffix_l
		if c != 0 goto bad_number_suffix
		suffix = NUMBER_SUFFIX_L
		goto number_suffix_return
	:number_suffix_ul
		s += 1
		c = *1s
		if c == 'l goto number_suffix_l  ; handle ll suffix (even though it's C99)
		if c == 'L goto number_suffix_l
		if c != 0 goto bad_number_suffix
		suffix = NUMBER_SUFFIX_UL
		goto number_suffix_return
	:number_suffix_f
		s += 1
		c = *1s
		if c != 0 goto bad_number_suffix
		suffix = NUMBER_SUFFIX_F
		goto number_suffix_return
	:number_suffix_return
	*8p_s = s
	return suffix
	
	:bad_number_suffix
		compile_error(file, line_number, .str_bad_number_suffix)
	:str_bad_number_suffix
		string Bad number suffix.
		byte 0
	
function print_tokens
	argument tokens
	argument tokens_end
	local p
	local s
	p = tokens
	:print_tokens_loop
		if p ]= tokens_end goto print_tokens_loop_end
		if *1p == 0 goto print_tokens_loop_end
		if *1p > 20 goto print_token_keyword 
		if *1p == TOKEN_CONSTANT_INT goto print_token_int
		if *1p == TOKEN_CONSTANT_CHAR goto print_token_char
		if *1p == TOKEN_CONSTANT_FLOAT goto print_token_float
		if *1p == TOKEN_STRING_LITERAL goto print_token_string_literal
		if *1p == TOKEN_IDENTIFIER goto print_token_identifier
		if *1p == TOKEN_EOF goto print_token_eof
		puts(.str_print_bad_token)
		exit(1)
		:print_token_keyword
			s = get_keyword_str(*1p)
			puts(s)
			goto print_token_data
		:print_token_int
			puts(.str_constant_int)
			goto print_token_info
		:print_token_char
			puts(.str_constant_char)
			goto print_token_data
		:print_token_string_literal
			puts(.str_string_literal)
			goto print_token_data
		:print_token_identifier
			s = p + 8
			puts(*8s)
			goto print_token_data
		:print_token_float
			p += 8
			puts(.str_constant_float)
			putx64(*8p)
			p += 8
			putc(32)
			goto print_tokens_loop
		:print_token_eof
			puts(.str_eof)
			goto print_token_data
		:print_token_info
		p += 1
		putc('~)
		putn(*1p)
		p -= 1
		:print_token_data
		p += 2
		putc('@)
		putn(*2p)
		p += 2
		putc(':)
		putn(*4p)
		p += 4
		putc(61)
		putn(*8p)
		p += 8
		putc(32)
		goto print_tokens_loop
	:print_tokens_loop_end
	putc(10)
	return
	:str_constant_int
		string integer
		byte 0
	:str_constant_float
		string float
		byte 0
	:str_constant_char
		string character
		byte 0
	:str_string_literal
		string string
		byte 0
	:str_print_bad_token
		string Unrecognized token type in print_tokens. Aborting.
		byte 10
		byte 0
	:str_eof
		string EOF
		byte 0

function print_token
	argument token
	local p
	p = token + 16
	print_tokens(token, p)
	return