1 //
2 // Copyright (c) 2006, Brian Frank and Andy Frank
3 // Licensed under the Academic Free License version 3.0
4 //
5 // History:
6 // 5 Jun 06 Brian Frank Creation
7 //
8
9 **
10 ** Tokenize is responsible for parsing all the source files into a
11 ** a list of tokens. Each source file is mapped to a CompilationUnit
12 ** and stored in the PodDef.units field:
13 ** Compiler.srcFiles -> Compiler.pod.units
14 **
15 class Tokenize : CompilerStep
16 {
17
18 //////////////////////////////////////////////////////////////////////////
19 // Construction
20 //////////////////////////////////////////////////////////////////////////
21
22 **
23 ** Constructor takes the associated Compiler
24 **
25 new make(Compiler compiler)
26 : super(compiler)
27 {
28 }
29
30 //////////////////////////////////////////////////////////////////////////
31 // Run
32 //////////////////////////////////////////////////////////////////////////
33
34 **
35 ** Run the step on the list of source files
36 **
37 override Void run()
38 {
39 log.debug("Tokenize")
40
41 units := CompilationUnit[,]
42 compiler.srcFiles.each |File srcFile|
43 {
44 try
45 {
46 location := Location.makeFile(srcFile)
47 src := srcFile.readAllStr
48 units.add(tokenize(location, src))
49 }
50 catch (CompilerErr err)
51 {
52 throw err
53 }
54 catch
55 {
56 if (srcFile.exists)
57 throw err("Cannot read source file", Location.makeFile(srcFile))
58 else
59 throw err("Source file not found", Location.makeFile(srcFile))
60 }
61 }
62 compiler.pod.units = units
63 }
64
65 **
66 ** Run the step on the specified source string
67 **
68 Void runSource(Location location, Str src)
69 {
70 log.debug("Tokenize")
71 unit := tokenize(location, src)
72 compiler.pod.units = [unit]
73 }
74
75 **
76 ** Tokenize the source into a CompilationUnit
77 **
78 CompilationUnit tokenize(Location location, Str src)
79 {
80 unit := CompilationUnit.make(location, compiler.pod)
81 tokenizer := Tokenizer.make(compiler, location, src, compiler.input.includeDoc)
82 unit.tokens = tokenizer.tokenize
83 return unit
84 }
85
86 }