logo

class

compiler::Tokenize

sys::Obj
  compiler::CompilerSupport
    compiler::CompilerStep
      compiler::Tokenize
  1  //
  2  // Copyright (c) 2006, Brian Frank and Andy Frank
  3  // Licensed under the Academic Free License version 3.0
  4  //
  5  // History:
  6  //   5 Jun 06  Brian Frank  Creation
  7  //
  8  
  9  **
 10  ** Tokenize is responsible for parsing all the source files into a
 11  ** a list of tokens.  Each source file is mapped to a CompilationUnit
 12  ** and stored in the PodDef.units field:
 13  **   Compiler.srcFiles -> Compiler.pod.units
 14  **
 15  class Tokenize : CompilerStep
 16  {
 17  
 18  //////////////////////////////////////////////////////////////////////////
 19  // Construction
 20  //////////////////////////////////////////////////////////////////////////
 21  
 22    **
 23    ** Constructor takes the associated Compiler
 24    **
 25    new make(Compiler compiler)
 26      : super(compiler)
 27    {
 28    }
 29  
 30  //////////////////////////////////////////////////////////////////////////
 31  // Run
 32  //////////////////////////////////////////////////////////////////////////
 33  
 34    **
 35    ** Run the step on the list of source files
 36    **
 37    override Void run()
 38    {
 39      log.debug("Tokenize")
 40  
 41      units := CompilationUnit[,]
 42      compiler.srcFiles.each |File srcFile|
 43      {
 44        try
 45        {
 46          location := Location.makeFile(srcFile)
 47          src := srcFile.readAllStr
 48          units.add(tokenize(location, src))
 49        }
 50        catch (CompilerErr err)
 51        {
 52          throw err
 53        }
 54        catch
 55        {
 56          if (srcFile.exists)
 57            throw err("Cannot read source file", Location.makeFile(srcFile))
 58          else
 59            throw err("Source file not found", Location.makeFile(srcFile))
 60        }
 61      }
 62      compiler.pod.units = units
 63    }
 64  
 65    **
 66    ** Run the step on the specified source string
 67    **
 68    Void runSource(Location location, Str src)
 69    {
 70      log.debug("Tokenize")
 71      unit := tokenize(location, src)
 72      compiler.pod.units = [unit]
 73    }
 74  
 75    **
 76    ** Tokenize the source into a CompilationUnit
 77    **
 78    CompilationUnit tokenize(Location location, Str src)
 79    {
 80      unit := CompilationUnit.make(location, compiler.pod)
 81      tokenizer := Tokenizer.make(compiler, location, src, compiler.input.includeDoc)
 82      unit.tokens = tokenizer.tokenize
 83      return unit
 84    }
 85  
 86  }