1. -- 
  2. -- Copyright (c) 2012 Kevin Wellwood 
  3. -- All rights reserved. 
  4. -- 
  5. -- This source code is distributed under the Modified BSD License. For terms and 
  6. -- conditions, see license.txt. 
  7. -- 
  8.  
  9. with Ada.Streams;                       use Ada.Streams; 
  10. with Ada.Streams.Stream_IO;             use Ada.Streams.Stream_IO; 
  11.  
  12. private with Ada.Containers.Doubly_Linked_Lists; 
  13. private with Ada.Unchecked_Deallocation; 
  14.  
  15. package Tokens.Tokenizers is 
  16.  
  17.     -- A Tokenizer reads from an input stream and splits the text using a set of 
  18.     -- token-delimiter characters to returns a series of Token objects. If the 
  19.     -- stream contains character sequences that are not recognizable tokens, or 
  20.     -- are malformed tokens, an exception will be raised and the erroneous text 
  21.     -- discarded. 
  22.     type Tokenizer is new Limited_Object with private; 
  23.     type A_Tokenizer is access all Tokenizer'Class; 
  24.  
  25.     -- Creates a new Tokenizer. 
  26.     function Create_Tokenizer return A_Tokenizer; 
  27.  
  28.     -- Returns the tokenizer's current location in the input stream (or the 
  29.     -- location of the next token to be returned, if tokens have been pushed 
  30.     -- back.) 
  31.     function Get_Location( this : not null access Tokenizer'Class ) return Token_Location; 
  32.  
  33.     -- Returns the next Token read from the input stream. If tokens were 
  34.     -- previously pushed back to the tokenizer, the most recent will be returned 
  35.     -- now instead. Token_Exception will be raised if the input stream contents 
  36.     -- can't be recognized as a token or the token is malformed. 
  37.     function Get_Next( this : not null access Tokenizer'Class ) return A_Token; 
  38.  
  39.     -- Returns the most recent Token received from the Tokenizer. The next call 
  40.     -- to Get_Next will return 'token' instead of reading from the token stream. 
  41.     -- Multiple tokens can be returned to the Tokenizer, in the order in which 
  42.     -- they were originally received by the caller. 'token' will be consumed. 
  43.     procedure Push_Back( this  : not null access Tokenizer'Class; 
  44.                          token : in out A_Token ); 
  45.     pragma Precondition( token /= null ); 
  46.     pragma Postcondition( token = null ); 
  47.  
  48.     -- Sets the input stream for reading characters. The state of the Tokenizer 
  49.     -- will also be reset. If 'stream' is null, the Tokenizer's input will be 
  50.     -- cleared. 
  51.     procedure Set_Input( this   : not null access Tokenizer'Class; 
  52.                          stream : Stream_Access ); 
  53.  
  54.     -- Deletes the Tokenizer. 
  55.     procedure Delete( this : in out A_Tokenizer ); 
  56.     pragma Postcondition( this = null ); 
  57.  
  58.     -- Raised when a token is malformed or unrecognized. 
  59.     Token_Exception : exception; 
  60.  
  61. private 
  62.  
  63.     -- Raised when the input stream is empty. 
  64.     End_Error : exception renames Ada.Streams.Stream_IO.End_Error; 
  65.  
  66.     package Token_Lists is new Ada.Containers.Doubly_Linked_Lists( A_Token, "=" ); 
  67.  
  68.     type A_String is access all String; 
  69.  
  70.     procedure Delete is new Ada.Unchecked_Deallocation( String, A_String ); 
  71.  
  72.     ---------------------------------------------------------------------------- 
  73.  
  74.     type Tokenizer is new Limited_Object with 
  75.         record 
  76.             stream         : Stream_Access := null; 
  77.             loc            : Token_Location := (line => 1, col => 1); 
  78.             tokenLoc       : Token_Location; 
  79.             peeked         : Boolean := False; 
  80.             peekChar       : Character := ASCII.NUL; 
  81.             returnedTokens : Token_Lists.List; 
  82.             tokenBuf       : A_String := new String(1..32); 
  83.         end record; 
  84.  
  85.     procedure Delete( this : in out Tokenizer ); 
  86.  
  87. end Tokens.Tokenizers;