with Ada.Streams; use Ada.Streams;
with Ada.Streams.Stream_IO; use Ada.Streams.Stream_IO;
with Interfaces; use Interfaces;
private with Ada.Containers.Doubly_Linked_Lists;
private with Ada.Unchecked_Deallocation;
package Tokens.Tokenizers is
type Tokenizer is new Limited_Object with private;
type A_Tokenizer is access all Tokenizer'Class;
function Create_Tokenizer return A_Tokenizer;
function Get_Location( this : not null access Tokenizer'Class ) return Token_Location;
function Get_Next( this : not null access Tokenizer'Class ) return A_Token;
procedure Push_Back( this : not null access Tokenizer'Class;
token : in out A_Token );
pragma Precondition( token /= null );
pragma Postcondition( token = null );
procedure Set_Input( this : not null access Tokenizer'Class;
stream : Stream_Access );
procedure Delete( this : in out A_Tokenizer );
pragma Postcondition( this = null );
Token_Exception : exception;
private
End_Error : exception renames Ada.Streams.Stream_IO.End_Error;
CT_ANY : constant := 0;
CT_WHITESPACE : constant := 1;
CT_DELIMITER : constant := 2;
type Character_Type_Array is array(Character) of Unsigned_8;
charTypes : Character_Type_Array := Character_Type_Array'(others => CT_ANY);
function Is_Delimiter( c : Character ) return Boolean;
function Is_Whitespace( c : Character ) return Boolean;
package Token_Lists is new Ada.Containers.Doubly_Linked_Lists( A_Token, "=" );
type Character_Array is array(Integer range <>) of Character;
type A_Character_Array is access all Character_Array;
type A_String is access all String;
procedure Delete is new Ada.Unchecked_Deallocation( String, A_String );
type Tokenizer is new Limited_Object with
record
stream : Stream_Access := null;
loc : Token_Location := (line => 1, col => 1);
tokenLoc : Token_Location;
peeked : Boolean := False;
peekChar : Character := ASCII.NUL;
returnedTokens : Token_Lists.List;
tokenBuf : A_String := new String(1..32);
end record;
procedure Delete( this : in out Tokenizer );
procedure Grow_Buffer( this : not null access Tokenizer'Class );
procedure Mark_Token_Location( this : not null access Tokenizer'Class );
function Read( this : not null access Tokenizer'Class ) return Character;
end Tokens.Tokenizers;