--
-- Copyright (c) 2012 Kevin Wellwood
-- All rights reserved.
--
-- This source code is distributed under the Modified BSD License. For terms and
-- conditions, see license.txt.
--
with Ada.Streams; use Ada.Streams;
with Ada.Streams.Stream_IO; use Ada.Streams.Stream_IO;
private with Ada.Containers.Doubly_Linked_Lists;
private with Ada.Unchecked_Deallocation;
package Tokens.Tokenizers is
-- A Tokenizer reads from an input stream and splits the text using a set of
-- token-delimiter characters to returns a series of Token objects. If the
-- stream contains character sequences that are not recognizable tokens, or
-- are malformed tokens, an exception will be raised and the erroneous text
-- discarded.
type Tokenizer is new Limited_Object with private;
type A_Tokenizer is access all Tokenizer'Class;
-- Creates a new Tokenizer.
function Create_Tokenizer return A_Tokenizer;
-- Returns the tokenizer's current location in the input stream (or the
-- location of the next token to be returned, if tokens have been pushed
-- back.)
function Get_Location( this : not null access Tokenizer'Class ) return Token_Location;
-- Returns the next Token read from the input stream. If tokens were
-- previously pushed back to the tokenizer, the most recent will be returned
-- now instead. Token_Exception will be raised if the input stream contents
-- can't be recognized as a token or the token is malformed.
function Get_Next( this : not null access Tokenizer'Class ) return A_Token;
-- Returns the most recent Token received from the Tokenizer. The next call
-- to Get_Next will return 'token' instead of reading from the token stream.
-- Multiple tokens can be returned to the Tokenizer, in the order in which
-- they were originally received by the caller. 'token' will be consumed.
procedure Push_Back( this : not null access Tokenizer'Class;
token : in out A_Token );
pragma Precondition( token /= null );
pragma Postcondition( token = null );
-- Sets the input stream for reading characters. The state of the Tokenizer
-- will also be reset. If 'stream' is null, the Tokenizer's input will be
-- cleared.
procedure Set_Input( this : not null access Tokenizer'Class;
stream : Stream_Access );
-- Deletes the Tokenizer.
procedure Delete( this : in out A_Tokenizer );
pragma Postcondition( this = null );
-- Raised when a token is malformed or unrecognized.
Token_Exception : exception;
private
-- Raised when the input stream is empty.
End_Error : exception renames Ada.Streams.Stream_IO.End_Error;
package Token_Lists is new Ada.Containers.Doubly_Linked_Lists( A_Token, "=" );
type A_String is access all String;
procedure Delete is new Ada.Unchecked_Deallocation( String, A_String );
----------------------------------------------------------------------------
type Tokenizer is new Limited_Object with
record
stream : Stream_Access := null;
loc : Token_Location := (line => 1, col => 1);
tokenLoc : Token_Location;
peeked : Boolean := False;
peekChar : Character := ASCII.NUL;
returnedTokens : Token_Lists.List;
tokenBuf : A_String := new String(1..32);
end record;
procedure Delete( this : in out Tokenizer );
end Tokens.Tokenizers;