% Tokenizer and file input from Ryan Wagner, Fall 2005
%The tokenizer code was changed slightly from the original. The call
%            read_file(Filename,X).
%   returns X as a list with the tokens from the file. 
%
%---------------------TOKENIZER---------------------------

gather(Chars) --> [C],  {alphaNumeric(C)},  gather(Rest),
{Chars=[C|Rest]}.

gather([]) --> {true}.
alphaNumeric(C):- 96<C,C<123;
                    64<C, C<91;
                    47<C, C<58.
tokenize(Result)-->gather(Chars),{\+ Chars =[]},tokenize(RestResult),
                        {name(N,Chars), Result=[N|RestResult]}.
tokenize(R)-->[C],{C<33},tokenize(R).
tokenize([N|R]) --> [C],{C>32},
                        {name(N,[C])},tokenize(R).
tokenize([])-->[].

tokenize(X,Y):- tokenize(X, Y, _).

%---------------------FILE IO-----------------------------

read_file(Filename, Y) :-
        see(Filename), read_chars(X), seen, tokenize(Y, X).


read_chars( Chars ):- get0(C), read_chars_2( C, Chars ).

read_chars_2(X, Y):- (X == -1), Y = [].
read_chars_2( C, [C|T] ):- read_chars( T ).