From ok@atlas.otago.ac.nz  Fri Aug 18 02:44:34 2000
Received: from atlas.otago.ac.nz (atlas.otago.ac.nz [139.80.32.250])
	by swi.psy.uva.nl (8.9.3/8.9.3) with ESMTP id CAA29360
	for <prolog@swi.psy.uva.nl>; Fri, 18 Aug 2000 02:44:32 +0200 (MET DST)
Received: (from ok@localhost)
	by atlas.otago.ac.nz (8.9.3/8.9.3) id MAA26656;
	Fri, 18 Aug 2000 12:44:37 +1200 (NZST)
Date: Fri, 18 Aug 2000 12:44:37 +1200 (NZST)
From: "Richard A. O'Keefe" <ok@atlas.otago.ac.nz>
Message-Id: <200008180044.MAA26656@atlas.otago.ac.nz>
To: aobdeijn@wins.uva.nl, prolog@swi.psy.uva.nl
Subject: Re:  replace Capital letters

	Can anyone tell me if there's a simple way to replace CAPITAL LETTERS by
	small letters, in a list of words?
	
	the case is the following:
	i ask for a sentence, the user writes a sentence, including capital
	letters, and then i want to parse this sentence,
	which is not possible, since the capitals are seen as variables.

But a sentence is not a Prolog term, so why would anything in it be seen
as a Prolog variable?  If you use RDSENT.PL from the DEC-10 Prolog
library, you'll do
	?- read_sent(S).
	|: "'Twas brillig", said the Walrus to the Carpenter.

	S = [string('\'Twas brillig'),  (,), atom(said), atom(the),
	atom(walrus), atom(to), atom(the), atom(carpenter), '.'] 

	Yes

I enclose a copy of read_sent.pl, originally the PDP-11 version,
but revised for Latin-1 and SWI Prolog.

%   File   : /usr/lib/prolog/read_sent
%   Author : R.A.O'Keefe
%   Updated: 11 November 1983
%   Revised: 2000 for SWI Prolog and ISO 8859-1.
%   Purpose: to provide a flexible input facility
%   Needs  : memberchk from utils.


/*  The main predicates provided exported by this file are
	read_until(+Delimiters, -String)
	read_line(-String)
	trim_blanks(+RawString, -Trimmed)
	read_sent-ListOfTokens).
    Tokens are
	atom(X)    for X a sequence of letters (- and ' NOT included) lowercased
	integer(X) for X the value of a sequence of digits
	aposts     for 's
	apost      for '
	string(X)  for X the content of "....", as atom NOT lowercased
	';' '.'    and so on for punctuation marks ()[]{},;:.!?
    This may need revising for your purposes.
*/

/* public
	case_shift/2,
	chars_to_words/2,
	is_digit/1,
	is_endfile/1,
	is_layout/1,
	is_letter/1,
	is_lower/1,
	is_newline/1,
	is_paren/2,
	is_period/1,
	is_punct/1,
	is_upper/1,
	read_line/1,
	read_sent/1,
	read_until/2,
	trim_blanks/2.
*/
/* mode
	case_shift(+, -),
	chars_to_atom(-, ?, ?),
	chars_to_integer(+, -, ?, ?),
	chars_to_string(+, -, ?, ?),
	chars_to_words(+, -),
	chars_to_words(-, ?, ?),
	chars_to_word(-, ?, ?),
	is_digit(+),
	is_endfile(?),
	is_layout(+),
	is_letter(+),
	is_lower(+),
	is_newline(?),
	is_paren(?, ?),
	is_period(+),
	is_punct(+),
	is_upper(+),
	read_line(-),
	read_sent(-),
	read_until(+, -),
	read_until(+, +, -),
	trim_blanks(+, -),
	trim_blanks_rest_word(+, -),
	trim_blanks_next_word(+, -).
*/


/*  read_until(Delimiters, Answer)
    reads characters from the current input until  a  character  in  the
    Delimiters  string  is  read.  The characters are accumulated in the
    Answer string, and include the closing  delimiter.   Prolog  returns
    end-of-file as ^Z (26)  regardless of the user's assignment (e.g. if
    you use ^D as end of file, Prolog still returns ^Z).  The end of the
    file is always a delimiter.
*/
read_until(Delimiters, [Char|Rest]) :-
    get0(Char),
    read_until(Char, Delimiters, Rest).


read_until(Char, _, S) :-
    is_endfile(Char), !,
    S = [].
read_until(Char, Delimiters, S) :-
    memberchk(Char, Delimiters), !,
    S = [].
read_until(_, Delimiters, Rest) :-
    read_until(Delimiters, Rest).



%   The following predicates define useful character classes.

%is_newline(31).			% the Dec-10 line terminator.
is_newline(10).				% UNIX systems: use 10.

%is_endfile(26).			% the file terminator ^Z
is_endfile(-1).				% modern systems: use -1.

is_layout(Char) :-
    Char =< " ".			% includes tab, newline, ^S, &c

is_lower(Char) :-
    (   Char =< "z" -> Char >= "a"	% lower case ASCII letter
    ;   Char >= 223, Char =< 255,	% lower case Latin-1 letter
	Char =\= 247			% (not division sign)
    ).

is_upper(Char) :-
    (	Char =< "Z" -> Char >= "A"	% upper case ASCII letter
    ;   Char >= 192, Char =< 222,	% upper case Latin-1 letter
	Char =\= 215			% (not multiplication sign)
    ).

is_letter(Char) :-
    is_lower(Char).
is_letter(Char) :-
    is_upper(Char).

is_digit(Char) :-
    Char >= "0", Char =< "9".		% decimal digit

is_period(46 /* . */).			% sentence terminator
is_period(33 /* ! */).
is_period(63 /* ? */).

is_punct(44 /* , */).			% other punctuation mark
is_punct(59 /* ; */).
is_punct(58 /* : */).

is_paren( 40 /* ( */,  41 /* ) */).	% is_paren(Left, Right)
is_paren( 91 /* [ */,  93 /* ] */).	% for matching pairs of
is_paren(123 /* { */, 125 /* } */).	% brackets.
is_paren(171 /* <<*/, 187 /* >>*/).	% Guillemet quotation marks.


/*  trim_blanks(RawInput, Cleaned)
    removes leading and trailing layout characters  from  RawInput,  and
    replaces  internal  groups  of  layout  characters by single spaces.
    Thus trim_blanks(<|TAB TAB a SP ^M ^E b ^Z|>, "a b") would be true.
*/
trim_blanks([Char|Chars], Cleaned) :-
    is_layout(Char), !,
    trim_blanks(Chars, Cleaned).
trim_blanks([Char|Chars], [Char|Cleaned]) :-
    trim_blanks_rest_word(Chars, Cleaned).
trim_blanks([], []).


trim_blanks_rest_word([Char|Chars], Cleaned) :-
    is_layout(Char), !,
    trim_blanks_next_word(Chars, Cleaned).
trim_blanks_rest_word([Char|Chars], [Char|Cleaned]) :-
    trim_blanks_rest_word(Chars, Cleaned).
trim_blanks_rest_word([], []).


trim_blanks_next_word([Char|Chars], Cleaned) :-
    is_layout(Char), !,
    trim_blanks_next_word(Chars, Cleaned).
trim_blanks_next_word([Char|Chars], [32 /*   */,Char|Cleaned]) :-
    trim_blanks_rest_word(Chars, Cleaned).
trim_blanks_next_word([], []).



/*  chars_to_words(Chars, Words)
    parses a list of characters (read by read_until) into a list of
    tokens, where a token is
	'X' for X a period or other punctuation mark, e.g. ';'
	atom(X) for X a sequence of letters, e.g. atom(the)
	integer(X) for X a sequence of digits, e.g. integer(12)
	apost for '
	aposts for 's
	string(X) for X "..sequence of any.."
    Thus the string "the "Z-80" is on card 12." would be parsed as
	[atom(the),string('Z-80'),atom(is),atom(on),atom(card),
	 integer(12),'.'].
    It is up to the sentence parser to decide what to do with these.
    Note that the final period, if any, is retained.  The parser may
    need it.
*/
chars_to_words(Chars, Words) :-
    chars_to_words(Words, Chars, []).


chars_to_words([Word|Words]) -->
    chars_to_word(Word), !,
    chars_to_words(Words).
chars_to_words([]) --> [].


chars_to_word(Word) -->
    [Char], {is_layout(Char)}, !,
    chars_to_word(Word).
chars_to_word(atom(Word)) -->
    [Char], {is_letter(Char)}, !,
    chars_to_atom(Chars),
    {case_shift([Char|Chars], Name)},
    {name(Word, Name)}.
chars_to_word(integer(Word)) -->
    [Char], {is_digit(Char)}, !,
    {Init is Char-"0"},		% integer value of char
    chars_to_integer(Init, Word).
chars_to_word(aposts) -->
    "'s", !.
chars_to_word(apost) -->
    "'", !.
chars_to_word(string(Word)) -->
    [Quote], {Quote is 34 /* " */}, !,
    chars_to_string(Quote, String),
    {name(Word, String)}.
chars_to_word(Punct) -->
    [Char],
    {name(Punct, [Char])}.


/*  chars_to_atom(Tail)
    reads the remaining characters of a word.  Case conversion  is  left
    to  another  routine.   In this application, a word may only contain
    letters but they may be in either case.  If you want to parse French
    you will have to decide what to do about accents.  I suggest putting
    them after the vowel, and adding a clause
	chars_to_atom([Vowel,Accent|Chars]) -->
		[Vowel],	{accentable_vowel(Vowel)},
		[Accent],	{accent_for(Vowel, Accent)},
		!.
	with the obvious definitions of accentable_vowel and accent_for.
    Note that the ASCII characters ' ` ^ are officially  designated  the
    "accent acute", "accent grave", and "circumflex".  But this file was
    originally written for an English parser and there was no problem.
    Note added in 2000: now that this handles Latin-1 letters, there's
    no problem for French either.
*/
chars_to_atom([Char|Chars]) -->
    [Char], {is_letter(Char)}, !,
    chars_to_atom(Chars).
chars_to_atom([]) --> [].


/*  case_shift(Mixed, Lower)
    converts all the upper case letters in Mixed to lower  case.   Other
    characters (not necessarily letters!) are left alone.  If you decide
    to accept other characters in words only chars_to_atom has to alter.
    Note added in 2000: this works for Latin-1 too, but if I'd converted
    to upper case, it wouldn't have!
*/
case_shift([Given|Mixed], [Letter|Lower]) :-
    (   is_upper(Given) ->
	Letter is Given-"A"+"a"
    ;/* is_lower(Given) */
	Letter is Given
    ),
    case_shift(Mixed, Lower).
case_shift([], []).


/*  chars_to_integer(Init, Final)
    reads the remaining characters of an integer which starts  as  Init.
    NB:  this  parser  does  not  know about negative numbers or radices
    other than 10, as it was written for PDP-11 Prolog.
*/
chars_to_integer(Init, Final) -->
    [Char], {is_digit(Char)}, !,
    {Next is Init*10-"0"+Char},
    chars_to_integer(Next, Final).
chars_to_integer(Final, Final) --> [].


/*  chars_to_string(Quote, String)
    reads the rest of a string which was opened by  a  Quote  character.
    The  string is expected to end with a Quote as well.  If there isn't
    a matching Quote, the attempt to parse the  string  will  FAIL,  and
    thus the whole parse will FAIL.  I would prefer to give some sort of
    error  message and try to recover but that is application dependent.
    Two adjacent Quotes are taken as one, as they are in Prolog itself.
*/
chars_to_string(Quote, [Quote|String]) -->
    [Quote,Quote], !,
    chars_to_string(Quote, String).
chars_to_string(Quote, []) -->
    [Quote], !.
chars_to_string(Quote, [Char|String]) -->
    [Char],
    chars_to_string(Quote, String).


/*  read_line(Chars)
    reads characters up to the next newline or the end of the file, and
    returns them in a list, including the newline or end of file.  When
    you want multiple spaces crushed out, and the newline dropped, that
    is most of the time, call trim_blanks on the result.
*/
read_line(Chars) :-
    is_newline(NL),
    read_until([NL], Chars).


/*  read_sent(Words)
    reads characters up to the next period, which may be  several  lines
    distant from the start, skips to the end of that line, and turns the
    result  into  a  list of tokens.  It can happen that the sentence is
    not well formed, if say there is an unmatched double quote.  In that
    case all the characters will still be read, but chars_to_words  will
    fail  and  so  read_sent  will fail.  read_sent will NOT try to read
    another sentence.
*/
read_sent(Words) :-
    read_until("!?.", Chars),
    is_newline(NL),
    read_until([NL], _),		% skip to end of line
    chars_to_words(Chars, Words).

end_of_file.

