Change case-folding of keywords to conform to SQL99 and fix misbehavior

in Turkish locale.  Keywords are now checked under pure ASCII case-folding
rules ('A'-'Z'->'a'-'z' and nothing else).  However, once a word is
determined not to be a keyword, it will be case-folded under the current
locale, same as before.  See pghackers discussion 20-Feb-01.

src/backend/parser/keywords.c

@@ -1,23 +1,22 @@
 /*-------------------------------------------------------------------------
  *
  * keywords.c
- *	  lexical token lookup for reserved words in postgres SQL
+ *	  lexical token lookup for reserved words in PostgreSQL
  *
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/parser/keywords.c,v 1.88 2001/01/24 19:43:01 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/parser/keywords.c,v 1.89 2001/02/21 18:53:46 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
-#include <ctype.h>
-
 #include "postgres.h"
 
+#include <ctype.h>
+
 #include "nodes/parsenodes.h"
-#include "nodes/pg_list.h"
 #include "parser/keywords.h"
 #include "parser/parse.h"
 
@@ -286,18 +285,62 @@ static ScanKeyword ScanKeywords[] = {
 	{"zone", ZONE},
 };
 
+/*
+ * ScanKeywordLookup - see if a given word is a keyword
+ *
+ * Returns a pointer to the ScanKeyword table entry, or NULL if no match.
+ *
+ * The match is done case-insensitively.  Note that we deliberately use a
+ * dumbed-down case conversion that will only translate 'A'-'Z' into 'a'-'z',
+ * even if we are in a locale where tolower() would produce more or different
+ * translations.  This is to conform to the SQL99 spec, which says that
+ * keywords are to be matched in this way even though non-keyword identifiers
+ * receive a different case-normalization mapping.
+ */
 ScanKeyword *
 ScanKeywordLookup(char *text)
 {
-	ScanKeyword *low = &ScanKeywords[0];
+	int			len,
-	ScanKeyword *high = endof(ScanKeywords) - 1;
+				i;
-	ScanKeyword *middle;
+	char		word[NAMEDATALEN];
-	int			difference;
+	ScanKeyword *low;
+	ScanKeyword *high;
 
+	len = strlen(text);
+	/* We assume all keywords are shorter than NAMEDATALEN. */
+	if (len >= NAMEDATALEN)
+		return NULL;
+
+	/*
+	 * Apply an ASCII-only downcasing.  We must not use tolower() since
+	 * it may produce the wrong translation in some locales (eg, Turkish),
+	 * and we don't trust isupper() very much either.  In an ASCII-based
+	 * encoding the tests against A and Z are sufficient, but we also check
+	 * isupper() so that we will work correctly under EBCDIC.  The actual
+	 * case conversion step should work for either ASCII or EBCDIC.
+	 */
+	for (i = 0; i < len; i++)
+	{
+		char	ch = text[i];
+
+		if (ch >= 'A' && ch <= 'Z' && isupper((unsigned char) ch))
+			ch += 'a' - 'A';
+		word[i] = ch;
+	}
+	word[len] = '\0';
+
+	/*
+	 * Now do a binary search using plain strcmp() comparison.
+	 */
+	low = &ScanKeywords[0];
+	high = endof(ScanKeywords) - 1;
 	while (low <= high)
 	{
+		ScanKeyword *middle;
+		int			difference;
+
 		middle = low + (high - low) / 2;
-		difference = strcmp(middle->name, text);
+		difference = strcmp(middle->name, word);
 		if (difference == 0)
 			return middle;
 		else if (difference < 0)

src/backend/parser/scan.l

@@ -2,14 +2,14 @@
 /*-------------------------------------------------------------------------
  *
  * scan.l
- *	  lexical scanner for POSTGRES
+ *	  lexical scanner for PostgreSQL
  *
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/parser/scan.l,v 1.86 2001/02/03 20:13:05 petere Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/parser/scan.l,v 1.87 2001/02/21 18:53:47 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -477,12 +477,27 @@ other			.
 
 
 {identifier}	{
-					int i;
+					ScanKeyword	   *keyword;
-					ScanKeyword		*keyword;
+					int				i;
 
-					for(i = 0; yytext[i]; i++)
+					/* Is it a keyword? */
+					keyword = ScanKeywordLookup((char*) yytext);
+					if (keyword != NULL)
+						return keyword->value;
+
+					/*
+					 * No.  Convert the identifier to lower case, and truncate
+					 * if necessary.
+					 *
+					 * Note: here we use a locale-dependent case conversion,
+					 * which seems appropriate under SQL99 rules, whereas
+					 * the keyword comparison was NOT locale-dependent.
+					 */
+					for (i = 0; yytext[i]; i++)
+					{
 						if (isupper((unsigned char) yytext[i]))
 							yytext[i] = tolower((unsigned char) yytext[i]);
+					}
 					if (i >= NAMEDATALEN)
                     {
 #ifdef MULTIBYTE
@@ -497,15 +512,8 @@ other			.
 						yytext[NAMEDATALEN-1] = '\0';
 #endif
                     }
-					keyword = ScanKeywordLookup((char*)yytext);
+					yylval.str = pstrdup((char*) yytext);
-					if (keyword != NULL) {
+					return IDENT;
-						return keyword->value;
-					}
-					else
-					{
-						yylval.str = pstrdup((char*)yytext);
-						return IDENT;
-					}
 				}
 
 {other}			{ return yytext[0]; }

src/backend/utils/adt/ruleutils.c

@@ -3,7 +3,7 @@
  *				back to source text
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/adt/ruleutils.c,v 1.72 2001/02/14 21:35:05 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/adt/ruleutils.c,v 1.73 2001/02/21 18:53:47 tgl Exp $
  *
  *	  This software is copyrighted by Jan Wieck - Hamburg.
  *
@@ -2563,8 +2563,8 @@ quote_identifier(char *ident)
 		 * but the parser doesn't provide any easy way to test for whether
 		 * an identifier is safe or not... so be safe not sorry.
 		 *
-		 * Note: ScanKeywordLookup() expects an all-lower-case input, but
+		 * Note: ScanKeywordLookup() does case-insensitive comparison,
-		 * we've already checked we have that.
+		 * but that's fine, since we already know we have all-lower-case.
 		 */
 		if (ScanKeywordLookup(ident) != NULL)
 			safe = false;

src/interfaces/ecpg/preproc/ecpg_keywords.c

@@ -1,8 +1,11 @@
 /*-------------------------------------------------------------------------
  *
- * keywords.c
+ * ecpg_keywords.c
  *	  lexical token lookup for reserved words in postgres embedded SQL
  *
+ * IDENTIFICATION
+ *	  $Header: /cvsroot/pgsql/src/interfaces/ecpg/preproc/ecpg_keywords.c,v 1.22 2001/02/21 18:53:47 tgl Exp $
+ *
  *-------------------------------------------------------------------------
  */
 #include "postgres_fe.h"
@@ -12,6 +15,7 @@
 #include "extern.h"
 #include "preproc.h"
 
+
 /*
  * List of (keyword-name, keyword-token-value) pairs.
  *
@@ -73,18 +77,62 @@ static ScanKeyword ScanKeywords[] = {
 	{"whenever", SQL_WHENEVER},
 };
 
+/*
+ * ScanECPGKeywordLookup - see if a given word is a keyword
+ *
+ * Returns a pointer to the ScanKeyword table entry, or NULL if no match.
+ *
+ * The match is done case-insensitively.  Note that we deliberately use a
+ * dumbed-down case conversion that will only translate 'A'-'Z' into 'a'-'z',
+ * even if we are in a locale where tolower() would produce more or different
+ * translations.  This is to conform to the SQL99 spec, which says that
+ * keywords are to be matched in this way even though non-keyword identifiers
+ * receive a different case-normalization mapping.
+ */
 ScanKeyword *
 ScanECPGKeywordLookup(char *text)
 {
-	ScanKeyword *low = &ScanKeywords[0];
+	int			len,
-	ScanKeyword *high = endof(ScanKeywords) - 1;
+				i;
-	ScanKeyword *middle;
+	char		word[NAMEDATALEN];
-	int			difference;
+	ScanKeyword *low;
+	ScanKeyword *high;
 
+	len = strlen(text);
+	/* We assume all keywords are shorter than NAMEDATALEN. */
+	if (len >= NAMEDATALEN)
+		return NULL;
+
+	/*
+	 * Apply an ASCII-only downcasing.  We must not use tolower() since
+	 * it may produce the wrong translation in some locales (eg, Turkish),
+	 * and we don't trust isupper() very much either.  In an ASCII-based
+	 * encoding the tests against A and Z are sufficient, but we also check
+	 * isupper() so that we will work correctly under EBCDIC.  The actual
+	 * case conversion step should work for either ASCII or EBCDIC.
+	 */
+	for (i = 0; i < len; i++)
+	{
+		char	ch = text[i];
+
+		if (ch >= 'A' && ch <= 'Z' && isupper((unsigned char) ch))
+			ch += 'a' - 'A';
+		word[i] = ch;
+	}
+	word[len] = '\0';
+
+	/*
+	 * Now do a binary search using plain strcmp() comparison.
+	 */
+	low = &ScanKeywords[0];
+	high = endof(ScanKeywords) - 1;
 	while (low <= high)
 	{
+		ScanKeyword *middle;
+		int			difference;
+
 		middle = low + (high - low) / 2;
-		difference = strcmp(middle->name, text);
+		difference = strcmp(middle->name, word);
 		if (difference == 0)
 			return middle;
 		else if (difference < 0)

src/interfaces/ecpg/preproc/keywords.c

@@ -1,14 +1,14 @@
 /*-------------------------------------------------------------------------
  *
  * keywords.c
- *	  lexical token lookup for reserved words in postgres SQL
+ *	  lexical token lookup for reserved words in PostgreSQL
  *
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/interfaces/ecpg/preproc/keywords.c,v 1.37 2001/02/10 02:31:29 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/interfaces/ecpg/preproc/keywords.c,v 1.38 2001/02/21 18:53:47 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -19,6 +19,7 @@
 #include "extern.h"
 #include "preproc.h"
 
+
 /*
  * List of (keyword-name, keyword-token-value) pairs.
  *
@@ -36,7 +37,7 @@ static ScanKeyword ScanKeywords[] = {
 	{"aggregate", AGGREGATE},
 	{"all", ALL},
 	{"alter", ALTER},
-	{"analyse", ANALYSE},
+	{"analyse", ANALYSE}, /* British spelling */
 	{"analyze", ANALYZE},
 	{"and", AND},
 	{"any", ANY},
@@ -58,7 +59,7 @@ static ScanKeyword ScanKeywords[] = {
 	{"chain", CHAIN},
 	{"char", CHAR},
 	{"character", CHARACTER},
-	{"characteristics", CHARACTERISTICS},     
+	{"characteristics", CHARACTERISTICS},
 	{"check", CHECK},
 	{"checkpoint", CHECKPOINT},
 	{"close", CLOSE},
@@ -133,7 +134,7 @@ static ScanKeyword ScanKeywords[] = {
 	{"inherits", INHERITS},
 	{"initially", INITIALLY},
 	{"inner", INNER_P},
-	{"inout", INOUT},    
+	{"inout", INOUT},
 	{"insensitive", INSENSITIVE},
 	{"insert", INSERT},
 	{"instead", INSTEAD},
@@ -182,7 +183,7 @@ static ScanKeyword ScanKeywords[] = {
 	{"nullif", NULLIF},
 	{"numeric", NUMERIC},
 	{"of", OF},
-	{"off", OFF}, 
+	{"off", OFF},
 	{"offset", OFFSET},
 	{"oids", OIDS},
 	{"old", OLD},
@@ -192,13 +193,13 @@ static ScanKeyword ScanKeywords[] = {
 	{"option", OPTION},
 	{"or", OR},
 	{"order", ORDER},
-	{"out", OUT},   
+	{"out", OUT},
 	{"outer", OUTER_P},
 	{"overlaps", OVERLAPS},
 	{"owner", OWNER},
 	{"partial", PARTIAL},
 	{"password", PASSWORD},
-	{"path", PATH_P}, 
+	{"path", PATH_P},
 	{"pendant", PENDANT},
 	{"position", POSITION},
 	{"precision", PRECISION},
@@ -221,14 +222,14 @@ static ScanKeyword ScanKeywords[] = {
 	{"rollback", ROLLBACK},
 	{"row", ROW},
 	{"rule", RULE},
-	{"schema", SCHEMA},  
+	{"schema", SCHEMA},
 	{"scroll", SCROLL},
 	{"second", SECOND_P},
 	{"select", SELECT},
 	{"sequence", SEQUENCE},
 	{"serial", SERIAL},
 	{"serializable", SERIALIZABLE},
-	{"session", SESSION},   
+	{"session", SESSION},
 	{"session_user", SESSION_USER},
 	{"set", SET},
 	{"setof", SETOF},
@@ -251,7 +252,7 @@ static ScanKeyword ScanKeywords[] = {
 	{"timezone_hour", TIMEZONE_HOUR},
 	{"timezone_minute", TIMEZONE_MINUTE},
 	{"to", TO},
-	{"toast", TOAST},    
+	{"toast", TOAST},
 	{"trailing", TRAILING},
 	{"transaction", TRANSACTION},
 	{"trigger", TRIGGER},
@@ -284,18 +285,62 @@ static ScanKeyword ScanKeywords[] = {
 	{"zone", ZONE},
 };
 
+/*
+ * ScanKeywordLookup - see if a given word is a keyword
+ *
+ * Returns a pointer to the ScanKeyword table entry, or NULL if no match.
+ *
+ * The match is done case-insensitively.  Note that we deliberately use a
+ * dumbed-down case conversion that will only translate 'A'-'Z' into 'a'-'z',
+ * even if we are in a locale where tolower() would produce more or different
+ * translations.  This is to conform to the SQL99 spec, which says that
+ * keywords are to be matched in this way even though non-keyword identifiers
+ * receive a different case-normalization mapping.
+ */
 ScanKeyword *
 ScanKeywordLookup(char *text)
 {
-	ScanKeyword *low = &ScanKeywords[0];
+	int			len,
-	ScanKeyword *high = endof(ScanKeywords) - 1;
+				i;
-	ScanKeyword *middle;
+	char		word[NAMEDATALEN];
-	int			difference;
+	ScanKeyword *low;
+	ScanKeyword *high;
 
+	len = strlen(text);
+	/* We assume all keywords are shorter than NAMEDATALEN. */
+	if (len >= NAMEDATALEN)
+		return NULL;
+
+	/*
+	 * Apply an ASCII-only downcasing.  We must not use tolower() since
+	 * it may produce the wrong translation in some locales (eg, Turkish),
+	 * and we don't trust isupper() very much either.  In an ASCII-based
+	 * encoding the tests against A and Z are sufficient, but we also check
+	 * isupper() so that we will work correctly under EBCDIC.  The actual
+	 * case conversion step should work for either ASCII or EBCDIC.
+	 */
+	for (i = 0; i < len; i++)
+	{
+		char	ch = text[i];
+
+		if (ch >= 'A' && ch <= 'Z' && isupper((unsigned char) ch))
+			ch += 'a' - 'A';
+		word[i] = ch;
+	}
+	word[len] = '\0';
+
+	/*
+	 * Now do a binary search using plain strcmp() comparison.
+	 */
+	low = &ScanKeywords[0];
+	high = endof(ScanKeywords) - 1;
 	while (low <= high)
 	{
+		ScanKeyword *middle;
+		int			difference;
+
 		middle = low + (high - low) / 2;
-		difference = strcmp(middle->name, text);
+		difference = strcmp(middle->name, word);
 		if (difference == 0)
 			return middle;
 		else if (difference < 0)

src/interfaces/ecpg/preproc/pgc.l

@@ -12,7 +12,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/interfaces/ecpg/preproc/pgc.l,v 1.76 2001/02/10 02:31:29 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/interfaces/ecpg/preproc/pgc.l,v 1.77 2001/02/21 18:53:47 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -527,74 +527,53 @@ cppline			{space}*#(.*\\{line_end})*.*
 					return(CVARIABLE);
 			}
 <SQL>{identifier}	{
-					int i;
+					ScanKeyword	   *keyword;
-					ScanKeyword		*keyword;
+					struct _defines *ptr;
-					char lower_text[NAMEDATALEN];
 
-					/* this should leave the last byte set to '\0' */
+					/* Is it an SQL keyword? */
-					strncpy(lower_text, yytext, NAMEDATALEN-1);
+					keyword = ScanKeywordLookup((char*) yytext);
-					for(i = 0; lower_text[i]; i++)
+					if (keyword != NULL)
-						if (isupper((unsigned char) lower_text[i]))
-							lower_text[i] = tolower((unsigned char) lower_text[i]);
-
-					if (i >= NAMEDATALEN)
-					{
-#ifdef MULTIBYTE_NOTUSED
-                                                int len;
-
-	                                        len = pg_mbcliplen(lower_text,strlen(lower_text),NAMEDATALEN-1);
-						sprintf(errortext, "identifier \"%s\" will be truncated to \"%.*s\"",
-                                                        lower_text, len, lower_text);
-	                                        lower_text[len] = '\0';
-#else
-						sprintf(errortext, "identifier \"%s\" will be truncated to \"%.*s\"",
-                                                        lower_text, NAMEDATALEN-1, lower_text);
-                                                lower_text[NAMEDATALEN-1] = '\0';
-#endif
- 						mmerror(ET_NOTICE, errortext);
-                                                yytext[NAMEDATALEN-1] = '\0';
-					}
-
-					keyword = ScanKeywordLookup((char*)lower_text);
-					if (keyword != NULL) {
 						return keyword->value;
-					}
+
-					else
+					/* Is it an ECPG keyword? */
+					keyword = ScanECPGKeywordLookup((char*) yytext);
+					if (keyword != NULL)
+						return keyword->value;
+
+					/* How about a DEFINE? */
+					for (ptr = defines; ptr; ptr = ptr->next)
 					{
-						keyword = ScanECPGKeywordLookup((char*)lower_text);
+						if (strcmp(yytext, ptr->old) == 0)
-						if (keyword != NULL) {
-							return keyword->value;
-						}
-						else
 						{
-							struct _defines *ptr;
+							struct _yy_buffer *yb;
 
-							for (ptr = defines; ptr; ptr = ptr->next)
+							yb = mm_alloc(sizeof(struct _yy_buffer));
-							{
-								if (strcmp(yytext, ptr->old) == 0)
-								{
-									struct _yy_buffer *yb;
 
-									yb = mm_alloc(sizeof(struct _yy_buffer));
+							yb->buffer =  YY_CURRENT_BUFFER;
+							yb->lineno = yylineno;
+							yb->filename = mm_strdup(input_filename);
+							yb->next = yy_buffer;
 
-						                        yb->buffer =  YY_CURRENT_BUFFER;
+							yy_buffer = yb;
-						                        yb->lineno = yylineno;
-						                        yb->filename = mm_strdup(input_filename);
-						                        yb->next = yy_buffer;
 
-						                        yy_buffer = yb;
+							yy_scan_string(ptr->new);
-
+							break;
- 									yy_scan_string(ptr->new);
-									break;
-								}
-							}
-							if (ptr == NULL) 
-							{
-								yylval.str = mm_strdup((char*)yytext);
-								return IDENT;
-							}
 						}
 					}
+
+					/*
+					 * None of the above.  Return it as an identifier.
+					 *
+					 * The backend would attempt to truncate and case-fold
+					 * the identifier, but I see no good reason for ecpg
+					 * to do so; that's just another way that ecpg could get
+					 * out of step with the backend.
+					 */
+					if (ptr == NULL) 
+					{
+						yylval.str = mm_strdup((char*) yytext);
+						return IDENT;
+					}
 				}
 <SQL>{other}			{ return yytext[0]; }
 <C>{exec_sql}			{ BEGIN SQL; return SQL_START; }