Align ECPG lexer more closely with the core and psql lexers.

Make a bunch of basically-cosmetic changes to reduce the diffs between
the flex rules in scan.l, psqlscan.l, and pgc.l.  Reorder some code,
adjust a lot of whitespace, sync some comments, make use of flex start
condition scopes to do that.

There are a few non-cosmetic changes in the ECPG lexer:

* Bring over the decimalfail rule (and support function
process_integer_literal) so that ECPG will lex "1..10" into
the same tokens as the backend would.  I'm not sure this makes any
visible difference to users, but I'm not sure it doesn't, either.

* <xdc><<EOF>> gets its own rule so as to produce a more on-point
error message.

* Remove duplicate <SQL>{xdstart} rule.

John Naylor, with a few additional changes by me

Discussion: https://postgr.es/m/CAJVSVGWGqY9YBs2EwtRUkbNv=hXkN8yRPOoD1wxE6COgvvrz5g@mail.gmail.com
This commit is contained in:
Tom Lane 2018-11-13 12:57:52 -05:00
parent d20dceaf50
commit ec937d0805
3 changed files with 623 additions and 471 deletions

View File

@ -6,7 +6,8 @@
* *
* NOTE NOTE NOTE: * NOTE NOTE NOTE:
* *
* The rules in this file must be kept in sync with src/fe_utils/psqlscan.l! * The rules in this file must be kept in sync with src/fe_utils/psqlscan.l
* and src/interfaces/ecpg/preproc/pgc.l!
* *
* The rules are designed so that the scanner never has to backtrack, * The rules are designed so that the scanner never has to backtrack,
* in the sense that there is always a rule that can match the input * in the sense that there is always a rule that can match the input
@ -168,8 +169,8 @@ extern void core_yyset_column(int column_no, yyscan_t yyscanner);
%x xc %x xc
%x xd %x xd
%x xh %x xh
%x xe
%x xq %x xq
%x xe
%x xdolq %x xdolq
%x xui %x xui
%x xuiend %x xuiend
@ -192,7 +193,7 @@ extern void core_yyset_column(int column_no, yyscan_t yyscanner);
* XXX perhaps \f (formfeed) should be treated as a newline as well? * XXX perhaps \f (formfeed) should be treated as a newline as well?
* *
* XXX if you change the set of whitespace characters, fix scanner_isspace() * XXX if you change the set of whitespace characters, fix scanner_isspace()
* to agree, and see also the plpgsql lexer. * to agree.
*/ */
space [ \t\n\r\f] space [ \t\n\r\f]
@ -417,32 +418,36 @@ other .
yyless(2); yyless(2);
} }
<xc>{xcstart} { <xc>{
{xcstart} {
(yyextra->xcdepth)++; (yyextra->xcdepth)++;
/* Put back any characters past slash-star; see above */ /* Put back any characters past slash-star; see above */
yyless(2); yyless(2);
} }
<xc>{xcstop} { {xcstop} {
if (yyextra->xcdepth <= 0) if (yyextra->xcdepth <= 0)
BEGIN(INITIAL); BEGIN(INITIAL);
else else
(yyextra->xcdepth)--; (yyextra->xcdepth)--;
} }
<xc>{xcinside} { {xcinside} {
/* ignore */ /* ignore */
} }
<xc>{op_chars} { {op_chars} {
/* ignore */ /* ignore */
} }
<xc>\*+ { \*+ {
/* ignore */ /* ignore */
} }
<xc><<EOF>> { yyerror("unterminated /* comment"); } <<EOF>> {
yyerror("unterminated /* comment");
}
} /* <xc> */
{xbstart} { {xbstart} {
/* Binary bit type. /* Binary bit type.

View File

@ -23,6 +23,7 @@
* *
* See psqlscan_int.h for additional commentary. * See psqlscan_int.h for additional commentary.
* *
*
* Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California * Portions Copyright (c) 1994, Regents of the University of California
* *
@ -39,6 +40,9 @@
} }
%{ %{
/* LCOV_EXCL_START */
#include "fe_utils/psqlscan_int.h" #include "fe_utils/psqlscan_int.h"
/* /*
@ -71,8 +75,6 @@ typedef int YYSTYPE;
extern int psql_yyget_column(yyscan_t yyscanner); extern int psql_yyget_column(yyscan_t yyscanner);
extern void psql_yyset_column(int column_no, yyscan_t yyscanner); extern void psql_yyset_column(int column_no, yyscan_t yyscanner);
/* LCOV_EXCL_START */
%} %}
%option reentrant %option reentrant
@ -128,8 +130,8 @@ extern void psql_yyset_column(int column_no, yyscan_t yyscanner);
%x xc %x xc
%x xd %x xd
%x xh %x xh
%x xe
%x xq %x xq
%x xe
%x xdolq %x xdolq
%x xui %x xui
%x xuiend %x xuiend
@ -151,7 +153,7 @@ extern void psql_yyset_column(int column_no, yyscan_t yyscanner);
* XXX perhaps \f (formfeed) should be treated as a newline as well? * XXX perhaps \f (formfeed) should be treated as a newline as well?
* *
* XXX if you change the set of whitespace characters, fix scanner_isspace() * XXX if you change the set of whitespace characters, fix scanner_isspace()
* to agree, and see also the plpgsql lexer. * to agree.
*/ */
space [ \t\n\r\f] space [ \t\n\r\f]
@ -402,14 +404,15 @@ other .
ECHO; ECHO;
} }
<xc>{xcstart} { <xc>{
{xcstart} {
cur_state->xcdepth++; cur_state->xcdepth++;
/* Put back any characters past slash-star; see above */ /* Put back any characters past slash-star; see above */
yyless(2); yyless(2);
ECHO; ECHO;
} }
<xc>{xcstop} { {xcstop} {
if (cur_state->xcdepth <= 0) if (cur_state->xcdepth <= 0)
BEGIN(INITIAL); BEGIN(INITIAL);
else else
@ -417,17 +420,18 @@ other .
ECHO; ECHO;
} }
<xc>{xcinside} { {xcinside} {
ECHO; ECHO;
} }
<xc>{op_chars} { {op_chars} {
ECHO; ECHO;
} }
<xc>\*+ { \*+ {
ECHO; ECHO;
} }
} /* <xc> */
{xbstart} { {xbstart} {
BEGIN(xb); BEGIN(xb);

File diff suppressed because it is too large Load Diff