rdelaage
/
postgresql
mirror of https://git.postgresql.org/git/postgresql.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

postgres_fdw: Fix issues with generated columns in foreign tables. 

postgres_fdw imported generated columns from the remote tables as plain
columns, and caused failures like "ERROR: cannot insert a non-DEFAULT
value into column "foo"" when inserting into the foreign tables, as it
tried to insert values into the generated columns.  To fix, we do the
following under the assumption that generated columns in a postgres_fdw
foreign table are defined so that they represent generated columns in
the underlying remote table:

* Send DEFAULT for the generated columns to the foreign server on insert
  or update, not generated column values computed on the local server.
* Add to postgresImportForeignSchema() an option "import_generated" to
  include column generated expressions in the definitions of foreign
  tables imported from a foreign server.  The option is true by default.

The assumption seems reasonable, because that would make a query of the
postgres_fdw foreign table return values for the generated columns that
are consistent with the generated expression.

While here, fix another issue in postgresImportForeignSchema(): it tried
to include column generated expressions as column default expressions in
the foreign table definitions when the import_default option was enabled.

Per bug #16631 from Daniel Cherniy.  Back-patch to v12 where generated
columns were added.

Discussion: https://postgr.es/m/16631-e929fe9db0ffc7cf%40postgresql.org
This commit is contained in:
Etsuro Fujita 2021-08-05 20:00:00 +09:00
parent 93d573d865
commit aa769f80ed
6 changed files with 238 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
contrib/postgres_fdw/deparse.c
View File

@ -1844,6 +1844,7 @@ deparseInsertSql(StringInfo buf, RangeTblEntry *rte,
List *withCheckOptionList, List *returningList,
List **retrieved_attrs, int *values_end_len)
{
TupleDesc tupdesc = RelationGetDescr(rel);
AttrNumber pindex;
bool first;
ListCell *lc;
@ -1873,12 +1874,20 @@ deparseInsertSql(StringInfo buf, RangeTblEntry *rte,
first = true;
foreach(lc, targetAttrs)
{
int attnum = lfirst_int(lc);
Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1);
if (!first)
appendStringInfoString(buf, ", ");
first = false;
appendStringInfo(buf, "$%d", pindex);
pindex++;
if (attr->attgenerated)
appendStringInfoString(buf, "DEFAULT");
else
{
appendStringInfo(buf, "$%d", pindex);
pindex++;
}
}
appendStringInfoChar(buf, ')');
@ -1902,14 +1911,16 @@ deparseInsertSql(StringInfo buf, RangeTblEntry *rte,
* right number of parameters.
*/
void
rebuildInsertSql(StringInfo buf, char *orig_query,
int values_end_len, int num_cols,
rebuildInsertSql(StringInfo buf, Relation rel,
char *orig_query, List *target_attrs,
int values_end_len, int num_params,
int num_rows)
{
int i,
j;
TupleDesc tupdesc = RelationGetDescr(rel);
int i;
int pindex;
bool first;
ListCell *lc;
/* Make sure the values_end_len is sensible */
Assert((values_end_len > 0) && (values_end_len <= strlen(orig_query)));
@ -1921,20 +1932,28 @@ rebuildInsertSql(StringInfo buf, char *orig_query,
* Add records to VALUES clause (we already have parameters for the first
* row, so start at the right offset).
*/
pindex = num_cols + 1;
pindex = num_params + 1;
for (i = 0; i < num_rows; i++)
{
appendStringInfoString(buf, ", (");
first = true;
for (j = 0; j < num_cols; j++)
foreach(lc, target_attrs)
{
int attnum = lfirst_int(lc);
Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1);
if (!first)
appendStringInfoString(buf, ", ");
first = false;
appendStringInfo(buf, "$%d", pindex);
pindex++;
if (attr->attgenerated)
appendStringInfoString(buf, "DEFAULT");
else
{
appendStringInfo(buf, "$%d", pindex);
pindex++;
}
}
appendStringInfoChar(buf, ')');
@ -1958,6 +1977,7 @@ deparseUpdateSql(StringInfo buf, RangeTblEntry *rte,
List *withCheckOptionList, List *returningList,
List **retrieved_attrs)
{
TupleDesc tupdesc = RelationGetDescr(rel);
AttrNumber pindex;
bool first;
ListCell *lc;
@ -1971,14 +1991,20 @@ deparseUpdateSql(StringInfo buf, RangeTblEntry *rte,
foreach(lc, targetAttrs)
{
int attnum = lfirst_int(lc);
Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1);
if (!first)
appendStringInfoString(buf, ", ");
first = false;
deparseColumnRef(buf, rtindex, attnum, rte, false);
appendStringInfo(buf, " = $%d", pindex);
pindex++;
if (attr->attgenerated)
appendStringInfoString(buf, " = DEFAULT");
else
{
appendStringInfo(buf, " = $%d", pindex);
pindex++;
}
}
appendStringInfoString(buf, " WHERE ctid = $1");

113
contrib/postgres_fdw/expected/postgres_fdw.out
View File

@ -6548,13 +6548,37 @@ select * from rem1;
-- ===================================================================
-- test generated columns
-- ===================================================================
create table gloc1 (a int, b int);
create table gloc1 (
a int,
b int generated always as (a * 2) stored);
alter table gloc1 set (autovacuum_enabled = 'false');
create foreign table grem1 (
a int,
b int generated always as (a * 2) stored)
server loopback options(table_name 'gloc1');
explain (verbose, costs off)
insert into grem1 (a) values (1), (2);
QUERY PLAN
-------------------------------------------------------------------
Insert on public.grem1
Remote SQL: INSERT INTO public.gloc1(a, b) VALUES ($1, DEFAULT)
Batch Size: 1
-> Values Scan on "*VALUES*"
Output: "*VALUES*".column1, NULL::integer
(5 rows)
insert into grem1 (a) values (1), (2);
explain (verbose, costs off)
update grem1 set a = 22 where a = 2;
QUERY PLAN
------------------------------------------------------------------------------------
Update on public.grem1
Remote SQL: UPDATE public.gloc1 SET a = $2, b = DEFAULT WHERE ctid = $1
-> Foreign Scan on public.grem1
Output: 22, ctid, grem1.*
Remote SQL: SELECT a, b, ctid FROM public.gloc1 WHERE ((a = 2)) FOR UPDATE
(5 rows)
update grem1 set a = 22 where a = 2;
select * from gloc1;
a | b
@ -6570,6 +6594,54 @@ select * from grem1;
22 | 44
(2 rows)
delete from grem1;
-- test copy from
copy grem1 from stdin;
select * from gloc1;
a | b
---+---
1 | 2
2 | 4
(2 rows)
select * from grem1;
a | b
---+---
1 | 2
2 | 4
(2 rows)
delete from grem1;
-- test batch insert
alter server loopback options (add batch_size '10');
explain (verbose, costs off)
insert into grem1 (a) values (1), (2);
QUERY PLAN
-------------------------------------------------------------------
Insert on public.grem1
Remote SQL: INSERT INTO public.gloc1(a, b) VALUES ($1, DEFAULT)
Batch Size: 10
-> Values Scan on "*VALUES*"
Output: "*VALUES*".column1, NULL::integer
(5 rows)
insert into grem1 (a) values (1), (2);
select * from gloc1;
a | b
---+---
1 | 2
2 | 4
(2 rows)
select * from grem1;
a | b
---+---
1 | 2
2 | 4
(2 rows)
delete from grem1;
alter server loopback options (drop batch_size);
-- ===================================================================
-- test local triggers
-- ===================================================================
@ -8656,6 +8728,7 @@ CREATE TABLE import_source.t3 (c1 timestamptz default now(), c2 typ1);
CREATE TABLE import_source."x 4" (c1 float8, "C 2" text, c3 varchar(42));
CREATE TABLE import_source."x 5" (c1 float8);
ALTER TABLE import_source."x 5" DROP COLUMN c1;
CREATE TABLE import_source."x 6" (c1 int, c2 int generated always as (c1 * 2) stored);
CREATE TABLE import_source.t4 (c1 int) PARTITION BY RANGE (c1);
CREATE TABLE import_source.t4_part PARTITION OF import_source.t4
FOR VALUES FROM (1) TO (100);
@ -8673,7 +8746,8 @@ IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest1;
import_dest1 | t4 | loopback | (schema_name 'import_source', table_name 't4') |
import_dest1 | x 4 | loopback | (schema_name 'import_source', table_name 'x 4') |
import_dest1 | x 5 | loopback | (schema_name 'import_source', table_name 'x 5') |
(6 rows)
import_dest1 | x 6 | loopback | (schema_name 'import_source', table_name 'x 6') |
(7 rows)
\d import_dest1.*
Foreign table "import_dest1.t1"
@ -8723,6 +8797,14 @@ FDW options: (schema_name 'import_source', table_name 'x 4')
Server: loopback
FDW options: (schema_name 'import_source', table_name 'x 5')
Foreign table "import_dest1.x 6"
Column | Type | Collation | Nullable | Default | FDW options
--------+---------+-----------+----------+-------------------------------------+--------------------
c1 | integer | | | | (column_name 'c1')
c2 | integer | | | generated always as (c1 * 2) stored | (column_name 'c2')
Server: loopback
FDW options: (schema_name 'import_source', table_name 'x 6')
-- Options
CREATE SCHEMA import_dest2;
IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest2
@ -8737,7 +8819,8 @@ IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest2
import_dest2 | t4 | loopback | (schema_name 'import_source', table_name 't4') |
import_dest2 | x 4 | loopback | (schema_name 'import_source', table_name 'x 4') |
import_dest2 | x 5 | loopback | (schema_name 'import_source', table_name 'x 5') |
(6 rows)
import_dest2 | x 6 | loopback | (schema_name 'import_source', table_name 'x 6') |
(7 rows)
\d import_dest2.*
Foreign table "import_dest2.t1"
@ -8787,9 +8870,17 @@ FDW options: (schema_name 'import_source', table_name 'x 4')
Server: loopback
FDW options: (schema_name 'import_source', table_name 'x 5')
Foreign table "import_dest2.x 6"
Column | Type | Collation | Nullable | Default | FDW options
--------+---------+-----------+----------+-------------------------------------+--------------------
c1 | integer | | | | (column_name 'c1')
c2 | integer | | | generated always as (c1 * 2) stored | (column_name 'c2')
Server: loopback
FDW options: (schema_name 'import_source', table_name 'x 6')
CREATE SCHEMA import_dest3;
IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest3
OPTIONS (import_collate 'false', import_not_null 'false');
OPTIONS (import_collate 'false', import_generated 'false', import_not_null 'false');
\det+ import_dest3.*
List of foreign tables
Schema | Table | Server | FDW options | Description
@ -8800,7 +8891,8 @@ IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest3
import_dest3 | t4 | loopback | (schema_name 'import_source', table_name 't4') |
import_dest3 | x 4 | loopback | (schema_name 'import_source', table_name 'x 4') |
import_dest3 | x 5 | loopback | (schema_name 'import_source', table_name 'x 5') |
(6 rows)
import_dest3 | x 6 | loopback | (schema_name 'import_source', table_name 'x 6') |
(7 rows)
\d import_dest3.*
Foreign table "import_dest3.t1"
@ -8850,6 +8942,14 @@ FDW options: (schema_name 'import_source', table_name 'x 4')
Server: loopback
FDW options: (schema_name 'import_source', table_name 'x 5')
Foreign table "import_dest3.x 6"
Column | Type | Collation | Nullable | Default | FDW options
--------+---------+-----------+----------+---------+--------------------
c1 | integer | | | | (column_name 'c1')
c2 | integer | | | | (column_name 'c2')
Server: loopback
FDW options: (schema_name 'import_source', table_name 'x 6')
-- Check LIMIT TO and EXCEPT
CREATE SCHEMA import_dest4;
IMPORT FOREIGN SCHEMA import_source LIMIT TO (t1, nonesuch, t4_part)
@ -8874,7 +8974,8 @@ IMPORT FOREIGN SCHEMA import_source EXCEPT (t1, "x 4", nonesuch, t4_part)
import_dest4 | t4 | loopback | (schema_name 'import_source', table_name 't4') |
import_dest4 | t4_part | loopback | (schema_name 'import_source', table_name 't4_part') |
import_dest4 | x 5 | loopback | (schema_name 'import_source', table_name 'x 5') |
(6 rows)
import_dest4 | x 6 | loopback | (schema_name 'import_source', table_name 'x 6') |
(7 rows)
-- Assorted error cases
IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest4;

65
contrib/postgres_fdw/postgres_fdw.c
View File

@ -4012,6 +4012,9 @@ create_foreign_modify(EState *estate,
Assert(!attr->attisdropped);
/* Ignore generated columns; they are set to DEFAULT */
if (attr->attgenerated)
continue;
getTypeOutputInfo(attr->atttypid, &typefnoid, &isvarlena);
fmgr_info(typefnoid, &fmstate->p_flinfo[fmstate->p_nums]);
fmstate->p_nums++;
@ -4075,8 +4078,10 @@ execute_foreign_modify(EState *estate,
/* Build INSERT string with numSlots records in its VALUES clause. */
initStringInfo(&sql);
rebuildInsertSql(&sql, fmstate->orig_query, fmstate->values_end,
fmstate->p_nums, *numSlots - 1);
rebuildInsertSql(&sql, fmstate->rel,
fmstate->orig_query, fmstate->target_attrs,
fmstate->values_end, fmstate->p_nums,
*numSlots - 1);
pfree(fmstate->query);
fmstate->query = sql.data;
fmstate->num_slots = *numSlots;
@ -4244,6 +4249,7 @@ convert_prep_stmt_params(PgFdwModifyState *fmstate,
/* get following parameters from slots */
if (slots != NULL && fmstate->target_attrs != NIL)
{
TupleDesc tupdesc = RelationGetDescr(fmstate->rel);
int nestlevel;
ListCell *lc;
@ -4255,9 +4261,13 @@ convert_prep_stmt_params(PgFdwModifyState *fmstate,
foreach(lc, fmstate->target_attrs)
{
int attnum = lfirst_int(lc);
Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1);
Datum value;
bool isnull;
/* Ignore generated columns; they are set to DEFAULT */
if (attr->attgenerated)
continue;
value = slot_getattr(slots[i], attnum, &isnull);
if (isnull)
p_values[pindex] = NULL;
@ -5188,6 +5198,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
List *commands = NIL;
bool import_collate = true;
bool import_default = false;
bool import_generated = true;
bool import_not_null = true;
ForeignServer *server;
UserMapping *mapping;
@ -5207,6 +5218,8 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
import_collate = defGetBoolean(def);
else if (strcmp(def->defname, "import_default") == 0)
import_default = defGetBoolean(def);
else if (strcmp(def->defname, "import_generated") == 0)
import_generated = defGetBoolean(def);
else if (strcmp(def->defname, "import_not_null") == 0)
import_not_null = defGetBoolean(def);
else
@ -5270,13 +5283,24 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
* include a schema name for types/functions in other schemas, which
* is what we want.
*/
appendStringInfoString(&buf,
"SELECT relname, "
" attname, "
" format_type(atttypid, atttypmod), "
" attnotnull, ");
/* Generated columns are supported since Postgres 12 */
if (PQserverVersion(conn) >= 120000)
appendStringInfoString(&buf,
" attgenerated, "
" pg_get_expr(adbin, adrelid), ");
else
appendStringInfoString(&buf,
" NULL, "
" pg_get_expr(adbin, adrelid), ");
if (import_collate)
appendStringInfoString(&buf,
"SELECT relname, "
" attname, "
" format_type(atttypid, atttypmod), "
" attnotnull, "
" pg_get_expr(adbin, adrelid), "
" collname, "
" collnsp.nspname "
"FROM pg_class c "
@ -5293,11 +5317,6 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
" collnsp.oid = collnamespace ");
else
appendStringInfoString(&buf,
"SELECT relname, "
" attname, "
" format_type(atttypid, atttypmod), "
" attnotnull, "
" pg_get_expr(adbin, adrelid), "
" NULL, NULL "
"FROM pg_class c "
" JOIN pg_namespace n ON "
@ -5374,6 +5393,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
char *attname;
char *typename;
char *attnotnull;
char *attgenerated;
char *attdefault;
char *collname;
char *collnamespace;
@ -5385,12 +5405,14 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
attname = PQgetvalue(res, i, 1);
typename = PQgetvalue(res, i, 2);
attnotnull = PQgetvalue(res, i, 3);
attdefault = PQgetisnull(res, i, 4) ? (char *) NULL :
attgenerated = PQgetisnull(res, i, 4) ? (char *) NULL :
PQgetvalue(res, i, 4);
collname = PQgetisnull(res, i, 5) ? (char *) NULL :
attdefault = PQgetisnull(res, i, 5) ? (char *) NULL :
PQgetvalue(res, i, 5);
collnamespace = PQgetisnull(res, i, 6) ? (char *) NULL :
collname = PQgetisnull(res, i, 6) ? (char *) NULL :
PQgetvalue(res, i, 6);
collnamespace = PQgetisnull(res, i, 7) ? (char *) NULL :
PQgetvalue(res, i, 7);
if (first_item)
first_item = false;
@ -5418,9 +5440,20 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
quote_identifier(collname));
/* Add DEFAULT if needed */
if (import_default && attdefault != NULL)
if (import_default && attdefault != NULL &&
(!attgenerated || !attgenerated[0]))
appendStringInfo(&buf, " DEFAULT %s", attdefault);
/* Add GENERATED if needed */
if (import_generated && attgenerated != NULL &&
attgenerated[0] == ATTRIBUTE_GENERATED_STORED)
{
Assert(attdefault != NULL);
appendStringInfo(&buf,
" GENERATED ALWAYS AS (%s) STORED",
attdefault);
}
/* Add NOT NULL if needed */
if (import_not_null && attnotnull[0] == 't')
appendStringInfoString(&buf, " NOT NULL");

5
contrib/postgres_fdw/postgres_fdw.h
View File

@ -176,8 +176,9 @@ extern void deparseInsertSql(StringInfo buf, RangeTblEntry *rte,
List *targetAttrs, bool doNothing,
List *withCheckOptionList, List *returningList,
List **retrieved_attrs, int *values_end_len);
extern void rebuildInsertSql(StringInfo buf, char *orig_query,
int values_end_len, int num_cols,
extern void rebuildInsertSql(StringInfo buf, Relation rel,
char *orig_query, List *target_attrs,
int values_end_len, int num_params,
int num_rows);
extern void deparseUpdateSql(StringInfo buf, RangeTblEntry *rte,
Index rtindex, Relation rel,

31
contrib/postgres_fdw/sql/postgres_fdw.sql
View File

@ -1511,16 +1511,42 @@ select * from rem1;
-- ===================================================================
-- test generated columns
-- ===================================================================
create table gloc1 (a int, b int);
create table gloc1 (
a int,
b int generated always as (a * 2) stored);
alter table gloc1 set (autovacuum_enabled = 'false');
create foreign table grem1 (
a int,
b int generated always as (a * 2) stored)
server loopback options(table_name 'gloc1');
explain (verbose, costs off)
insert into grem1 (a) values (1), (2);
insert into grem1 (a) values (1), (2);
explain (verbose, costs off)
update grem1 set a = 22 where a = 2;
update grem1 set a = 22 where a = 2;
select * from gloc1;
select * from grem1;
delete from grem1;
-- test copy from
copy grem1 from stdin;
1
2
\.
select * from gloc1;
select * from grem1;
delete from grem1;
-- test batch insert
alter server loopback options (add batch_size '10');
explain (verbose, costs off)
insert into grem1 (a) values (1), (2);
insert into grem1 (a) values (1), (2);
select * from gloc1;
select * from grem1;
delete from grem1;
alter server loopback options (drop batch_size);
-- ===================================================================
-- test local triggers
@ -2534,6 +2560,7 @@ CREATE TABLE import_source.t3 (c1 timestamptz default now(), c2 typ1);
CREATE TABLE import_source."x 4" (c1 float8, "C 2" text, c3 varchar(42));
CREATE TABLE import_source."x 5" (c1 float8);
ALTER TABLE import_source."x 5" DROP COLUMN c1;
CREATE TABLE import_source."x 6" (c1 int, c2 int generated always as (c1 * 2) stored);
CREATE TABLE import_source.t4 (c1 int) PARTITION BY RANGE (c1);
CREATE TABLE import_source.t4_part PARTITION OF import_source.t4
FOR VALUES FROM (1) TO (100);
@ -2553,7 +2580,7 @@ IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest2
\d import_dest2.*
CREATE SCHEMA import_dest3;
IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest3
OPTIONS (import_collate 'false', import_not_null 'false');
OPTIONS (import_collate 'false', import_generated 'false', import_not_null 'false');
\det+ import_dest3.*
\d import_dest3.*

12
doc/src/sgml/postgres-fdw.sgml
View File

@ -555,6 +555,18 @@ OPTIONS (ADD password_required 'false');
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><literal>import_generated</literal> (<type>boolean</type>)</term>
<listitem>
<para>
This option controls whether column <literal>GENERATED</literal> expressions
are included in the definitions of foreign tables imported
from a foreign server. The default is <literal>true</literal>.
The <command>IMPORT</command> will fail altogether if an imported generated
expression uses a function or operator that does not exist locally.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><literal>import_not_null</literal> (<type>boolean</type>)</term>
<listitem>