1996-07-09 08:22:35 +02:00
|
|
|
/*-------------------------------------------------------------------------
|
|
|
|
*
|
1999-02-14 00:22:53 +01:00
|
|
|
* copy.c
|
2006-08-31 02:35:32 +02:00
|
|
|
* Implements the COPY utility command
|
1996-07-09 08:22:35 +02:00
|
|
|
*
|
2020-01-01 18:21:45 +01:00
|
|
|
* Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
|
2000-01-26 06:58:53 +01:00
|
|
|
* Portions Copyright (c) 1994, Regents of the University of California
|
1996-07-09 08:22:35 +02:00
|
|
|
*
|
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
2010-09-20 22:08:53 +02:00
|
|
|
* src/backend/commands/copy.c
|
1996-07-09 08:22:35 +02:00
|
|
|
*
|
|
|
|
*-------------------------------------------------------------------------
|
|
|
|
*/
|
2000-12-02 21:49:24 +01:00
|
|
|
#include "postgres.h"
|
1996-08-27 09:42:29 +02:00
|
|
|
|
2005-09-01 17:34:31 +02:00
|
|
|
#include <ctype.h>
|
1996-11-06 09:21:43 +01:00
|
|
|
#include <unistd.h>
|
1999-07-16 07:00:38 +02:00
|
|
|
#include <sys/stat.h>
|
1996-11-06 09:21:43 +01:00
|
|
|
|
1999-07-16 07:00:38 +02:00
|
|
|
#include "access/heapam.h"
|
2012-08-30 22:15:44 +02:00
|
|
|
#include "access/htup_details.h"
|
2010-07-22 02:47:59 +02:00
|
|
|
#include "access/sysattr.h"
|
tableam: Add and use scan APIs.
Too allow table accesses to be not directly dependent on heap, several
new abstractions are needed. Specifically:
1) Heap scans need to be generalized into table scans. Do this by
introducing TableScanDesc, which will be the "base class" for
individual AMs. This contains the AM independent fields from
HeapScanDesc.
The previous heap_{beginscan,rescan,endscan} et al. have been
replaced with a table_ version.
There's no direct replacement for heap_getnext(), as that returned
a HeapTuple, which is undesirable for a other AMs. Instead there's
table_scan_getnextslot(). But note that heap_getnext() lives on,
it's still used widely to access catalog tables.
This is achieved by new scan_begin, scan_end, scan_rescan,
scan_getnextslot callbacks.
2) The portion of parallel scans that's shared between backends need
to be able to do so without the user doing per-AM work. To achieve
that new parallelscan_{estimate, initialize, reinitialize}
callbacks are introduced, which operate on a new
ParallelTableScanDesc, which again can be subclassed by AMs.
As it is likely that several AMs are going to be block oriented,
block oriented callbacks that can be shared between such AMs are
provided and used by heap. table_block_parallelscan_{estimate,
intiialize, reinitialize} as callbacks, and
table_block_parallelscan_{nextpage, init} for use in AMs. These
operate on a ParallelBlockTableScanDesc.
3) Index scans need to be able to access tables to return a tuple, and
there needs to be state across individual accesses to the heap to
store state like buffers. That's now handled by introducing a
sort-of-scan IndexFetchTable, which again is intended to be
subclassed by individual AMs (for heap IndexFetchHeap).
The relevant callbacks for an AM are index_fetch_{end, begin,
reset} to create the necessary state, and index_fetch_tuple to
retrieve an indexed tuple. Note that index_fetch_tuple
implementations need to be smarter than just blindly fetching the
tuples for AMs that have optimizations similar to heap's HOT - the
currently alive tuple in the update chain needs to be fetched if
appropriate.
Similar to table_scan_getnextslot(), it's undesirable to continue
to return HeapTuples. Thus index_fetch_heap (might want to rename
that later) now accepts a slot as an argument. Core code doesn't
have a lot of call sites performing index scans without going
through the systable_* API (in contrast to loads of heap_getnext
calls and working directly with HeapTuples).
Index scans now store the result of a search in
IndexScanDesc->xs_heaptid, rather than xs_ctup->t_self. As the
target is not generally a HeapTuple anymore that seems cleaner.
To be able to sensible adapt code to use the above, two further
callbacks have been introduced:
a) slot_callbacks returns a TupleTableSlotOps* suitable for creating
slots capable of holding a tuple of the AMs
type. table_slot_callbacks() and table_slot_create() are based
upon that, but have additional logic to deal with views, foreign
tables, etc.
While this change could have been done separately, nearly all the
call sites that needed to be adapted for the rest of this commit
also would have been needed to be adapted for
table_slot_callbacks(), making separation not worthwhile.
b) tuple_satisfies_snapshot checks whether the tuple in a slot is
currently visible according to a snapshot. That's required as a few
places now don't have a buffer + HeapTuple around, but a
slot (which in heap's case internally has that information).
Additionally a few infrastructure changes were needed:
I) SysScanDesc, as used by systable_{beginscan, getnext} et al. now
internally uses a slot to keep track of tuples. While
systable_getnext() still returns HeapTuples, and will so for the
foreseeable future, the index API (see 1) above) now only deals with
slots.
The remainder, and largest part, of this commit is then adjusting all
scans in postgres to use the new APIs.
Author: Andres Freund, Haribabu Kommi, Alvaro Herrera
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20160812231527.GA690404@alvherre.pgsql
2019-03-11 20:46:41 +01:00
|
|
|
#include "access/tableam.h"
|
2006-07-13 18:49:20 +02:00
|
|
|
#include "access/xact.h"
|
2014-11-06 12:52:08 +01:00
|
|
|
#include "access/xlog.h"
|
2018-04-06 20:47:10 +02:00
|
|
|
#include "catalog/dependency.h"
|
|
|
|
#include "catalog/pg_authid.h"
|
1999-07-16 07:00:38 +02:00
|
|
|
#include "catalog/pg_type.h"
|
1999-07-16 01:04:24 +02:00
|
|
|
#include "commands/copy.h"
|
2009-09-21 22:10:21 +02:00
|
|
|
#include "commands/defrem.h"
|
1997-09-01 09:59:06 +02:00
|
|
|
#include "commands/trigger.h"
|
2017-11-15 16:23:28 +01:00
|
|
|
#include "executor/execPartition.h"
|
1999-07-16 07:00:38 +02:00
|
|
|
#include "executor/executor.h"
|
2019-03-30 08:13:09 +01:00
|
|
|
#include "executor/nodeModifyTable.h"
|
Use slots more widely in tuple mapping code and make naming more consistent.
It's inefficient to use a single slot for mapping between tuple
descriptors for multiple tuples, as previously done when using
ConvertPartitionTupleSlot(), as that means the slot's tuple descriptors
change for every tuple.
Previously we also, via ConvertPartitionTupleSlot(), built new tuples
after the mapping even in cases where we, immediately afterwards,
access individual columns again.
Refactor the code so one slot, on demand, is used for each
partition. That avoids having to change the descriptor (and allows to
use the more efficient "fixed" tuple slots). Then use slot->slot
mapping, to avoid unnecessarily forming a tuple.
As the naming between the tuple and slot mapping functions wasn't
consistent, rename them to execute_attr_map_{tuple,slot}. It's likely
that we'll also rename convert_tuples_by_* to denote that these
functions "only" build a map, but that's left for later.
Author: Amit Khandekar and Amit Langote, editorialized by me
Reviewed-By: Amit Langote, Amit Khandekar, Andres Freund
Discussion:
https://postgr.es/m/CAJ3gD9fR0wRNeAE8VqffNTyONS_UfFPRpqxhnD9Q42vZB+Jvpg@mail.gmail.com
https://postgr.es/m/e4f9d743-cd4b-efb0-7574-da21d86a7f36%40lab.ntt.co.jp
Backpatch: -
2018-10-02 20:14:26 +02:00
|
|
|
#include "executor/tuptable.h"
|
2018-04-07 01:16:11 +02:00
|
|
|
#include "foreign/fdwapi.h"
|
1999-07-16 01:04:24 +02:00
|
|
|
#include "libpq/libpq.h"
|
2003-04-19 02:02:30 +02:00
|
|
|
#include "libpq/pqformat.h"
|
2002-09-20 05:52:50 +02:00
|
|
|
#include "mb/pg_wchar.h"
|
1999-07-16 07:00:38 +02:00
|
|
|
#include "miscadmin.h"
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
#include "nodes/makefuncs.h"
|
2019-11-12 04:00:16 +01:00
|
|
|
#include "optimizer/optimizer.h"
|
2019-01-19 23:48:16 +01:00
|
|
|
#include "parser/parse_coerce.h"
|
|
|
|
#include "parser/parse_collate.h"
|
|
|
|
#include "parser/parse_expr.h"
|
2017-04-18 05:22:04 +02:00
|
|
|
#include "parser/parse_relation.h"
|
2017-10-02 00:36:14 +02:00
|
|
|
#include "port/pg_bswap.h"
|
2002-09-20 05:52:50 +02:00
|
|
|
#include "rewrite/rewriteHandler.h"
|
2004-02-10 02:55:27 +01:00
|
|
|
#include "storage/fd.h"
|
1999-09-27 22:00:44 +02:00
|
|
|
#include "tcop/tcopprot.h"
|
2020-03-10 10:22:52 +01:00
|
|
|
#include "utils/acl.h"
|
1999-07-16 07:00:38 +02:00
|
|
|
#include "utils/builtins.h"
|
2002-08-22 02:01:51 +02:00
|
|
|
#include "utils/lsyscache.h"
|
2005-05-06 19:24:55 +02:00
|
|
|
#include "utils/memutils.h"
|
2018-08-01 10:23:09 +02:00
|
|
|
#include "utils/partcache.h"
|
2012-12-01 13:54:20 +01:00
|
|
|
#include "utils/portal.h"
|
2011-02-23 18:18:09 +01:00
|
|
|
#include "utils/rel.h"
|
Fix column-privilege leak in error-message paths
While building error messages to return to the user,
BuildIndexValueDescription, ExecBuildSlotValueDescription and
ri_ReportViolation would happily include the entire key or entire row in
the result returned to the user, even if the user didn't have access to
view all of the columns being included.
Instead, include only those columns which the user is providing or which
the user has select rights on. If the user does not have any rights
to view the table or any of the columns involved then no detail is
provided and a NULL value is returned from BuildIndexValueDescription
and ExecBuildSlotValueDescription. Note that, for key cases, the user
must have access to all of the columns for the key to be shown; a
partial key will not be returned.
Further, in master only, do not return any data for cases where row
security is enabled on the relation and row security should be applied
for the user. This required a bit of refactoring and moving of things
around related to RLS- note the addition of utils/misc/rls.c.
Back-patch all the way, as column-level privileges are now in all
supported versions.
This has been assigned CVE-2014-8161, but since the issue and the patch
have already been publicized on pgsql-hackers, there's no point in trying
to hide this commit.
2015-01-12 23:04:11 +01:00
|
|
|
#include "utils/rls.h"
|
2008-03-26 19:48:59 +01:00
|
|
|
#include "utils/snapmgr.h"
|
1998-08-24 03:14:24 +02:00
|
|
|
|
1996-11-02 03:01:48 +01:00
|
|
|
#define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
|
2002-02-12 22:25:41 +01:00
|
|
|
#define OCTVALUE(c) ((c) - '0')
|
1996-07-09 08:22:35 +02:00
|
|
|
|
2003-04-19 02:02:30 +02:00
|
|
|
/*
|
|
|
|
* Represents the different source/dest cases we need to worry about at
|
|
|
|
* the bottom level
|
|
|
|
*/
|
|
|
|
typedef enum CopyDest
|
|
|
|
{
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
COPY_FILE, /* to/from file (or a piped program) */
|
2003-09-30 00:06:40 +02:00
|
|
|
COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
|
2017-03-23 13:36:36 +01:00
|
|
|
COPY_NEW_FE, /* to/from frontend (3.0 protocol) */
|
|
|
|
COPY_CALLBACK /* to/from callback function */
|
2003-08-08 23:42:59 +02:00
|
|
|
} CopyDest;
|
2003-04-19 02:02:30 +02:00
|
|
|
|
2003-04-19 22:36:03 +02:00
|
|
|
/*
|
2003-09-30 00:06:40 +02:00
|
|
|
* Represents the end-of-line terminator type of the input
|
2003-04-19 22:36:03 +02:00
|
|
|
*/
|
|
|
|
typedef enum EolType
|
|
|
|
{
|
|
|
|
EOL_UNKNOWN,
|
|
|
|
EOL_NL,
|
|
|
|
EOL_CR,
|
|
|
|
EOL_CRNL
|
2003-08-08 23:42:59 +02:00
|
|
|
} EolType;
|
2003-04-19 22:36:03 +02:00
|
|
|
|
2018-08-01 10:23:09 +02:00
|
|
|
/*
|
|
|
|
* Represents the heap insert method to be used during COPY FROM.
|
|
|
|
*/
|
|
|
|
typedef enum CopyInsertMethod
|
|
|
|
{
|
2019-05-24 01:25:48 +02:00
|
|
|
CIM_SINGLE, /* use table_tuple_insert or fdw routine */
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
CIM_MULTI, /* always use table_multi_insert */
|
|
|
|
CIM_MULTI_CONDITIONAL /* use table_multi_insert only if valid */
|
2018-08-01 10:23:09 +02:00
|
|
|
} CopyInsertMethod;
|
|
|
|
|
1999-07-22 04:40:07 +02:00
|
|
|
/*
|
2005-08-06 22:41:58 +02:00
|
|
|
* This struct contains all the state variables used throughout a COPY
|
2005-12-27 19:10:48 +01:00
|
|
|
* operation. For simplicity, we use the same struct for all variants of COPY,
|
|
|
|
* even though some fields are used in only some cases.
|
2005-08-06 22:41:58 +02:00
|
|
|
*
|
2005-12-27 19:10:48 +01:00
|
|
|
* Multi-byte encodings: all supported client-side encodings encode multi-byte
|
|
|
|
* characters by having the first byte's high bit set. Subsequent bytes of the
|
|
|
|
* character can have the high bit not set. When scanning data in such an
|
|
|
|
* encoding to look for a match to a single-byte (ie ASCII) character, we must
|
|
|
|
* use the full pg_encoding_mblen() machinery to skip over multibyte
|
|
|
|
* characters, else we might find a false match to a trailing byte. In
|
|
|
|
* supported server encodings, there is no possibility of a false match, and
|
|
|
|
* it's faster to make useless comparisons to trailing bytes than it is to
|
2017-08-16 06:22:32 +02:00
|
|
|
* invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
|
2005-12-27 19:10:48 +01:00
|
|
|
* when we have to do it the hard way.
|
1999-07-22 04:40:07 +02:00
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
typedef struct CopyStateData
|
|
|
|
{
|
|
|
|
/* low-level state data */
|
|
|
|
CopyDest copy_dest; /* type of copy source/destination */
|
|
|
|
FILE *copy_file; /* used if copy_dest == COPY_FILE */
|
2006-10-04 02:30:14 +02:00
|
|
|
StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for
|
|
|
|
* dest == COPY_NEW_FE in COPY FROM */
|
Handle EPIPE more sanely when we close a pipe reading from a program.
Previously, any program launched by COPY TO/FROM PROGRAM inherited the
server's setting of SIGPIPE handling, i.e. SIG_IGN. Hence, if we were
doing COPY FROM PROGRAM and closed the pipe early, the child process
would see EPIPE on its output file and typically would treat that as
a fatal error, in turn causing the COPY to report error. Similarly,
one could get a failure report from a query that didn't read all of
the output from a contrib/file_fdw foreign table that uses file_fdw's
PROGRAM option.
To fix, ensure that child programs inherit SIG_DFL not SIG_IGN processing
of SIGPIPE. This seems like an all-around better situation since if
the called program wants some non-default treatment of SIGPIPE, it would
expect to have to set that up for itself. Then in COPY, if it's COPY
FROM PROGRAM and we stop reading short of detecting EOF, treat a SIGPIPE
exit from the called program as a non-error condition. This still allows
us to report an error for any case where the called program gets SIGPIPE
on some other file descriptor.
As coded, we won't report a SIGPIPE if we stop reading as a result of
seeing an in-band EOF marker (e.g. COPY BINARY EOF marker). It's
somewhat debatable whether we should complain if the called program
continues to transmit data after an EOF marker. However, it seems like
we should avoid throwing error in any questionable cases, especially in a
back-patched fix, and anyway it would take additional code to make such
an error get reported consistently.
Back-patch to v10. We could go further back, since COPY FROM PROGRAM
has been around awhile, but AFAICS the only way to reach this situation
using core or contrib is via file_fdw, which has only supported PROGRAM
sources since v10. The COPY statement per se has no feature whereby
it'd stop reading without having hit EOF or an error already. Therefore,
I don't see any upside to back-patching further that'd outweigh the
risk of complaints about behavioral change.
Per bug #15449 from Eric Cyr.
Patch by me, review by Etsuro Fujita and Kyotaro Horiguchi
Discussion: https://postgr.es/m/15449-1cf737dd5929450e@postgresql.org
2018-11-19 23:02:25 +01:00
|
|
|
bool is_copy_from; /* COPY TO, or COPY FROM? */
|
|
|
|
bool reached_eof; /* true if we read to end of copy data (not
|
|
|
|
* all copy_dest types maintain this) */
|
2005-08-06 22:41:58 +02:00
|
|
|
EolType eol_type; /* EOL type of input */
|
2011-02-21 06:08:04 +01:00
|
|
|
int file_encoding; /* file or remote side's character encoding */
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
bool need_transcoding; /* file encoding diff from server? */
|
2005-12-27 19:10:48 +01:00
|
|
|
bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
/* parameters from the COPY command */
|
|
|
|
Relation rel; /* relation to copy to or from */
|
2006-08-31 01:34:22 +02:00
|
|
|
QueryDesc *queryDesc; /* executable query to copy from */
|
2005-08-06 22:41:58 +02:00
|
|
|
List *attnumlist; /* integer list of attnums to copy */
|
2006-08-31 01:34:22 +02:00
|
|
|
char *filename; /* filename, or NULL for STDIN/STDOUT */
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
bool is_program; /* is 'filename' a program to popen? */
|
2017-05-17 22:31:56 +02:00
|
|
|
copy_data_source_cb data_source_cb; /* function for reading data */
|
2005-08-06 22:41:58 +02:00
|
|
|
bool binary; /* binary format? */
|
2012-12-01 13:54:20 +01:00
|
|
|
bool freeze; /* freeze rows on loading? */
|
2005-08-06 22:41:58 +02:00
|
|
|
bool csv_mode; /* Comma Separated Value format? */
|
|
|
|
bool header_line; /* CSV header line? */
|
|
|
|
char *null_print; /* NULL marker string (server encoding!) */
|
2005-10-15 04:49:52 +02:00
|
|
|
int null_print_len; /* length of same */
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
char *null_print_client; /* same converted to file encoding */
|
2005-08-06 22:41:58 +02:00
|
|
|
char *delim; /* column delimiter (must be 1 byte) */
|
|
|
|
char *quote; /* CSV quote char (must be 1 byte) */
|
|
|
|
char *escape; /* CSV escape char (must be 1 byte) */
|
2011-02-20 20:06:59 +01:00
|
|
|
List *force_quote; /* list of column names */
|
2015-11-05 03:01:26 +01:00
|
|
|
bool force_quote_all; /* FORCE_QUOTE *? */
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
bool *force_quote_flags; /* per-column CSV FQ flags */
|
2011-02-20 20:06:59 +01:00
|
|
|
List *force_notnull; /* list of column names */
|
2006-08-31 01:34:22 +02:00
|
|
|
bool *force_notnull_flags; /* per-column CSV FNN flags */
|
2014-05-06 18:12:18 +02:00
|
|
|
List *force_null; /* list of column names */
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
bool *force_null_flags; /* per-column CSV FN flags */
|
2012-07-12 22:26:59 +02:00
|
|
|
bool convert_selectively; /* do selective binary conversion? */
|
2013-05-29 22:58:43 +02:00
|
|
|
List *convert_select; /* list of column names (can be NIL) */
|
2012-07-12 22:26:59 +02:00
|
|
|
bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
|
2019-01-19 23:48:16 +01:00
|
|
|
Node *whereClause; /* WHERE condition (or NULL) */
|
2005-08-06 22:41:58 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/* these are just for error messages, see CopyFromErrorCallback */
|
2005-08-06 22:41:58 +02:00
|
|
|
const char *cur_relname; /* table name for error messages */
|
2018-05-22 19:32:52 +02:00
|
|
|
uint64 cur_lineno; /* line number for error messages */
|
2005-08-06 22:41:58 +02:00
|
|
|
const char *cur_attname; /* current att for error messages */
|
|
|
|
const char *cur_attval; /* current att value for error messages */
|
2003-09-30 00:06:40 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/*
|
|
|
|
* Working state for COPY TO/FROM
|
|
|
|
*/
|
|
|
|
MemoryContext copycontext; /* per-copy execution context */
|
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
/*
|
|
|
|
* Working state for COPY TO
|
|
|
|
*/
|
2006-10-04 02:30:14 +02:00
|
|
|
FmgrInfo *out_functions; /* lookup info for output functions */
|
|
|
|
MemoryContext rowcontext; /* per-row evaluation context */
|
2006-08-31 01:34:22 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/*
|
|
|
|
* Working state for COPY FROM
|
|
|
|
*/
|
|
|
|
AttrNumber num_defaults;
|
|
|
|
FmgrInfo *in_functions; /* array of input functions for each attrs */
|
|
|
|
Oid *typioparams; /* array of element types for in_functions */
|
|
|
|
int *defmap; /* array of default att numbers */
|
|
|
|
ExprState **defexprs; /* array of default att expressions */
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
bool volatile_defexprs; /* is any of defexprs volatile? */
|
Fix column-privilege leak in error-message paths
While building error messages to return to the user,
BuildIndexValueDescription, ExecBuildSlotValueDescription and
ri_ReportViolation would happily include the entire key or entire row in
the result returned to the user, even if the user didn't have access to
view all of the columns being included.
Instead, include only those columns which the user is providing or which
the user has select rights on. If the user does not have any rights
to view the table or any of the columns involved then no detail is
provided and a NULL value is returned from BuildIndexValueDescription
and ExecBuildSlotValueDescription. Note that, for key cases, the user
must have access to all of the columns for the key to be shown; a
partial key will not be returned.
Further, in master only, do not return any data for cases where row
security is enabled on the relation and row security should be applied
for the user. This required a bit of refactoring and moving of things
around related to RLS- note the addition of utils/misc/rls.c.
Back-patch all the way, as column-level privileges are now in all
supported versions.
This has been assigned CVE-2014-8161, but since the issue and the patch
have already been publicized on pgsql-hackers, there's no point in trying
to hide this commit.
2015-01-12 23:04:11 +01:00
|
|
|
List *range_table;
|
2019-01-19 23:48:16 +01:00
|
|
|
ExprState *qualexpr;
|
2017-01-04 19:05:29 +01:00
|
|
|
|
2017-06-28 19:55:03 +02:00
|
|
|
TransitionCaptureState *transition_capture;
|
2011-02-16 03:19:11 +01:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
|
|
|
* These variables are used to reduce overhead in textual COPY FROM.
|
|
|
|
*
|
2005-11-22 19:17:34 +01:00
|
|
|
* attribute_buf holds the separated, de-escaped text for each field of
|
|
|
|
* the current line. The CopyReadAttributes functions return arrays of
|
2005-08-06 22:41:58 +02:00
|
|
|
* pointers into this buffer. We avoid palloc/pfree overhead by re-using
|
|
|
|
* the buffer on each cycle.
|
|
|
|
*/
|
|
|
|
StringInfoData attribute_buf;
|
2003-04-24 23:16:45 +02:00
|
|
|
|
2010-12-06 21:31:55 +01:00
|
|
|
/* field raw data pointers found by COPY FROM */
|
|
|
|
|
2011-04-10 17:42:00 +02:00
|
|
|
int max_fields;
|
|
|
|
char **raw_fields;
|
2010-12-06 21:31:55 +01:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
2005-10-15 04:49:52 +02:00
|
|
|
* Similarly, line_buf holds the whole input line being processed. The
|
|
|
|
* input cycle is first to read the whole line into line_buf, convert it
|
|
|
|
* to server encoding there, and then extract the individual attribute
|
|
|
|
* fields into attribute_buf. line_buf is preserved unmodified so that we
|
|
|
|
* can display it in error messages if appropriate.
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
|
|
|
StringInfoData line_buf;
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
bool line_buf_converted; /* converted to server encoding? */
|
2013-05-29 22:58:43 +02:00
|
|
|
bool line_buf_valid; /* contains the row being processed? */
|
1997-09-07 07:04:48 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
|
|
|
* Finally, raw_buf holds raw data read from the data source (file or
|
2014-05-06 18:12:18 +02:00
|
|
|
* client connection). CopyReadLine parses this data sufficiently to
|
2005-08-06 22:41:58 +02:00
|
|
|
* locate line boundaries, then transfers the data to line_buf and
|
|
|
|
* converts it. Note: we guarantee that there is a \0 at
|
|
|
|
* raw_buf[raw_buf_len].
|
|
|
|
*/
|
|
|
|
#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
|
|
|
|
char *raw_buf;
|
|
|
|
int raw_buf_index; /* next byte to process */
|
|
|
|
int raw_buf_len; /* total # of bytes stored */
|
|
|
|
} CopyStateData;
|
|
|
|
|
2015-11-27 17:11:22 +01:00
|
|
|
/* DestReceiver for COPY (query) TO */
|
2006-08-31 01:34:22 +02:00
|
|
|
typedef struct
|
|
|
|
{
|
|
|
|
DestReceiver pub; /* publicly-known function pointers */
|
|
|
|
CopyState cstate; /* CopyStateData for the command */
|
2011-02-16 03:19:11 +01:00
|
|
|
uint64 processed; /* # of tuples processed */
|
2006-08-31 01:34:22 +02:00
|
|
|
} DR_copy;
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/*
|
|
|
|
* No more than this many tuples per CopyMultiInsertBuffer
|
|
|
|
*
|
|
|
|
* Caution: Don't make this too big, as we could end up with this many
|
|
|
|
* CopyMultiInsertBuffer items stored in CopyMultiInsertInfo's
|
|
|
|
* multiInsertBuffers list. Increasing this can cause quadratic growth in
|
|
|
|
* memory requirements during copies into partitioned tables with a large
|
|
|
|
* number of partitions.
|
|
|
|
*/
|
|
|
|
#define MAX_BUFFERED_TUPLES 1000
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Flush buffers if there are >= this many bytes, as counted by the input
|
|
|
|
* size, of tuples stored.
|
|
|
|
*/
|
|
|
|
#define MAX_BUFFERED_BYTES 65535
|
|
|
|
|
|
|
|
/* Trim the list of buffers back down to this number after flushing */
|
|
|
|
#define MAX_PARTITION_BUFFERS 32
|
|
|
|
|
|
|
|
/* Stores multi-insert data related to a single relation in CopyFrom. */
|
|
|
|
typedef struct CopyMultiInsertBuffer
|
|
|
|
{
|
|
|
|
TupleTableSlot *slots[MAX_BUFFERED_TUPLES]; /* Array to store tuples */
|
|
|
|
ResultRelInfo *resultRelInfo; /* ResultRelInfo for 'relid' */
|
|
|
|
BulkInsertState bistate; /* BulkInsertState for this rel */
|
|
|
|
int nused; /* number of 'slots' containing tuples */
|
|
|
|
uint64 linenos[MAX_BUFFERED_TUPLES]; /* Line # of tuple in copy
|
|
|
|
* stream */
|
|
|
|
} CopyMultiInsertBuffer;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Stores one or many CopyMultiInsertBuffers and details about the size and
|
|
|
|
* number of tuples which are stored in them. This allows multiple buffers to
|
|
|
|
* exist at once when COPYing into a partitioned table.
|
|
|
|
*/
|
|
|
|
typedef struct CopyMultiInsertInfo
|
|
|
|
{
|
|
|
|
List *multiInsertBuffers; /* List of tracked CopyMultiInsertBuffers */
|
|
|
|
int bufferedTuples; /* number of tuples buffered over all buffers */
|
|
|
|
int bufferedBytes; /* number of bytes from all buffered tuples */
|
|
|
|
CopyState cstate; /* Copy state for this CopyMultiInsertInfo */
|
|
|
|
EState *estate; /* Executor state used for COPY */
|
|
|
|
CommandId mycid; /* Command Id used for COPY */
|
|
|
|
int ti_options; /* table insert options */
|
|
|
|
} CopyMultiInsertInfo;
|
|
|
|
|
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
/*
|
|
|
|
* These macros centralize code used to process line_buf and raw_buf buffers.
|
|
|
|
* They are macros because they often do continue/break control and to avoid
|
2009-06-03 17:06:48 +02:00
|
|
|
* function call overhead in tight COPY loops.
|
2005-12-27 19:10:48 +01:00
|
|
|
*
|
2009-06-03 17:06:48 +02:00
|
|
|
* We must use "if (1)" because the usual "do {...} while(0)" wrapper would
|
2014-05-06 18:12:18 +02:00
|
|
|
* prevent the continue/break processing from working. We end the "if (1)"
|
2009-06-03 17:06:48 +02:00
|
|
|
* with "else ((void) 0)" to ensure the "if" does not unintentionally match
|
|
|
|
* any "else" in the calling code, and to avoid any compiler warnings about
|
|
|
|
* empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
|
2005-12-27 19:10:48 +01:00
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This keeps the character read at the top of the loop in the buffer
|
|
|
|
* even if there is more than one read-ahead.
|
|
|
|
*/
|
|
|
|
#define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
|
|
|
|
if (1) \
|
|
|
|
{ \
|
|
|
|
if (raw_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
|
|
|
|
{ \
|
|
|
|
raw_buf_ptr = prev_raw_ptr; /* undo fetch */ \
|
|
|
|
need_data = true; \
|
|
|
|
continue; \
|
|
|
|
} \
|
2009-05-29 15:54:52 +02:00
|
|
|
} else ((void) 0)
|
2005-12-27 19:10:48 +01:00
|
|
|
|
|
|
|
/* This consumes the remainder of the buffer and breaks */
|
|
|
|
#define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
|
|
|
|
if (1) \
|
|
|
|
{ \
|
|
|
|
if (raw_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
|
|
|
|
{ \
|
|
|
|
if (extralen) \
|
|
|
|
raw_buf_ptr = copy_buf_len; /* consume the partial character */ \
|
|
|
|
/* backslash just before EOF, treat as data char */ \
|
|
|
|
result = true; \
|
|
|
|
break; \
|
|
|
|
} \
|
2009-05-29 15:54:52 +02:00
|
|
|
} else ((void) 0)
|
2005-12-27 19:10:48 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Transfer any approved data to line_buf; must do this to be sure
|
|
|
|
* there is some room in raw_buf.
|
|
|
|
*/
|
|
|
|
#define REFILL_LINEBUF \
|
|
|
|
if (1) \
|
|
|
|
{ \
|
|
|
|
if (raw_buf_ptr > cstate->raw_buf_index) \
|
|
|
|
{ \
|
|
|
|
appendBinaryStringInfo(&cstate->line_buf, \
|
|
|
|
cstate->raw_buf + cstate->raw_buf_index, \
|
|
|
|
raw_buf_ptr - cstate->raw_buf_index); \
|
|
|
|
cstate->raw_buf_index = raw_buf_ptr; \
|
|
|
|
} \
|
2009-05-29 15:54:52 +02:00
|
|
|
} else ((void) 0)
|
2005-12-27 19:10:48 +01:00
|
|
|
|
|
|
|
/* Undo any read-ahead and jump out of the block. */
|
|
|
|
#define NO_END_OF_COPY_GOTO \
|
|
|
|
if (1) \
|
|
|
|
{ \
|
|
|
|
raw_buf_ptr = prev_raw_ptr + 1; \
|
|
|
|
goto not_end_of_copy; \
|
2009-05-29 15:54:52 +02:00
|
|
|
} else ((void) 0)
|
2005-12-27 19:10:48 +01:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
|
2000-04-12 19:17:23 +02:00
|
|
|
|
2003-09-30 00:06:40 +02:00
|
|
|
|
|
|
|
/* non-export function prototypes */
|
Change representation of statement lists, and add statement location info.
This patch makes several changes that improve the consistency of
representation of lists of statements. It's always been the case
that the output of parse analysis is a list of Query nodes, whatever
the types of the individual statements in the list. This patch brings
similar consistency to the outputs of raw parsing and planning steps:
* The output of raw parsing is now always a list of RawStmt nodes;
the statement-type-dependent nodes are one level down from that.
* The output of pg_plan_queries() is now always a list of PlannedStmt
nodes, even for utility statements. In the case of a utility statement,
"planning" just consists of wrapping a CMD_UTILITY PlannedStmt around
the utility node. This list representation is now used in Portal and
CachedPlan plan lists, replacing the former convention of intermixing
PlannedStmts with bare utility-statement nodes.
Now, every list of statements has a consistent head-node type depending
on how far along it is in processing. This allows changing many places
that formerly used generic "Node *" pointers to use a more specific
pointer type, thus reducing the number of IsA() tests and casts needed,
as well as improving code clarity.
Also, the post-parse-analysis representation of DECLARE CURSOR is changed
so that it looks more like EXPLAIN, PREPARE, etc. That is, the contained
SELECT remains a child of the DeclareCursorStmt rather than getting flipped
around to be the other way. It's now true for both Query and PlannedStmt
that utilityStmt is non-null if and only if commandType is CMD_UTILITY.
That allows simplifying a lot of places that were testing both fields.
(I think some of those were just defensive programming, but in many places,
it was actually necessary to avoid confusing DECLARE CURSOR with SELECT.)
Because PlannedStmt carries a canSetTag field, we're also able to get rid
of some ad-hoc rules about how to reconstruct canSetTag for a bare utility
statement; specifically, the assumption that a utility is canSetTag if and
only if it's the only one in its list. While I see no near-term need for
relaxing that restriction, it's nice to get rid of the ad-hocery.
The API of ProcessUtility() is changed so that what it's passed is the
wrapper PlannedStmt not just the bare utility statement. This will affect
all users of ProcessUtility_hook, but the changes are pretty trivial; see
the affected contrib modules for examples of the minimum change needed.
(Most compilers should give pointer-type-mismatch warnings for uncorrected
code.)
There's also a change in the API of ExplainOneQuery_hook, to pass through
cursorOptions instead of expecting hook functions to know what to pick.
This is needed because of the DECLARE CURSOR changes, but really should
have been done in 9.6; it's unlikely that any extant hook functions
know about using CURSOR_OPT_PARALLEL_OK.
Finally, teach gram.y to save statement boundary locations in RawStmt
nodes, and pass those through to Query and PlannedStmt nodes. This allows
more intelligent handling of cases where a source query string contains
multiple statements. This patch doesn't actually do anything with the
information, but a follow-on patch will. (Passing this information through
cleanly is the true motivation for these changes; while I think this is all
good cleanup, it's unlikely we'd have bothered without this end goal.)
catversion bump because addition of location fields to struct Query
affects stored rules.
This patch is by me, but it owes a good deal to Fabien Coelho who did
a lot of preliminary work on the problem, and also reviewed the patch.
Discussion: https://postgr.es/m/alpine.DEB.2.20.1612200926310.29821@lancre
2017-01-14 22:02:35 +01:00
|
|
|
static CopyState BeginCopy(ParseState *pstate, bool is_from, Relation rel,
|
2019-05-22 19:04:48 +02:00
|
|
|
RawStmt *raw_query, Oid queryRelId, List *attnamelist,
|
|
|
|
List *options);
|
2011-02-16 03:19:11 +01:00
|
|
|
static void EndCopy(CopyState cstate);
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
static void ClosePipeToProgram(CopyState cstate);
|
Change representation of statement lists, and add statement location info.
This patch makes several changes that improve the consistency of
representation of lists of statements. It's always been the case
that the output of parse analysis is a list of Query nodes, whatever
the types of the individual statements in the list. This patch brings
similar consistency to the outputs of raw parsing and planning steps:
* The output of raw parsing is now always a list of RawStmt nodes;
the statement-type-dependent nodes are one level down from that.
* The output of pg_plan_queries() is now always a list of PlannedStmt
nodes, even for utility statements. In the case of a utility statement,
"planning" just consists of wrapping a CMD_UTILITY PlannedStmt around
the utility node. This list representation is now used in Portal and
CachedPlan plan lists, replacing the former convention of intermixing
PlannedStmts with bare utility-statement nodes.
Now, every list of statements has a consistent head-node type depending
on how far along it is in processing. This allows changing many places
that formerly used generic "Node *" pointers to use a more specific
pointer type, thus reducing the number of IsA() tests and casts needed,
as well as improving code clarity.
Also, the post-parse-analysis representation of DECLARE CURSOR is changed
so that it looks more like EXPLAIN, PREPARE, etc. That is, the contained
SELECT remains a child of the DeclareCursorStmt rather than getting flipped
around to be the other way. It's now true for both Query and PlannedStmt
that utilityStmt is non-null if and only if commandType is CMD_UTILITY.
That allows simplifying a lot of places that were testing both fields.
(I think some of those were just defensive programming, but in many places,
it was actually necessary to avoid confusing DECLARE CURSOR with SELECT.)
Because PlannedStmt carries a canSetTag field, we're also able to get rid
of some ad-hoc rules about how to reconstruct canSetTag for a bare utility
statement; specifically, the assumption that a utility is canSetTag if and
only if it's the only one in its list. While I see no near-term need for
relaxing that restriction, it's nice to get rid of the ad-hocery.
The API of ProcessUtility() is changed so that what it's passed is the
wrapper PlannedStmt not just the bare utility statement. This will affect
all users of ProcessUtility_hook, but the changes are pretty trivial; see
the affected contrib modules for examples of the minimum change needed.
(Most compilers should give pointer-type-mismatch warnings for uncorrected
code.)
There's also a change in the API of ExplainOneQuery_hook, to pass through
cursorOptions instead of expecting hook functions to know what to pick.
This is needed because of the DECLARE CURSOR changes, but really should
have been done in 9.6; it's unlikely that any extant hook functions
know about using CURSOR_OPT_PARALLEL_OK.
Finally, teach gram.y to save statement boundary locations in RawStmt
nodes, and pass those through to Query and PlannedStmt nodes. This allows
more intelligent handling of cases where a source query string contains
multiple statements. This patch doesn't actually do anything with the
information, but a follow-on patch will. (Passing this information through
cleanly is the true motivation for these changes; while I think this is all
good cleanup, it's unlikely we'd have bothered without this end goal.)
catversion bump because addition of location fields to struct Query
affects stored rules.
This patch is by me, but it owes a good deal to Fabien Coelho who did
a lot of preliminary work on the problem, and also reviewed the patch.
Discussion: https://postgr.es/m/alpine.DEB.2.20.1612200926310.29821@lancre
2017-01-14 22:02:35 +01:00
|
|
|
static CopyState BeginCopyTo(ParseState *pstate, Relation rel, RawStmt *query,
|
2019-05-22 19:04:48 +02:00
|
|
|
Oid queryRelId, const char *filename, bool is_program,
|
|
|
|
List *attnamelist, List *options);
|
2011-02-16 03:19:11 +01:00
|
|
|
static void EndCopyTo(CopyState cstate);
|
|
|
|
static uint64 DoCopyTo(CopyState cstate);
|
|
|
|
static uint64 CopyTo(CopyState cstate);
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
static void CopyOneRowTo(CopyState cstate, TupleTableSlot *slot);
|
2005-08-06 22:41:58 +02:00
|
|
|
static bool CopyReadLine(CopyState cstate);
|
|
|
|
static bool CopyReadLineText(CopyState cstate);
|
2011-04-10 17:42:00 +02:00
|
|
|
static int CopyReadAttributesText(CopyState cstate);
|
|
|
|
static int CopyReadAttributesCSV(CopyState cstate);
|
2005-08-06 22:41:58 +02:00
|
|
|
static Datum CopyReadBinaryAttribute(CopyState cstate,
|
2019-05-22 19:04:48 +02:00
|
|
|
int column_no, FmgrInfo *flinfo,
|
|
|
|
Oid typioparam, int32 typmod,
|
|
|
|
bool *isnull);
|
2006-05-25 20:42:17 +02:00
|
|
|
static void CopyAttributeOutText(CopyState cstate, char *string);
|
|
|
|
static void CopyAttributeOutCSV(CopyState cstate, char *string,
|
2019-05-22 19:04:48 +02:00
|
|
|
bool use_quote, bool single_attr);
|
2006-08-31 01:34:22 +02:00
|
|
|
static List *CopyGetAttnums(TupleDesc tupDesc, Relation rel,
|
2019-05-22 19:04:48 +02:00
|
|
|
List *attnamelist);
|
2005-08-06 22:41:58 +02:00
|
|
|
static char *limit_printout_length(const char *str);
|
|
|
|
|
|
|
|
/* Low-level communications functions */
|
|
|
|
static void SendCopyBegin(CopyState cstate);
|
|
|
|
static void ReceiveCopyBegin(CopyState cstate);
|
|
|
|
static void SendCopyEnd(CopyState cstate);
|
2011-09-11 20:54:32 +02:00
|
|
|
static void CopySendData(CopyState cstate, const void *databuf, int datasize);
|
2005-08-06 22:41:58 +02:00
|
|
|
static void CopySendString(CopyState cstate, const char *str);
|
|
|
|
static void CopySendChar(CopyState cstate, char c);
|
|
|
|
static void CopySendEndOfRow(CopyState cstate);
|
2019-05-22 19:04:48 +02:00
|
|
|
static int CopyGetData(CopyState cstate, void *databuf,
|
|
|
|
int minread, int maxread);
|
2005-08-06 22:41:58 +02:00
|
|
|
static void CopySendInt32(CopyState cstate, int32 val);
|
|
|
|
static bool CopyGetInt32(CopyState cstate, int32 *val);
|
|
|
|
static void CopySendInt16(CopyState cstate, int16 val);
|
|
|
|
static bool CopyGetInt16(CopyState cstate, int16 *val);
|
1999-01-11 04:56:11 +01:00
|
|
|
|
2003-09-30 00:06:40 +02:00
|
|
|
|
1999-01-11 04:56:11 +01:00
|
|
|
/*
|
2003-04-19 02:02:30 +02:00
|
|
|
* Send copy start/stop messages for frontend copies. These have changed
|
|
|
|
* in past protocol redesigns.
|
1999-01-11 04:56:11 +01:00
|
|
|
*/
|
1999-05-26 14:57:23 +02:00
|
|
|
static void
|
2005-08-06 22:41:58 +02:00
|
|
|
SendCopyBegin(CopyState cstate)
|
1999-05-25 18:15:34 +02:00
|
|
|
{
|
2003-04-19 02:02:30 +02:00
|
|
|
if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
|
|
|
|
{
|
2003-04-22 02:08:07 +02:00
|
|
|
/* new way */
|
|
|
|
StringInfoData buf;
|
2005-08-06 22:41:58 +02:00
|
|
|
int natts = list_length(cstate->attnumlist);
|
|
|
|
int16 format = (cstate->binary ? 1 : 0);
|
2003-08-04 02:43:34 +02:00
|
|
|
int i;
|
2003-04-22 02:08:07 +02:00
|
|
|
|
|
|
|
pq_beginmessage(&buf, 'H');
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
pq_sendbyte(&buf, format); /* overall format */
|
2017-10-12 06:00:46 +02:00
|
|
|
pq_sendint16(&buf, natts);
|
2003-05-08 20:16:37 +02:00
|
|
|
for (i = 0; i < natts; i++)
|
2017-11-29 15:24:24 +01:00
|
|
|
pq_sendint16(&buf, format); /* per-column formats */
|
2003-04-22 02:08:07 +02:00
|
|
|
pq_endmessage(&buf);
|
2005-08-06 22:41:58 +02:00
|
|
|
cstate->copy_dest = COPY_NEW_FE;
|
2003-04-19 02:02:30 +02:00
|
|
|
}
|
2016-10-11 18:19:18 +02:00
|
|
|
else
|
1999-07-22 04:40:07 +02:00
|
|
|
{
|
2003-04-22 02:08:07 +02:00
|
|
|
/* old way */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->binary)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("COPY BINARY is not supported to stdout or from stdin")));
|
2003-04-22 02:08:07 +02:00
|
|
|
pq_putemptymessage('H');
|
|
|
|
/* grottiness needed for old COPY OUT protocol */
|
2003-04-19 02:02:30 +02:00
|
|
|
pq_startcopyout();
|
2005-08-06 22:41:58 +02:00
|
|
|
cstate->copy_dest = COPY_OLD_FE;
|
1999-07-22 04:40:07 +02:00
|
|
|
}
|
1999-01-11 04:56:11 +01:00
|
|
|
}
|
1999-05-25 18:15:34 +02:00
|
|
|
|
1999-05-26 14:57:23 +02:00
|
|
|
static void
|
2005-08-06 22:41:58 +02:00
|
|
|
ReceiveCopyBegin(CopyState cstate)
|
1999-05-25 18:15:34 +02:00
|
|
|
{
|
2003-04-19 02:02:30 +02:00
|
|
|
if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3)
|
|
|
|
{
|
2003-04-22 02:08:07 +02:00
|
|
|
/* new way */
|
|
|
|
StringInfoData buf;
|
2005-08-06 22:41:58 +02:00
|
|
|
int natts = list_length(cstate->attnumlist);
|
|
|
|
int16 format = (cstate->binary ? 1 : 0);
|
2003-08-04 02:43:34 +02:00
|
|
|
int i;
|
2003-04-22 02:08:07 +02:00
|
|
|
|
|
|
|
pq_beginmessage(&buf, 'G');
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
pq_sendbyte(&buf, format); /* overall format */
|
2017-10-12 06:00:46 +02:00
|
|
|
pq_sendint16(&buf, natts);
|
2003-05-08 20:16:37 +02:00
|
|
|
for (i = 0; i < natts; i++)
|
2017-11-29 15:24:24 +01:00
|
|
|
pq_sendint16(&buf, format); /* per-column formats */
|
2003-04-22 02:08:07 +02:00
|
|
|
pq_endmessage(&buf);
|
2005-08-06 22:41:58 +02:00
|
|
|
cstate->copy_dest = COPY_NEW_FE;
|
2017-05-10 23:41:27 +02:00
|
|
|
cstate->fe_msgbuf = makeStringInfo();
|
2003-04-19 02:02:30 +02:00
|
|
|
}
|
2016-10-11 18:19:18 +02:00
|
|
|
else
|
2003-04-19 02:02:30 +02:00
|
|
|
{
|
2003-04-22 02:08:07 +02:00
|
|
|
/* old way */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->binary)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("COPY BINARY is not supported to stdout or from stdin")));
|
2003-04-22 02:08:07 +02:00
|
|
|
pq_putemptymessage('G');
|
Be more careful to not lose sync in the FE/BE protocol.
If any error occurred while we were in the middle of reading a protocol
message from the client, we could lose sync, and incorrectly try to
interpret a part of another message as a new protocol message. That will
usually lead to an "invalid frontend message" error that terminates the
connection. However, this is a security issue because an attacker might
be able to deliberately cause an error, inject a Query message in what's
supposed to be just user data, and have the server execute it.
We were quite careful to not have CHECK_FOR_INTERRUPTS() calls or other
operations that could ereport(ERROR) in the middle of processing a message,
but a query cancel interrupt or statement timeout could nevertheless cause
it to happen. Also, the V2 fastpath and COPY handling were not so careful.
It's very difficult to recover in the V2 COPY protocol, so we will just
terminate the connection on error. In practice, that's what happened
previously anyway, as we lost protocol sync.
To fix, add a new variable in pqcomm.c, PqCommReadingMsg, that is set
whenever we're in the middle of reading a message. When it's set, we cannot
safely ERROR out and continue running, because we might've read only part
of a message. PqCommReadingMsg acts somewhat similarly to critical sections
in that if an error occurs while it's set, the error handler will force the
connection to be terminated, as if the error was FATAL. It's not
implemented by promoting ERROR to FATAL in elog.c, like ERROR is promoted
to PANIC in critical sections, because we want to be able to use
PG_TRY/CATCH to recover and regain protocol sync. pq_getmessage() takes
advantage of that to prevent an OOM error from terminating the connection.
To prevent unnecessary connection terminations, add a holdoff mechanism
similar to HOLD/RESUME_INTERRUPTS() that can be used hold off query cancel
interrupts, but still allow die interrupts. The rules on which interrupts
are processed when are now a bit more complicated, so refactor
ProcessInterrupts() and the calls to it in signal handlers so that the
signal handlers always call it if ImmediateInterruptOK is set, and
ProcessInterrupts() can decide to not do anything if the other conditions
are not met.
Reported by Emil Lenngren. Patch reviewed by Noah Misch and Andres Freund.
Backpatch to all supported versions.
Security: CVE-2015-0244
2015-02-02 16:08:45 +01:00
|
|
|
/* any error in old protocol will make us lose sync */
|
|
|
|
pq_startmsgread();
|
2005-08-06 22:41:58 +02:00
|
|
|
cstate->copy_dest = COPY_OLD_FE;
|
2003-04-19 02:02:30 +02:00
|
|
|
}
|
|
|
|
/* We *must* flush here to ensure FE knows it can send. */
|
|
|
|
pq_flush();
|
1999-01-11 04:56:11 +01:00
|
|
|
}
|
|
|
|
|
1999-05-26 14:57:23 +02:00
|
|
|
static void
|
2005-08-06 22:41:58 +02:00
|
|
|
SendCopyEnd(CopyState cstate)
|
1999-05-25 18:15:34 +02:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->copy_dest == COPY_NEW_FE)
|
2003-04-19 22:36:03 +02:00
|
|
|
{
|
2006-05-27 00:50:02 +02:00
|
|
|
/* Shouldn't have any unsent data */
|
|
|
|
Assert(cstate->fe_msgbuf->len == 0);
|
2003-04-22 02:08:07 +02:00
|
|
|
/* Send Copy Done message */
|
|
|
|
pq_putemptymessage('c');
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2006-05-27 00:50:02 +02:00
|
|
|
CopySendData(cstate, "\\.", 2);
|
|
|
|
/* Need to flush out the trailer (this also appends a newline) */
|
|
|
|
CopySendEndOfRow(cstate);
|
2003-04-22 02:08:07 +02:00
|
|
|
pq_endcopyout(false);
|
2003-04-19 22:36:03 +02:00
|
|
|
}
|
1999-01-11 04:56:11 +01:00
|
|
|
}
|
|
|
|
|
2003-04-22 02:08:07 +02:00
|
|
|
/*----------
|
2003-04-19 02:02:30 +02:00
|
|
|
* CopySendData sends output data to the destination (file or frontend)
|
|
|
|
* CopySendString does the same for null-terminated strings
|
|
|
|
* CopySendChar does the same for single characters
|
2003-04-22 02:08:07 +02:00
|
|
|
* CopySendEndOfRow does the appropriate thing at end of each data row
|
2006-05-27 00:50:02 +02:00
|
|
|
* (data is not actually flushed except by CopySendEndOfRow)
|
1999-04-25 05:19:27 +02:00
|
|
|
*
|
|
|
|
* NB: no data conversion is applied by these functions
|
2003-04-22 02:08:07 +02:00
|
|
|
*----------
|
1999-01-11 04:56:11 +01:00
|
|
|
*/
|
1999-05-26 14:57:23 +02:00
|
|
|
static void
|
2011-09-11 20:54:32 +02:00
|
|
|
CopySendData(CopyState cstate, const void *databuf, int datasize)
|
1999-05-25 18:15:34 +02:00
|
|
|
{
|
2011-09-11 20:54:32 +02:00
|
|
|
appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
|
1999-01-11 04:56:11 +01:00
|
|
|
}
|
|
|
|
|
2003-04-19 02:02:30 +02:00
|
|
|
static void
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendString(CopyState cstate, const char *str)
|
2003-04-19 02:02:30 +02:00
|
|
|
{
|
2006-05-27 00:50:02 +02:00
|
|
|
appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
|
2003-04-19 02:02:30 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendChar(CopyState cstate, char c)
|
2003-04-19 02:02:30 +02:00
|
|
|
{
|
2006-05-27 00:50:02 +02:00
|
|
|
appendStringInfoCharMacro(cstate->fe_msgbuf, c);
|
2003-04-19 02:02:30 +02:00
|
|
|
}
|
|
|
|
|
2003-04-22 02:08:07 +02:00
|
|
|
static void
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendEndOfRow(CopyState cstate)
|
2003-04-22 02:08:07 +02:00
|
|
|
{
|
2006-05-27 00:50:02 +02:00
|
|
|
StringInfo fe_msgbuf = cstate->fe_msgbuf;
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
switch (cstate->copy_dest)
|
2003-04-22 02:08:07 +02:00
|
|
|
{
|
|
|
|
case COPY_FILE:
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!cstate->binary)
|
2003-04-22 02:08:07 +02:00
|
|
|
{
|
|
|
|
/* Default line termination depends on platform */
|
|
|
|
#ifndef WIN32
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendChar(cstate, '\n');
|
2003-04-22 02:08:07 +02:00
|
|
|
#else
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendString(cstate, "\r\n");
|
2003-04-22 02:08:07 +02:00
|
|
|
#endif
|
|
|
|
}
|
2006-05-27 00:50:02 +02:00
|
|
|
|
2011-10-19 03:37:51 +02:00
|
|
|
if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
|
|
|
|
cstate->copy_file) != 1 ||
|
|
|
|
ferror(cstate->copy_file))
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
{
|
|
|
|
if (cstate->is_program)
|
|
|
|
{
|
|
|
|
if (errno == EPIPE)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The pipe will be closed automatically on error at
|
|
|
|
* the end of transaction, but we might get a better
|
|
|
|
* error message from the subprocess' exit code than
|
|
|
|
* just "Broken Pipe"
|
|
|
|
*/
|
|
|
|
ClosePipeToProgram(cstate);
|
|
|
|
|
|
|
|
/*
|
2013-05-29 22:58:43 +02:00
|
|
|
* If ClosePipeToProgram() didn't throw an error, the
|
|
|
|
* program terminated normally, but closed the pipe
|
|
|
|
* first. Restore errno, and throw an error.
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
*/
|
|
|
|
errno = EPIPE;
|
|
|
|
}
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not write to COPY program: %m")));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not write to COPY file: %m")));
|
|
|
|
}
|
2003-04-22 02:08:07 +02:00
|
|
|
break;
|
|
|
|
case COPY_OLD_FE:
|
|
|
|
/* The FE/BE protocol uses \n as newline for all platforms */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!cstate->binary)
|
|
|
|
CopySendChar(cstate, '\n');
|
2006-05-27 00:50:02 +02:00
|
|
|
|
|
|
|
if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
|
|
|
|
{
|
|
|
|
/* no hope of recovering connection sync, so FATAL */
|
|
|
|
ereport(FATAL,
|
|
|
|
(errcode(ERRCODE_CONNECTION_FAILURE),
|
|
|
|
errmsg("connection lost during COPY to stdout")));
|
|
|
|
}
|
2003-04-22 02:08:07 +02:00
|
|
|
break;
|
|
|
|
case COPY_NEW_FE:
|
|
|
|
/* The FE/BE protocol uses \n as newline for all platforms */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!cstate->binary)
|
|
|
|
CopySendChar(cstate, '\n');
|
2006-05-27 00:50:02 +02:00
|
|
|
|
2003-04-22 02:08:07 +02:00
|
|
|
/* Dump the accumulated row as one CopyData message */
|
2006-05-27 00:50:02 +02:00
|
|
|
(void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
|
2003-04-22 02:08:07 +02:00
|
|
|
break;
|
2017-03-23 13:36:36 +01:00
|
|
|
case COPY_CALLBACK:
|
2017-05-17 22:31:56 +02:00
|
|
|
Assert(false); /* Not yet supported. */
|
2017-03-23 13:36:36 +01:00
|
|
|
break;
|
2003-04-22 02:08:07 +02:00
|
|
|
}
|
2006-05-27 00:50:02 +02:00
|
|
|
|
2007-03-03 20:32:55 +01:00
|
|
|
resetStringInfo(fe_msgbuf);
|
2003-04-22 02:08:07 +02:00
|
|
|
}
|
|
|
|
|
2003-04-19 02:02:30 +02:00
|
|
|
/*
|
|
|
|
* CopyGetData reads data from the source (file or frontend)
|
|
|
|
*
|
2005-08-06 22:41:58 +02:00
|
|
|
* We attempt to read at least minread, and at most maxread, bytes from
|
2014-05-06 18:12:18 +02:00
|
|
|
* the source. The actual number of bytes read is returned; if this is
|
2005-08-06 22:41:58 +02:00
|
|
|
* less than minread, EOF was detected.
|
2003-04-19 02:02:30 +02:00
|
|
|
*
|
|
|
|
* Note: when copying from the frontend, we expect a proper EOF mark per
|
|
|
|
* protocol; if the frontend simply drops the connection, we raise error.
|
|
|
|
* It seems unwise to allow the COPY IN to complete normally in that case.
|
|
|
|
*
|
2005-08-06 22:41:58 +02:00
|
|
|
* NB: no data conversion is applied here.
|
2003-04-19 02:02:30 +02:00
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
static int
|
|
|
|
CopyGetData(CopyState cstate, void *databuf, int minread, int maxread)
|
1999-05-25 18:15:34 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
int bytesread = 0;
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
switch (cstate->copy_dest)
|
1999-05-25 18:15:34 +02:00
|
|
|
{
|
2003-04-19 02:02:30 +02:00
|
|
|
case COPY_FILE:
|
2005-08-06 22:41:58 +02:00
|
|
|
bytesread = fread(databuf, 1, maxread, cstate->copy_file);
|
|
|
|
if (ferror(cstate->copy_file))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not read from COPY file: %m")));
|
Handle EPIPE more sanely when we close a pipe reading from a program.
Previously, any program launched by COPY TO/FROM PROGRAM inherited the
server's setting of SIGPIPE handling, i.e. SIG_IGN. Hence, if we were
doing COPY FROM PROGRAM and closed the pipe early, the child process
would see EPIPE on its output file and typically would treat that as
a fatal error, in turn causing the COPY to report error. Similarly,
one could get a failure report from a query that didn't read all of
the output from a contrib/file_fdw foreign table that uses file_fdw's
PROGRAM option.
To fix, ensure that child programs inherit SIG_DFL not SIG_IGN processing
of SIGPIPE. This seems like an all-around better situation since if
the called program wants some non-default treatment of SIGPIPE, it would
expect to have to set that up for itself. Then in COPY, if it's COPY
FROM PROGRAM and we stop reading short of detecting EOF, treat a SIGPIPE
exit from the called program as a non-error condition. This still allows
us to report an error for any case where the called program gets SIGPIPE
on some other file descriptor.
As coded, we won't report a SIGPIPE if we stop reading as a result of
seeing an in-band EOF marker (e.g. COPY BINARY EOF marker). It's
somewhat debatable whether we should complain if the called program
continues to transmit data after an EOF marker. However, it seems like
we should avoid throwing error in any questionable cases, especially in a
back-patched fix, and anyway it would take additional code to make such
an error get reported consistently.
Back-patch to v10. We could go further back, since COPY FROM PROGRAM
has been around awhile, but AFAICS the only way to reach this situation
using core or contrib is via file_fdw, which has only supported PROGRAM
sources since v10. The COPY statement per se has no feature whereby
it'd stop reading without having hit EOF or an error already. Therefore,
I don't see any upside to back-patching further that'd outweigh the
risk of complaints about behavioral change.
Per bug #15449 from Eric Cyr.
Patch by me, review by Etsuro Fujita and Kyotaro Horiguchi
Discussion: https://postgr.es/m/15449-1cf737dd5929450e@postgresql.org
2018-11-19 23:02:25 +01:00
|
|
|
if (bytesread == 0)
|
|
|
|
cstate->reached_eof = true;
|
2003-04-19 02:02:30 +02:00
|
|
|
break;
|
|
|
|
case COPY_OLD_FE:
|
2005-10-15 04:49:52 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
|
|
|
* We cannot read more than minread bytes (which in practice is 1)
|
|
|
|
* because old protocol doesn't have any clear way of separating
|
2005-10-15 04:49:52 +02:00
|
|
|
* the COPY stream from following data. This is slow, but not any
|
|
|
|
* slower than the code path was originally, and we don't care
|
|
|
|
* much anymore about the performance of old protocol.
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
|
|
|
if (pq_getbytes((char *) databuf, minread))
|
2003-04-19 02:02:30 +02:00
|
|
|
{
|
|
|
|
/* Only a \. terminator is legal EOF in old protocol */
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_CONNECTION_FAILURE),
|
2012-05-07 18:39:37 +02:00
|
|
|
errmsg("unexpected EOF on client connection with an open transaction")));
|
2003-04-19 02:02:30 +02:00
|
|
|
}
|
2005-08-06 22:41:58 +02:00
|
|
|
bytesread = minread;
|
2003-04-19 02:02:30 +02:00
|
|
|
break;
|
|
|
|
case COPY_NEW_FE:
|
Handle EPIPE more sanely when we close a pipe reading from a program.
Previously, any program launched by COPY TO/FROM PROGRAM inherited the
server's setting of SIGPIPE handling, i.e. SIG_IGN. Hence, if we were
doing COPY FROM PROGRAM and closed the pipe early, the child process
would see EPIPE on its output file and typically would treat that as
a fatal error, in turn causing the COPY to report error. Similarly,
one could get a failure report from a query that didn't read all of
the output from a contrib/file_fdw foreign table that uses file_fdw's
PROGRAM option.
To fix, ensure that child programs inherit SIG_DFL not SIG_IGN processing
of SIGPIPE. This seems like an all-around better situation since if
the called program wants some non-default treatment of SIGPIPE, it would
expect to have to set that up for itself. Then in COPY, if it's COPY
FROM PROGRAM and we stop reading short of detecting EOF, treat a SIGPIPE
exit from the called program as a non-error condition. This still allows
us to report an error for any case where the called program gets SIGPIPE
on some other file descriptor.
As coded, we won't report a SIGPIPE if we stop reading as a result of
seeing an in-band EOF marker (e.g. COPY BINARY EOF marker). It's
somewhat debatable whether we should complain if the called program
continues to transmit data after an EOF marker. However, it seems like
we should avoid throwing error in any questionable cases, especially in a
back-patched fix, and anyway it would take additional code to make such
an error get reported consistently.
Back-patch to v10. We could go further back, since COPY FROM PROGRAM
has been around awhile, but AFAICS the only way to reach this situation
using core or contrib is via file_fdw, which has only supported PROGRAM
sources since v10. The COPY statement per se has no feature whereby
it'd stop reading without having hit EOF or an error already. Therefore,
I don't see any upside to back-patching further that'd outweigh the
risk of complaints about behavioral change.
Per bug #15449 from Eric Cyr.
Patch by me, review by Etsuro Fujita and Kyotaro Horiguchi
Discussion: https://postgr.es/m/15449-1cf737dd5929450e@postgresql.org
2018-11-19 23:02:25 +01:00
|
|
|
while (maxread > 0 && bytesread < minread && !cstate->reached_eof)
|
2003-04-19 02:02:30 +02:00
|
|
|
{
|
2003-08-04 02:43:34 +02:00
|
|
|
int avail;
|
1999-05-25 18:15:34 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
|
2003-04-19 02:02:30 +02:00
|
|
|
{
|
|
|
|
/* Try to receive another message */
|
|
|
|
int mtype;
|
|
|
|
|
2004-08-29 07:07:03 +02:00
|
|
|
readmessage:
|
Be more careful to not lose sync in the FE/BE protocol.
If any error occurred while we were in the middle of reading a protocol
message from the client, we could lose sync, and incorrectly try to
interpret a part of another message as a new protocol message. That will
usually lead to an "invalid frontend message" error that terminates the
connection. However, this is a security issue because an attacker might
be able to deliberately cause an error, inject a Query message in what's
supposed to be just user data, and have the server execute it.
We were quite careful to not have CHECK_FOR_INTERRUPTS() calls or other
operations that could ereport(ERROR) in the middle of processing a message,
but a query cancel interrupt or statement timeout could nevertheless cause
it to happen. Also, the V2 fastpath and COPY handling were not so careful.
It's very difficult to recover in the V2 COPY protocol, so we will just
terminate the connection on error. In practice, that's what happened
previously anyway, as we lost protocol sync.
To fix, add a new variable in pqcomm.c, PqCommReadingMsg, that is set
whenever we're in the middle of reading a message. When it's set, we cannot
safely ERROR out and continue running, because we might've read only part
of a message. PqCommReadingMsg acts somewhat similarly to critical sections
in that if an error occurs while it's set, the error handler will force the
connection to be terminated, as if the error was FATAL. It's not
implemented by promoting ERROR to FATAL in elog.c, like ERROR is promoted
to PANIC in critical sections, because we want to be able to use
PG_TRY/CATCH to recover and regain protocol sync. pq_getmessage() takes
advantage of that to prevent an OOM error from terminating the connection.
To prevent unnecessary connection terminations, add a holdoff mechanism
similar to HOLD/RESUME_INTERRUPTS() that can be used hold off query cancel
interrupts, but still allow die interrupts. The rules on which interrupts
are processed when are now a bit more complicated, so refactor
ProcessInterrupts() and the calls to it in signal handlers so that the
signal handlers always call it if ImmediateInterruptOK is set, and
ProcessInterrupts() can decide to not do anything if the other conditions
are not met.
Reported by Emil Lenngren. Patch reviewed by Noah Misch and Andres Freund.
Backpatch to all supported versions.
Security: CVE-2015-0244
2015-02-02 16:08:45 +01:00
|
|
|
HOLD_CANCEL_INTERRUPTS();
|
|
|
|
pq_startmsgread();
|
2003-04-19 02:02:30 +02:00
|
|
|
mtype = pq_getbyte();
|
|
|
|
if (mtype == EOF)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_CONNECTION_FAILURE),
|
2012-06-10 21:20:04 +02:00
|
|
|
errmsg("unexpected EOF on client connection with an open transaction")));
|
2005-08-06 22:41:58 +02:00
|
|
|
if (pq_getmessage(cstate->fe_msgbuf, 0))
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_CONNECTION_FAILURE),
|
2012-06-10 21:20:04 +02:00
|
|
|
errmsg("unexpected EOF on client connection with an open transaction")));
|
Be more careful to not lose sync in the FE/BE protocol.
If any error occurred while we were in the middle of reading a protocol
message from the client, we could lose sync, and incorrectly try to
interpret a part of another message as a new protocol message. That will
usually lead to an "invalid frontend message" error that terminates the
connection. However, this is a security issue because an attacker might
be able to deliberately cause an error, inject a Query message in what's
supposed to be just user data, and have the server execute it.
We were quite careful to not have CHECK_FOR_INTERRUPTS() calls or other
operations that could ereport(ERROR) in the middle of processing a message,
but a query cancel interrupt or statement timeout could nevertheless cause
it to happen. Also, the V2 fastpath and COPY handling were not so careful.
It's very difficult to recover in the V2 COPY protocol, so we will just
terminate the connection on error. In practice, that's what happened
previously anyway, as we lost protocol sync.
To fix, add a new variable in pqcomm.c, PqCommReadingMsg, that is set
whenever we're in the middle of reading a message. When it's set, we cannot
safely ERROR out and continue running, because we might've read only part
of a message. PqCommReadingMsg acts somewhat similarly to critical sections
in that if an error occurs while it's set, the error handler will force the
connection to be terminated, as if the error was FATAL. It's not
implemented by promoting ERROR to FATAL in elog.c, like ERROR is promoted
to PANIC in critical sections, because we want to be able to use
PG_TRY/CATCH to recover and regain protocol sync. pq_getmessage() takes
advantage of that to prevent an OOM error from terminating the connection.
To prevent unnecessary connection terminations, add a holdoff mechanism
similar to HOLD/RESUME_INTERRUPTS() that can be used hold off query cancel
interrupts, but still allow die interrupts. The rules on which interrupts
are processed when are now a bit more complicated, so refactor
ProcessInterrupts() and the calls to it in signal handlers so that the
signal handlers always call it if ImmediateInterruptOK is set, and
ProcessInterrupts() can decide to not do anything if the other conditions
are not met.
Reported by Emil Lenngren. Patch reviewed by Noah Misch and Andres Freund.
Backpatch to all supported versions.
Security: CVE-2015-0244
2015-02-02 16:08:45 +01:00
|
|
|
RESUME_CANCEL_INTERRUPTS();
|
2003-04-19 02:02:30 +02:00
|
|
|
switch (mtype)
|
|
|
|
{
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
case 'd': /* CopyData */
|
2003-04-19 02:02:30 +02:00
|
|
|
break;
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
case 'c': /* CopyDone */
|
2003-04-19 02:02:30 +02:00
|
|
|
/* COPY IN correctly terminated by frontend */
|
Handle EPIPE more sanely when we close a pipe reading from a program.
Previously, any program launched by COPY TO/FROM PROGRAM inherited the
server's setting of SIGPIPE handling, i.e. SIG_IGN. Hence, if we were
doing COPY FROM PROGRAM and closed the pipe early, the child process
would see EPIPE on its output file and typically would treat that as
a fatal error, in turn causing the COPY to report error. Similarly,
one could get a failure report from a query that didn't read all of
the output from a contrib/file_fdw foreign table that uses file_fdw's
PROGRAM option.
To fix, ensure that child programs inherit SIG_DFL not SIG_IGN processing
of SIGPIPE. This seems like an all-around better situation since if
the called program wants some non-default treatment of SIGPIPE, it would
expect to have to set that up for itself. Then in COPY, if it's COPY
FROM PROGRAM and we stop reading short of detecting EOF, treat a SIGPIPE
exit from the called program as a non-error condition. This still allows
us to report an error for any case where the called program gets SIGPIPE
on some other file descriptor.
As coded, we won't report a SIGPIPE if we stop reading as a result of
seeing an in-band EOF marker (e.g. COPY BINARY EOF marker). It's
somewhat debatable whether we should complain if the called program
continues to transmit data after an EOF marker. However, it seems like
we should avoid throwing error in any questionable cases, especially in a
back-patched fix, and anyway it would take additional code to make such
an error get reported consistently.
Back-patch to v10. We could go further back, since COPY FROM PROGRAM
has been around awhile, but AFAICS the only way to reach this situation
using core or contrib is via file_fdw, which has only supported PROGRAM
sources since v10. The COPY statement per se has no feature whereby
it'd stop reading without having hit EOF or an error already. Therefore,
I don't see any upside to back-patching further that'd outweigh the
risk of complaints about behavioral change.
Per bug #15449 from Eric Cyr.
Patch by me, review by Etsuro Fujita and Kyotaro Horiguchi
Discussion: https://postgr.es/m/15449-1cf737dd5929450e@postgresql.org
2018-11-19 23:02:25 +01:00
|
|
|
cstate->reached_eof = true;
|
2005-08-06 22:41:58 +02:00
|
|
|
return bytesread;
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
case 'f': /* CopyFail */
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_QUERY_CANCELED),
|
|
|
|
errmsg("COPY from stdin failed: %s",
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
pq_getmsgstring(cstate->fe_msgbuf))));
|
2003-04-19 02:02:30 +02:00
|
|
|
break;
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
case 'H': /* Flush */
|
|
|
|
case 'S': /* Sync */
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2003-08-13 20:56:21 +02:00
|
|
|
/*
|
2005-10-15 04:49:52 +02:00
|
|
|
* Ignore Flush/Sync for the convenience of client
|
|
|
|
* libraries (such as libpq) that may send those
|
|
|
|
* without noticing that the command they just
|
|
|
|
* sent was COPY.
|
2003-08-13 20:56:21 +02:00
|
|
|
*/
|
|
|
|
goto readmessage;
|
2003-04-19 02:02:30 +02:00
|
|
|
default:
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_PROTOCOL_VIOLATION),
|
|
|
|
errmsg("unexpected message type 0x%02X during COPY from stdin",
|
|
|
|
mtype)));
|
2003-04-19 02:02:30 +02:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2005-08-06 22:41:58 +02:00
|
|
|
avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
|
|
|
|
if (avail > maxread)
|
|
|
|
avail = maxread;
|
|
|
|
pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
|
2003-04-19 02:02:30 +02:00
|
|
|
databuf = (void *) ((char *) databuf + avail);
|
2005-08-06 22:41:58 +02:00
|
|
|
maxread -= avail;
|
|
|
|
bytesread += avail;
|
2003-08-04 02:43:34 +02:00
|
|
|
}
|
2003-04-19 02:02:30 +02:00
|
|
|
break;
|
2017-03-23 13:36:36 +01:00
|
|
|
case COPY_CALLBACK:
|
|
|
|
bytesread = cstate->data_source_cb(databuf, minread, maxread);
|
|
|
|
break;
|
1999-07-22 04:40:07 +02:00
|
|
|
}
|
2003-08-04 02:43:34 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
return bytesread;
|
1999-01-11 04:56:11 +01:00
|
|
|
}
|
1999-05-25 18:15:34 +02:00
|
|
|
|
1999-01-11 04:56:11 +01:00
|
|
|
|
2003-05-09 23:19:50 +02:00
|
|
|
/*
|
|
|
|
* These functions do apply some data conversion
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* CopySendInt32 sends an int32 in network byte order
|
|
|
|
*/
|
|
|
|
static void
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendInt32(CopyState cstate, int32 val)
|
2003-05-09 23:19:50 +02:00
|
|
|
{
|
|
|
|
uint32 buf;
|
|
|
|
|
2017-10-02 00:36:14 +02:00
|
|
|
buf = pg_hton32((uint32) val);
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendData(cstate, &buf, sizeof(buf));
|
2003-05-09 23:19:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* CopyGetInt32 reads an int32 that appears in network byte order
|
2005-08-06 22:41:58 +02:00
|
|
|
*
|
|
|
|
* Returns true if OK, false if EOF
|
2003-05-09 23:19:50 +02:00
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
static bool
|
|
|
|
CopyGetInt32(CopyState cstate, int32 *val)
|
2003-05-09 23:19:50 +02:00
|
|
|
{
|
|
|
|
uint32 buf;
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
|
2005-09-25 00:54:44 +02:00
|
|
|
{
|
|
|
|
*val = 0; /* suppress compiler warning */
|
2005-08-06 22:41:58 +02:00
|
|
|
return false;
|
2005-09-25 00:54:44 +02:00
|
|
|
}
|
2017-10-02 00:36:14 +02:00
|
|
|
*val = (int32) pg_ntoh32(buf);
|
2005-08-06 22:41:58 +02:00
|
|
|
return true;
|
2003-05-09 23:19:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* CopySendInt16 sends an int16 in network byte order
|
|
|
|
*/
|
|
|
|
static void
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendInt16(CopyState cstate, int16 val)
|
2003-05-09 23:19:50 +02:00
|
|
|
{
|
|
|
|
uint16 buf;
|
|
|
|
|
2017-10-02 00:36:14 +02:00
|
|
|
buf = pg_hton16((uint16) val);
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendData(cstate, &buf, sizeof(buf));
|
2003-05-09 23:19:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* CopyGetInt16 reads an int16 that appears in network byte order
|
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
static bool
|
|
|
|
CopyGetInt16(CopyState cstate, int16 *val)
|
2003-05-09 23:19:50 +02:00
|
|
|
{
|
|
|
|
uint16 buf;
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (CopyGetData(cstate, &buf, sizeof(buf), sizeof(buf)) != sizeof(buf))
|
2005-09-25 00:54:44 +02:00
|
|
|
{
|
|
|
|
*val = 0; /* suppress compiler warning */
|
2005-08-06 22:41:58 +02:00
|
|
|
return false;
|
2005-09-25 00:54:44 +02:00
|
|
|
}
|
2017-10-02 00:36:14 +02:00
|
|
|
*val = (int16) pg_ntoh16(buf);
|
2005-08-06 22:41:58 +02:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* CopyLoadRawBuf loads some more data into raw_buf
|
|
|
|
*
|
2017-08-16 06:22:32 +02:00
|
|
|
* Returns true if able to obtain at least one more byte, else false.
|
2005-08-06 22:41:58 +02:00
|
|
|
*
|
|
|
|
* If raw_buf_index < raw_buf_len, the unprocessed bytes are transferred
|
|
|
|
* down to the start of the buffer and then we load more data after that.
|
|
|
|
* This case is used only when a frontend multibyte character crosses a
|
|
|
|
* bufferload boundary.
|
|
|
|
*/
|
|
|
|
static bool
|
|
|
|
CopyLoadRawBuf(CopyState cstate)
|
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
int nbytes;
|
|
|
|
int inbytes;
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
if (cstate->raw_buf_index < cstate->raw_buf_len)
|
|
|
|
{
|
|
|
|
/* Copy down the unprocessed data */
|
|
|
|
nbytes = cstate->raw_buf_len - cstate->raw_buf_index;
|
|
|
|
memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
|
|
|
|
nbytes);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
nbytes = 0; /* no data need be saved */
|
|
|
|
|
|
|
|
inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
|
|
|
|
1, RAW_BUF_SIZE - nbytes);
|
|
|
|
nbytes += inbytes;
|
|
|
|
cstate->raw_buf[nbytes] = '\0';
|
|
|
|
cstate->raw_buf_index = 0;
|
|
|
|
cstate->raw_buf_len = nbytes;
|
|
|
|
return (inbytes > 0);
|
2003-05-09 23:19:50 +02:00
|
|
|
}
|
|
|
|
|
1999-01-11 04:56:11 +01:00
|
|
|
|
1996-11-02 03:01:48 +01:00
|
|
|
/*
|
2006-08-31 05:17:50 +02:00
|
|
|
* DoCopy executes the SQL COPY statement
|
2001-08-10 20:57:42 +02:00
|
|
|
*
|
2002-03-29 20:06:29 +01:00
|
|
|
* Either unload or reload contents of table <relation>, depending on <from>.
|
2017-08-16 06:22:32 +02:00
|
|
|
* (<from> = true means we are inserting into the table.) In the "TO" case
|
2015-11-27 17:11:22 +01:00
|
|
|
* we also support copying the output of an arbitrary SELECT, INSERT, UPDATE
|
|
|
|
* or DELETE query.
|
2001-08-10 20:57:42 +02:00
|
|
|
*
|
|
|
|
* If <pipe> is false, transfer is between the table and the file named
|
2014-05-06 18:12:18 +02:00
|
|
|
* <filename>. Otherwise, transfer is between the table and our regular
|
2001-10-25 07:50:21 +02:00
|
|
|
* input/output stream. The latter could be either stdin/stdout or a
|
2001-08-10 20:57:42 +02:00
|
|
|
* socket, depending on whether we're running under Postmaster control.
|
|
|
|
*
|
2019-06-17 09:13:16 +02:00
|
|
|
* Do not allow a Postgres user without the 'pg_read_server_files' or
|
|
|
|
* 'pg_write_server_files' role to read from or write to a file.
|
2011-02-16 03:19:11 +01:00
|
|
|
*
|
|
|
|
* Do not allow the copy if user doesn't have proper permission to access
|
|
|
|
* the table or the specifically requested columns.
|
|
|
|
*/
|
Change representation of statement lists, and add statement location info.
This patch makes several changes that improve the consistency of
representation of lists of statements. It's always been the case
that the output of parse analysis is a list of Query nodes, whatever
the types of the individual statements in the list. This patch brings
similar consistency to the outputs of raw parsing and planning steps:
* The output of raw parsing is now always a list of RawStmt nodes;
the statement-type-dependent nodes are one level down from that.
* The output of pg_plan_queries() is now always a list of PlannedStmt
nodes, even for utility statements. In the case of a utility statement,
"planning" just consists of wrapping a CMD_UTILITY PlannedStmt around
the utility node. This list representation is now used in Portal and
CachedPlan plan lists, replacing the former convention of intermixing
PlannedStmts with bare utility-statement nodes.
Now, every list of statements has a consistent head-node type depending
on how far along it is in processing. This allows changing many places
that formerly used generic "Node *" pointers to use a more specific
pointer type, thus reducing the number of IsA() tests and casts needed,
as well as improving code clarity.
Also, the post-parse-analysis representation of DECLARE CURSOR is changed
so that it looks more like EXPLAIN, PREPARE, etc. That is, the contained
SELECT remains a child of the DeclareCursorStmt rather than getting flipped
around to be the other way. It's now true for both Query and PlannedStmt
that utilityStmt is non-null if and only if commandType is CMD_UTILITY.
That allows simplifying a lot of places that were testing both fields.
(I think some of those were just defensive programming, but in many places,
it was actually necessary to avoid confusing DECLARE CURSOR with SELECT.)
Because PlannedStmt carries a canSetTag field, we're also able to get rid
of some ad-hoc rules about how to reconstruct canSetTag for a bare utility
statement; specifically, the assumption that a utility is canSetTag if and
only if it's the only one in its list. While I see no near-term need for
relaxing that restriction, it's nice to get rid of the ad-hocery.
The API of ProcessUtility() is changed so that what it's passed is the
wrapper PlannedStmt not just the bare utility statement. This will affect
all users of ProcessUtility_hook, but the changes are pretty trivial; see
the affected contrib modules for examples of the minimum change needed.
(Most compilers should give pointer-type-mismatch warnings for uncorrected
code.)
There's also a change in the API of ExplainOneQuery_hook, to pass through
cursorOptions instead of expecting hook functions to know what to pick.
This is needed because of the DECLARE CURSOR changes, but really should
have been done in 9.6; it's unlikely that any extant hook functions
know about using CURSOR_OPT_PARALLEL_OK.
Finally, teach gram.y to save statement boundary locations in RawStmt
nodes, and pass those through to Query and PlannedStmt nodes. This allows
more intelligent handling of cases where a source query string contains
multiple statements. This patch doesn't actually do anything with the
information, but a follow-on patch will. (Passing this information through
cleanly is the true motivation for these changes; while I think this is all
good cleanup, it's unlikely we'd have bothered without this end goal.)
catversion bump because addition of location fields to struct Query
affects stored rules.
This patch is by me, but it owes a good deal to Fabien Coelho who did
a lot of preliminary work on the problem, and also reviewed the patch.
Discussion: https://postgr.es/m/alpine.DEB.2.20.1612200926310.29821@lancre
2017-01-14 22:02:35 +01:00
|
|
|
void
|
|
|
|
DoCopy(ParseState *pstate, const CopyStmt *stmt,
|
|
|
|
int stmt_location, int stmt_len,
|
|
|
|
uint64 *processed)
|
2011-02-16 03:19:11 +01:00
|
|
|
{
|
|
|
|
CopyState cstate;
|
|
|
|
bool is_from = stmt->is_from;
|
|
|
|
bool pipe = (stmt->filename == NULL);
|
|
|
|
Relation rel;
|
2013-05-29 22:58:43 +02:00
|
|
|
Oid relid;
|
Change representation of statement lists, and add statement location info.
This patch makes several changes that improve the consistency of
representation of lists of statements. It's always been the case
that the output of parse analysis is a list of Query nodes, whatever
the types of the individual statements in the list. This patch brings
similar consistency to the outputs of raw parsing and planning steps:
* The output of raw parsing is now always a list of RawStmt nodes;
the statement-type-dependent nodes are one level down from that.
* The output of pg_plan_queries() is now always a list of PlannedStmt
nodes, even for utility statements. In the case of a utility statement,
"planning" just consists of wrapping a CMD_UTILITY PlannedStmt around
the utility node. This list representation is now used in Portal and
CachedPlan plan lists, replacing the former convention of intermixing
PlannedStmts with bare utility-statement nodes.
Now, every list of statements has a consistent head-node type depending
on how far along it is in processing. This allows changing many places
that formerly used generic "Node *" pointers to use a more specific
pointer type, thus reducing the number of IsA() tests and casts needed,
as well as improving code clarity.
Also, the post-parse-analysis representation of DECLARE CURSOR is changed
so that it looks more like EXPLAIN, PREPARE, etc. That is, the contained
SELECT remains a child of the DeclareCursorStmt rather than getting flipped
around to be the other way. It's now true for both Query and PlannedStmt
that utilityStmt is non-null if and only if commandType is CMD_UTILITY.
That allows simplifying a lot of places that were testing both fields.
(I think some of those were just defensive programming, but in many places,
it was actually necessary to avoid confusing DECLARE CURSOR with SELECT.)
Because PlannedStmt carries a canSetTag field, we're also able to get rid
of some ad-hoc rules about how to reconstruct canSetTag for a bare utility
statement; specifically, the assumption that a utility is canSetTag if and
only if it's the only one in its list. While I see no near-term need for
relaxing that restriction, it's nice to get rid of the ad-hocery.
The API of ProcessUtility() is changed so that what it's passed is the
wrapper PlannedStmt not just the bare utility statement. This will affect
all users of ProcessUtility_hook, but the changes are pretty trivial; see
the affected contrib modules for examples of the minimum change needed.
(Most compilers should give pointer-type-mismatch warnings for uncorrected
code.)
There's also a change in the API of ExplainOneQuery_hook, to pass through
cursorOptions instead of expecting hook functions to know what to pick.
This is needed because of the DECLARE CURSOR changes, but really should
have been done in 9.6; it's unlikely that any extant hook functions
know about using CURSOR_OPT_PARALLEL_OK.
Finally, teach gram.y to save statement boundary locations in RawStmt
nodes, and pass those through to Query and PlannedStmt nodes. This allows
more intelligent handling of cases where a source query string contains
multiple statements. This patch doesn't actually do anything with the
information, but a follow-on patch will. (Passing this information through
cleanly is the true motivation for these changes; while I think this is all
good cleanup, it's unlikely we'd have bothered without this end goal.)
catversion bump because addition of location fields to struct Query
affects stored rules.
This patch is by me, but it owes a good deal to Fabien Coelho who did
a lot of preliminary work on the problem, and also reviewed the patch.
Discussion: https://postgr.es/m/alpine.DEB.2.20.1612200926310.29821@lancre
2017-01-14 22:02:35 +01:00
|
|
|
RawStmt *query = NULL;
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
Node *whereClause = NULL;
|
2011-02-16 03:19:11 +01:00
|
|
|
|
2018-04-06 20:47:10 +02:00
|
|
|
/*
|
|
|
|
* Disallow COPY to/from file or program except to users with the
|
|
|
|
* appropriate role.
|
|
|
|
*/
|
|
|
|
if (!pipe)
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
{
|
|
|
|
if (stmt->is_program)
|
2018-04-06 20:47:10 +02:00
|
|
|
{
|
|
|
|
if (!is_member_of_role(GetUserId(), DEFAULT_ROLE_EXECUTE_SERVER_PROGRAM))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
|
|
|
|
errmsg("must be superuser or a member of the pg_execute_server_program role to COPY to or from an external program"),
|
|
|
|
errhint("Anyone can COPY to stdout or from stdin. "
|
|
|
|
"psql's \\copy command also works for anyone.")));
|
|
|
|
}
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
else
|
2018-04-06 20:47:10 +02:00
|
|
|
{
|
|
|
|
if (is_from && !is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_SERVER_FILES))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
|
|
|
|
errmsg("must be superuser or a member of the pg_read_server_files role to COPY from a file"),
|
|
|
|
errhint("Anyone can COPY to stdout or from stdin. "
|
|
|
|
"psql's \\copy command also works for anyone.")));
|
|
|
|
|
|
|
|
if (!is_from && !is_member_of_role(GetUserId(), DEFAULT_ROLE_WRITE_SERVER_FILES))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
|
|
|
|
errmsg("must be superuser or a member of the pg_write_server_files role to COPY to a file"),
|
|
|
|
errhint("Anyone can COPY to stdout or from stdin. "
|
|
|
|
"psql's \\copy command also works for anyone.")));
|
|
|
|
}
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
}
|
2011-02-16 03:19:11 +01:00
|
|
|
|
|
|
|
if (stmt->relation)
|
|
|
|
{
|
Create an RTE field to record the query's lock mode for each relation.
Add RangeTblEntry.rellockmode, which records the appropriate lock mode for
each RTE_RELATION rangetable entry (either AccessShareLock, RowShareLock,
or RowExclusiveLock depending on the RTE's role in the query).
This patch creates the field and makes all creators of RTE nodes fill it
in reasonably, but for the moment nothing much is done with it. The plan
is to replace assorted post-parser logic that re-determines the right
lockmode to use with simple uses of rte->rellockmode. For now, just add
Asserts in each of those places that the rellockmode matches what they are
computing today. (In some cases the match isn't perfect, so the Asserts
are weaker than you might expect; but this seems OK, as per discussion.)
This passes check-world for me, but it seems worth pushing in this state
to see if the buildfarm finds any problems in cases I failed to test.
catversion bump due to change of stored rules.
Amit Langote, reviewed by David Rowley and Jesper Pedersen,
and whacked around a bit more by me
Discussion: https://postgr.es/m/468c85d9-540e-66a2-1dde-fec2b741e688@lab.ntt.co.jp
2018-09-30 19:55:51 +02:00
|
|
|
LOCKMODE lockmode = is_from ? RowExclusiveLock : AccessShareLock;
|
Make parser rely more heavily on the ParseNamespaceItem data structure.
When I added the ParseNamespaceItem data structure (in commit 5ebaaa494),
it wasn't very tightly integrated into the parser's APIs. In the wake of
adding p_rtindex to that struct (commit b541e9acc), there is a good reason
to make more use of it: by passing around ParseNamespaceItem pointers
instead of bare RTE pointers, we can get rid of various messy methods for
passing back or deducing the rangetable index of an RTE during parsing.
Hence, refactor the addRangeTableEntryXXX functions to build and return
a ParseNamespaceItem struct, not just the RTE proper; and replace
addRTEtoQuery with addNSItemToQuery, which is passed a ParseNamespaceItem
rather than building one internally.
Also, add per-column data (a ParseNamespaceColumn array) to each
ParseNamespaceItem. These arrays are built during addRangeTableEntryXXX,
where we have column type data at hand so that it's nearly free to fill
the data structure. Later, when we need to build Vars referencing RTEs,
we can use the ParseNamespaceColumn info to avoid the rather expensive
operations done in get_rte_attribute_type() or expandRTE().
get_rte_attribute_type() is indeed dead code now, so I've removed it.
This makes for a useful improvement in parse analysis speed, around 20%
in one moderately-complex test query.
The ParseNamespaceColumn structs also include Var identity information
(varno/varattno). That info isn't actually being used in this patch,
except that p_varno == 0 is a handy test for a dropped column.
A follow-on patch will make more use of it.
Discussion: https://postgr.es/m/2461.1577764221@sss.pgh.pa.us
2020-01-02 17:29:01 +01:00
|
|
|
ParseNamespaceItem *nsitem;
|
Create an RTE field to record the query's lock mode for each relation.
Add RangeTblEntry.rellockmode, which records the appropriate lock mode for
each RTE_RELATION rangetable entry (either AccessShareLock, RowShareLock,
or RowExclusiveLock depending on the RTE's role in the query).
This patch creates the field and makes all creators of RTE nodes fill it
in reasonably, but for the moment nothing much is done with it. The plan
is to replace assorted post-parser logic that re-determines the right
lockmode to use with simple uses of rte->rellockmode. For now, just add
Asserts in each of those places that the rellockmode matches what they are
computing today. (In some cases the match isn't perfect, so the Asserts
are weaker than you might expect; but this seems OK, as per discussion.)
This passes check-world for me, but it seems worth pushing in this state
to see if the buildfarm finds any problems in cases I failed to test.
catversion bump due to change of stored rules.
Amit Langote, reviewed by David Rowley and Jesper Pedersen,
and whacked around a bit more by me
Discussion: https://postgr.es/m/468c85d9-540e-66a2-1dde-fec2b741e688@lab.ntt.co.jp
2018-09-30 19:55:51 +02:00
|
|
|
RangeTblEntry *rte;
|
2011-04-10 17:42:00 +02:00
|
|
|
TupleDesc tupDesc;
|
|
|
|
List *attnums;
|
|
|
|
ListCell *cur;
|
2011-02-16 03:19:11 +01:00
|
|
|
|
|
|
|
Assert(!stmt->query);
|
|
|
|
|
|
|
|
/* Open and lock the relation, using the appropriate lock type. */
|
2019-01-21 19:32:19 +01:00
|
|
|
rel = table_openrv(stmt->relation, lockmode);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
2012-12-29 13:55:37 +01:00
|
|
|
relid = RelationGetRelid(rel);
|
|
|
|
|
Make parser rely more heavily on the ParseNamespaceItem data structure.
When I added the ParseNamespaceItem data structure (in commit 5ebaaa494),
it wasn't very tightly integrated into the parser's APIs. In the wake of
adding p_rtindex to that struct (commit b541e9acc), there is a good reason
to make more use of it: by passing around ParseNamespaceItem pointers
instead of bare RTE pointers, we can get rid of various messy methods for
passing back or deducing the rangetable index of an RTE during parsing.
Hence, refactor the addRangeTableEntryXXX functions to build and return
a ParseNamespaceItem struct, not just the RTE proper; and replace
addRTEtoQuery with addNSItemToQuery, which is passed a ParseNamespaceItem
rather than building one internally.
Also, add per-column data (a ParseNamespaceColumn array) to each
ParseNamespaceItem. These arrays are built during addRangeTableEntryXXX,
where we have column type data at hand so that it's nearly free to fill
the data structure. Later, when we need to build Vars referencing RTEs,
we can use the ParseNamespaceColumn info to avoid the rather expensive
operations done in get_rte_attribute_type() or expandRTE().
get_rte_attribute_type() is indeed dead code now, so I've removed it.
This makes for a useful improvement in parse analysis speed, around 20%
in one moderately-complex test query.
The ParseNamespaceColumn structs also include Var identity information
(varno/varattno). That info isn't actually being used in this patch,
except that p_varno == 0 is a handy test for a dropped column.
A follow-on patch will make more use of it.
Discussion: https://postgr.es/m/2461.1577764221@sss.pgh.pa.us
2020-01-02 17:29:01 +01:00
|
|
|
nsitem = addRangeTableEntryForRelation(pstate, rel, lockmode,
|
|
|
|
NULL, false, false);
|
|
|
|
rte = nsitem->p_rte;
|
2017-04-18 05:22:04 +02:00
|
|
|
rte->requiredPerms = (is_from ? ACL_INSERT : ACL_SELECT);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
2019-01-19 23:48:16 +01:00
|
|
|
if (stmt->whereClause)
|
|
|
|
{
|
Make parser rely more heavily on the ParseNamespaceItem data structure.
When I added the ParseNamespaceItem data structure (in commit 5ebaaa494),
it wasn't very tightly integrated into the parser's APIs. In the wake of
adding p_rtindex to that struct (commit b541e9acc), there is a good reason
to make more use of it: by passing around ParseNamespaceItem pointers
instead of bare RTE pointers, we can get rid of various messy methods for
passing back or deducing the rangetable index of an RTE during parsing.
Hence, refactor the addRangeTableEntryXXX functions to build and return
a ParseNamespaceItem struct, not just the RTE proper; and replace
addRTEtoQuery with addNSItemToQuery, which is passed a ParseNamespaceItem
rather than building one internally.
Also, add per-column data (a ParseNamespaceColumn array) to each
ParseNamespaceItem. These arrays are built during addRangeTableEntryXXX,
where we have column type data at hand so that it's nearly free to fill
the data structure. Later, when we need to build Vars referencing RTEs,
we can use the ParseNamespaceColumn info to avoid the rather expensive
operations done in get_rte_attribute_type() or expandRTE().
get_rte_attribute_type() is indeed dead code now, so I've removed it.
This makes for a useful improvement in parse analysis speed, around 20%
in one moderately-complex test query.
The ParseNamespaceColumn structs also include Var identity information
(varno/varattno). That info isn't actually being used in this patch,
except that p_varno == 0 is a handy test for a dropped column.
A follow-on patch will make more use of it.
Discussion: https://postgr.es/m/2461.1577764221@sss.pgh.pa.us
2020-01-02 17:29:01 +01:00
|
|
|
/* add nsitem to query namespace */
|
|
|
|
addNSItemToQuery(pstate, nsitem, false, true, true);
|
2019-01-19 23:48:16 +01:00
|
|
|
|
|
|
|
/* Transform the raw expression tree */
|
|
|
|
whereClause = transformExpr(pstate, stmt->whereClause, EXPR_KIND_COPY_WHERE);
|
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* Make sure it yields a boolean result. */
|
2019-01-19 23:48:16 +01:00
|
|
|
whereClause = coerce_to_boolean(pstate, whereClause, "WHERE");
|
|
|
|
|
|
|
|
/* we have to fix its collations too */
|
|
|
|
assign_expr_collations(pstate, whereClause);
|
|
|
|
|
|
|
|
whereClause = eval_const_expressions(NULL, whereClause);
|
|
|
|
|
|
|
|
whereClause = (Node *) canonicalize_qual((Expr *) whereClause, false);
|
|
|
|
whereClause = (Node *) make_ands_implicit((Expr *) whereClause);
|
|
|
|
}
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
tupDesc = RelationGetDescr(rel);
|
|
|
|
attnums = CopyGetAttnums(tupDesc, rel, stmt->attlist);
|
2011-02-20 20:06:59 +01:00
|
|
|
foreach(cur, attnums)
|
2011-02-16 03:19:11 +01:00
|
|
|
{
|
2011-04-10 17:42:00 +02:00
|
|
|
int attno = lfirst_int(cur) -
|
|
|
|
FirstLowInvalidHeapAttributeNumber;
|
2011-02-16 03:19:11 +01:00
|
|
|
|
|
|
|
if (is_from)
|
2015-05-08 00:20:46 +02:00
|
|
|
rte->insertedCols = bms_add_member(rte->insertedCols, attno);
|
2011-02-16 03:19:11 +01:00
|
|
|
else
|
|
|
|
rte->selectedCols = bms_add_member(rte->selectedCols, attno);
|
|
|
|
}
|
2017-04-18 05:22:04 +02:00
|
|
|
ExecCheckRTPerms(pstate->p_rtable, true);
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
|
|
|
|
/*
|
Rename pg_rowsecurity -> pg_policy and other fixes
As pointed out by Robert, we should really have named pg_rowsecurity
pg_policy, as the objects stored in that catalog are policies. This
patch fixes that and updates the column names to start with 'pol' to
match the new catalog name.
The security consideration for COPY with row level security, also
pointed out by Robert, has also been addressed by remembering and
re-checking the OID of the relation initially referenced during COPY
processing, to make sure it hasn't changed under us by the time we
finish planning out the query which has been built.
Robert and Alvaro also commented on missing OCLASS and OBJECT entries
for POLICY (formerly ROWSECURITY or POLICY, depending) in various
places. This patch fixes that too, which also happens to add the
ability to COMMENT on policies.
In passing, attempt to improve the consistency of messages, comments,
and documentation as well. This removes various incarnations of
'row-security', 'row-level security', 'Row-security', etc, in favor
of 'policy', 'row level security' or 'row_security' as appropriate.
Happy Thanksgiving!
2014-11-27 07:06:36 +01:00
|
|
|
* Permission check for row security policies.
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
*
|
|
|
|
* check_enable_rls will ereport(ERROR) if the user has requested
|
|
|
|
* something invalid and will otherwise indicate if we should enable
|
|
|
|
* RLS (returns RLS_ENABLED) or not for this COPY statement.
|
|
|
|
*
|
|
|
|
* If the relation has a row security policy and we are to apply it
|
2015-05-24 03:35:49 +02:00
|
|
|
* then perform a "query" copy and allow the normal query processing
|
|
|
|
* to handle the policies.
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
*
|
|
|
|
* If RLS is not enabled for this, then just fall through to the
|
|
|
|
* normal non-filtering relation handling.
|
|
|
|
*/
|
Fix column-privilege leak in error-message paths
While building error messages to return to the user,
BuildIndexValueDescription, ExecBuildSlotValueDescription and
ri_ReportViolation would happily include the entire key or entire row in
the result returned to the user, even if the user didn't have access to
view all of the columns being included.
Instead, include only those columns which the user is providing or which
the user has select rights on. If the user does not have any rights
to view the table or any of the columns involved then no detail is
provided and a NULL value is returned from BuildIndexValueDescription
and ExecBuildSlotValueDescription. Note that, for key cases, the user
must have access to all of the columns for the key to be shown; a
partial key will not be returned.
Further, in master only, do not return any data for cases where row
security is enabled on the relation and row security should be applied
for the user. This required a bit of refactoring and moving of things
around related to RLS- note the addition of utils/misc/rls.c.
Back-patch all the way, as column-level privileges are now in all
supported versions.
This has been assigned CVE-2014-8161, but since the issue and the patch
have already been publicized on pgsql-hackers, there's no point in trying
to hide this commit.
2015-01-12 23:04:11 +01:00
|
|
|
if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED)
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
{
|
|
|
|
SelectStmt *select;
|
|
|
|
ColumnRef *cr;
|
|
|
|
ResTarget *target;
|
|
|
|
RangeVar *from;
|
2016-10-03 22:22:57 +02:00
|
|
|
List *targetList = NIL;
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
|
|
|
|
if (is_from)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("COPY FROM not supported with row-level security"),
|
2015-10-29 01:23:53 +01:00
|
|
|
errhint("Use INSERT statements instead.")));
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
|
2016-10-03 22:22:57 +02:00
|
|
|
/*
|
|
|
|
* Build target list
|
|
|
|
*
|
|
|
|
* If no columns are specified in the attribute list of the COPY
|
|
|
|
* command, then the target list is 'all' columns. Therefore, '*'
|
|
|
|
* should be used as the target list for the resulting SELECT
|
|
|
|
* statement.
|
|
|
|
*
|
|
|
|
* In the case that columns are specified in the attribute list,
|
2016-11-10 20:13:43 +01:00
|
|
|
* create a ColumnRef and ResTarget for each column and add them
|
|
|
|
* to the target list for the resulting SELECT statement.
|
2016-10-03 22:22:57 +02:00
|
|
|
*/
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
if (!stmt->attlist)
|
2016-10-03 22:22:57 +02:00
|
|
|
{
|
|
|
|
cr = makeNode(ColumnRef);
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
cr->fields = list_make1(makeNode(A_Star));
|
2016-10-03 22:22:57 +02:00
|
|
|
cr->location = -1;
|
|
|
|
|
|
|
|
target = makeNode(ResTarget);
|
|
|
|
target->name = NULL;
|
|
|
|
target->indirection = NIL;
|
|
|
|
target->val = (Node *) cr;
|
|
|
|
target->location = -1;
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
|
2016-10-03 22:22:57 +02:00
|
|
|
targetList = list_make1(target);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
ListCell *lc;
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
|
2016-10-03 22:22:57 +02:00
|
|
|
foreach(lc, stmt->attlist)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Build the ColumnRef for each column. The ColumnRef
|
|
|
|
* 'fields' property is a String 'Value' node (see
|
|
|
|
* nodes/value.h) that corresponds to the column name
|
|
|
|
* respectively.
|
|
|
|
*/
|
|
|
|
cr = makeNode(ColumnRef);
|
|
|
|
cr->fields = list_make1(lfirst(lc));
|
|
|
|
cr->location = -1;
|
|
|
|
|
|
|
|
/* Build the ResTarget and add the ColumnRef to it. */
|
|
|
|
target = makeNode(ResTarget);
|
|
|
|
target->name = NULL;
|
|
|
|
target->indirection = NIL;
|
|
|
|
target->val = (Node *) cr;
|
|
|
|
target->location = -1;
|
|
|
|
|
|
|
|
/* Add each column to the SELECT statement's target list */
|
|
|
|
targetList = lappend(targetList, target);
|
|
|
|
}
|
|
|
|
}
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
|
2015-07-27 22:48:26 +02:00
|
|
|
/*
|
|
|
|
* Build RangeVar for from clause, fully qualified based on the
|
|
|
|
* relation which we have opened and locked.
|
|
|
|
*/
|
|
|
|
from = makeRangeVar(get_namespace_name(RelationGetNamespace(rel)),
|
2017-03-06 22:50:47 +01:00
|
|
|
pstrdup(RelationGetRelationName(rel)),
|
|
|
|
-1);
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
|
|
|
|
/* Build query */
|
|
|
|
select = makeNode(SelectStmt);
|
2016-10-03 22:22:57 +02:00
|
|
|
select->targetList = targetList;
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
select->fromClause = list_make1(from);
|
|
|
|
|
Change representation of statement lists, and add statement location info.
This patch makes several changes that improve the consistency of
representation of lists of statements. It's always been the case
that the output of parse analysis is a list of Query nodes, whatever
the types of the individual statements in the list. This patch brings
similar consistency to the outputs of raw parsing and planning steps:
* The output of raw parsing is now always a list of RawStmt nodes;
the statement-type-dependent nodes are one level down from that.
* The output of pg_plan_queries() is now always a list of PlannedStmt
nodes, even for utility statements. In the case of a utility statement,
"planning" just consists of wrapping a CMD_UTILITY PlannedStmt around
the utility node. This list representation is now used in Portal and
CachedPlan plan lists, replacing the former convention of intermixing
PlannedStmts with bare utility-statement nodes.
Now, every list of statements has a consistent head-node type depending
on how far along it is in processing. This allows changing many places
that formerly used generic "Node *" pointers to use a more specific
pointer type, thus reducing the number of IsA() tests and casts needed,
as well as improving code clarity.
Also, the post-parse-analysis representation of DECLARE CURSOR is changed
so that it looks more like EXPLAIN, PREPARE, etc. That is, the contained
SELECT remains a child of the DeclareCursorStmt rather than getting flipped
around to be the other way. It's now true for both Query and PlannedStmt
that utilityStmt is non-null if and only if commandType is CMD_UTILITY.
That allows simplifying a lot of places that were testing both fields.
(I think some of those were just defensive programming, but in many places,
it was actually necessary to avoid confusing DECLARE CURSOR with SELECT.)
Because PlannedStmt carries a canSetTag field, we're also able to get rid
of some ad-hoc rules about how to reconstruct canSetTag for a bare utility
statement; specifically, the assumption that a utility is canSetTag if and
only if it's the only one in its list. While I see no near-term need for
relaxing that restriction, it's nice to get rid of the ad-hocery.
The API of ProcessUtility() is changed so that what it's passed is the
wrapper PlannedStmt not just the bare utility statement. This will affect
all users of ProcessUtility_hook, but the changes are pretty trivial; see
the affected contrib modules for examples of the minimum change needed.
(Most compilers should give pointer-type-mismatch warnings for uncorrected
code.)
There's also a change in the API of ExplainOneQuery_hook, to pass through
cursorOptions instead of expecting hook functions to know what to pick.
This is needed because of the DECLARE CURSOR changes, but really should
have been done in 9.6; it's unlikely that any extant hook functions
know about using CURSOR_OPT_PARALLEL_OK.
Finally, teach gram.y to save statement boundary locations in RawStmt
nodes, and pass those through to Query and PlannedStmt nodes. This allows
more intelligent handling of cases where a source query string contains
multiple statements. This patch doesn't actually do anything with the
information, but a follow-on patch will. (Passing this information through
cleanly is the true motivation for these changes; while I think this is all
good cleanup, it's unlikely we'd have bothered without this end goal.)
catversion bump because addition of location fields to struct Query
affects stored rules.
This patch is by me, but it owes a good deal to Fabien Coelho who did
a lot of preliminary work on the problem, and also reviewed the patch.
Discussion: https://postgr.es/m/alpine.DEB.2.20.1612200926310.29821@lancre
2017-01-14 22:02:35 +01:00
|
|
|
query = makeNode(RawStmt);
|
|
|
|
query->stmt = (Node *) select;
|
|
|
|
query->stmt_location = stmt_location;
|
|
|
|
query->stmt_len = stmt_len;
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
|
2015-07-27 22:48:26 +02:00
|
|
|
/*
|
|
|
|
* Close the relation for now, but keep the lock on it to prevent
|
|
|
|
* changes between now and when we start the query-based COPY.
|
|
|
|
*
|
|
|
|
* We'll reopen it later as part of the query-based COPY.
|
|
|
|
*/
|
2019-01-21 19:32:19 +01:00
|
|
|
table_close(rel, NoLock);
|
Row-Level Security Policies (RLS)
Building on the updatable security-barrier views work, add the
ability to define policies on tables to limit the set of rows
which are returned from a query and which are allowed to be added
to a table. Expressions defined by the policy for filtering are
added to the security barrier quals of the query, while expressions
defined to check records being added to a table are added to the
with-check options of the query.
New top-level commands are CREATE/ALTER/DROP POLICY and are
controlled by the table owner. Row Security is able to be enabled
and disabled by the owner on a per-table basis using
ALTER TABLE .. ENABLE/DISABLE ROW SECURITY.
Per discussion, ROW SECURITY is disabled on tables by default and
must be enabled for policies on the table to be used. If no
policies exist on a table with ROW SECURITY enabled, a default-deny
policy is used and no records will be visible.
By default, row security is applied at all times except for the
table owner and the superuser. A new GUC, row_security, is added
which can be set to ON, OFF, or FORCE. When set to FORCE, row
security will be applied even for the table owner and superusers.
When set to OFF, row security will be disabled when allowed and an
error will be thrown if the user does not have rights to bypass row
security.
Per discussion, pg_dump sets row_security = OFF by default to ensure
that exports and backups will have all data in the table or will
error if there are insufficient privileges to bypass row security.
A new option has been added to pg_dump, --enable-row-security, to
ask pg_dump to export with row security enabled.
A new role capability, BYPASSRLS, which can only be set by the
superuser, is added to allow other users to be able to bypass row
security using row_security = OFF.
Many thanks to the various individuals who have helped with the
design, particularly Robert Haas for his feedback.
Authors include Craig Ringer, KaiGai Kohei, Adam Brightwell, Dean
Rasheed, with additional changes and rework by me.
Reviewers have included all of the above, Greg Smith,
Jeff McCormick, and Robert Haas.
2014-09-19 17:18:35 +02:00
|
|
|
rel = NULL;
|
|
|
|
}
|
2011-02-16 03:19:11 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
Assert(stmt->query);
|
|
|
|
|
Change representation of statement lists, and add statement location info.
This patch makes several changes that improve the consistency of
representation of lists of statements. It's always been the case
that the output of parse analysis is a list of Query nodes, whatever
the types of the individual statements in the list. This patch brings
similar consistency to the outputs of raw parsing and planning steps:
* The output of raw parsing is now always a list of RawStmt nodes;
the statement-type-dependent nodes are one level down from that.
* The output of pg_plan_queries() is now always a list of PlannedStmt
nodes, even for utility statements. In the case of a utility statement,
"planning" just consists of wrapping a CMD_UTILITY PlannedStmt around
the utility node. This list representation is now used in Portal and
CachedPlan plan lists, replacing the former convention of intermixing
PlannedStmts with bare utility-statement nodes.
Now, every list of statements has a consistent head-node type depending
on how far along it is in processing. This allows changing many places
that formerly used generic "Node *" pointers to use a more specific
pointer type, thus reducing the number of IsA() tests and casts needed,
as well as improving code clarity.
Also, the post-parse-analysis representation of DECLARE CURSOR is changed
so that it looks more like EXPLAIN, PREPARE, etc. That is, the contained
SELECT remains a child of the DeclareCursorStmt rather than getting flipped
around to be the other way. It's now true for both Query and PlannedStmt
that utilityStmt is non-null if and only if commandType is CMD_UTILITY.
That allows simplifying a lot of places that were testing both fields.
(I think some of those were just defensive programming, but in many places,
it was actually necessary to avoid confusing DECLARE CURSOR with SELECT.)
Because PlannedStmt carries a canSetTag field, we're also able to get rid
of some ad-hoc rules about how to reconstruct canSetTag for a bare utility
statement; specifically, the assumption that a utility is canSetTag if and
only if it's the only one in its list. While I see no near-term need for
relaxing that restriction, it's nice to get rid of the ad-hocery.
The API of ProcessUtility() is changed so that what it's passed is the
wrapper PlannedStmt not just the bare utility statement. This will affect
all users of ProcessUtility_hook, but the changes are pretty trivial; see
the affected contrib modules for examples of the minimum change needed.
(Most compilers should give pointer-type-mismatch warnings for uncorrected
code.)
There's also a change in the API of ExplainOneQuery_hook, to pass through
cursorOptions instead of expecting hook functions to know what to pick.
This is needed because of the DECLARE CURSOR changes, but really should
have been done in 9.6; it's unlikely that any extant hook functions
know about using CURSOR_OPT_PARALLEL_OK.
Finally, teach gram.y to save statement boundary locations in RawStmt
nodes, and pass those through to Query and PlannedStmt nodes. This allows
more intelligent handling of cases where a source query string contains
multiple statements. This patch doesn't actually do anything with the
information, but a follow-on patch will. (Passing this information through
cleanly is the true motivation for these changes; while I think this is all
good cleanup, it's unlikely we'd have bothered without this end goal.)
catversion bump because addition of location fields to struct Query
affects stored rules.
This patch is by me, but it owes a good deal to Fabien Coelho who did
a lot of preliminary work on the problem, and also reviewed the patch.
Discussion: https://postgr.es/m/alpine.DEB.2.20.1612200926310.29821@lancre
2017-01-14 22:02:35 +01:00
|
|
|
query = makeNode(RawStmt);
|
|
|
|
query->stmt = stmt->query;
|
|
|
|
query->stmt_location = stmt_location;
|
|
|
|
query->stmt_len = stmt_len;
|
|
|
|
|
2012-12-31 06:13:40 +01:00
|
|
|
relid = InvalidOid;
|
2011-02-16 03:19:11 +01:00
|
|
|
rel = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (is_from)
|
|
|
|
{
|
2012-03-14 21:44:40 +01:00
|
|
|
Assert(rel);
|
|
|
|
|
Create an infrastructure for parallel computation in PostgreSQL.
This does four basic things. First, it provides convenience routines
to coordinate the startup and shutdown of parallel workers. Second,
it synchronizes various pieces of state (e.g. GUCs, combo CID
mappings, transaction snapshot) from the parallel group leader to the
worker processes. Third, it prohibits various operations that would
result in unsafe changes to that state while parallelism is active.
Finally, it propagates events that would result in an ErrorResponse,
NoticeResponse, or NotifyResponse message being sent to the client
from the parallel workers back to the master, from which they can then
be sent on to the client.
Robert Haas, Amit Kapila, Noah Misch, Rushabh Lathia, Jeevan Chalke.
Suggestions and review from Andres Freund, Heikki Linnakangas, Noah
Misch, Simon Riggs, Euler Taveira, and Jim Nasby.
2015-04-30 21:02:14 +02:00
|
|
|
/* check read-only transaction and parallel mode */
|
2012-12-18 02:15:32 +01:00
|
|
|
if (XactReadOnly && !rel->rd_islocaltemp)
|
2011-02-16 03:19:11 +01:00
|
|
|
PreventCommandIfReadOnly("COPY FROM");
|
|
|
|
|
2016-09-06 18:00:00 +02:00
|
|
|
cstate = BeginCopyFrom(pstate, rel, stmt->filename, stmt->is_program,
|
2017-03-23 13:36:36 +01:00
|
|
|
NULL, stmt->attlist, stmt->options);
|
2019-01-19 23:48:16 +01:00
|
|
|
cstate->whereClause = whereClause;
|
2012-12-29 13:55:37 +01:00
|
|
|
*processed = CopyFrom(cstate); /* copy from file to database */
|
2011-02-16 03:19:11 +01:00
|
|
|
EndCopyFrom(cstate);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2016-09-06 18:00:00 +02:00
|
|
|
cstate = BeginCopyTo(pstate, rel, query, relid,
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
stmt->filename, stmt->is_program,
|
2011-02-16 03:19:11 +01:00
|
|
|
stmt->attlist, stmt->options);
|
2012-12-29 13:55:37 +01:00
|
|
|
*processed = DoCopyTo(cstate); /* copy from database to file */
|
2011-02-16 03:19:11 +01:00
|
|
|
EndCopyTo(cstate);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Close the relation. If reading, we can release the AccessShareLock we
|
|
|
|
* got; if writing, we should hold the lock until end of transaction to
|
|
|
|
* ensure that updates will be committed before lock is released.
|
|
|
|
*/
|
|
|
|
if (rel != NULL)
|
2019-01-21 19:32:19 +01:00
|
|
|
table_close(rel, (is_from ? NoLock : AccessShareLock));
|
2011-02-16 03:19:11 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2011-02-20 20:06:59 +01:00
|
|
|
* Process the statement option list for COPY.
|
2011-02-16 03:19:11 +01:00
|
|
|
*
|
2011-02-20 20:06:59 +01:00
|
|
|
* Scan the options list (a list of DefElem) and transpose the information
|
|
|
|
* into cstate, applying appropriate error checking.
|
2001-08-10 20:57:42 +02:00
|
|
|
*
|
2011-02-20 20:06:59 +01:00
|
|
|
* cstate is assumed to be filled with zeroes initially.
|
2001-08-10 20:57:42 +02:00
|
|
|
*
|
2011-02-20 20:06:59 +01:00
|
|
|
* This is exported so that external users of the COPY API can sanity-check
|
|
|
|
* a list of options. In that usage, cstate should be passed as NULL
|
|
|
|
* (since external users don't know sizeof(CopyStateData)) and the collected
|
|
|
|
* data is just leaked until CurrentMemoryContext is reset.
|
|
|
|
*
|
|
|
|
* Note that additional checking, such as whether column names listed in FORCE
|
|
|
|
* QUOTE actually exist, has to be applied later. This just checks for
|
|
|
|
* self-consistency of the options list.
|
1996-11-02 03:01:48 +01:00
|
|
|
*/
|
2011-02-20 20:06:59 +01:00
|
|
|
void
|
2016-09-06 18:00:00 +02:00
|
|
|
ProcessCopyOptions(ParseState *pstate,
|
|
|
|
CopyState cstate,
|
2011-02-20 20:06:59 +01:00
|
|
|
bool is_from,
|
|
|
|
List *options)
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
2009-09-21 22:10:21 +02:00
|
|
|
bool format_specified = false;
|
2005-08-06 22:41:58 +02:00
|
|
|
ListCell *option;
|
2011-02-16 03:19:11 +01:00
|
|
|
|
2011-02-20 20:06:59 +01:00
|
|
|
/* Support external use for option sanity checking */
|
|
|
|
if (cstate == NULL)
|
|
|
|
cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
|
2011-02-16 03:19:11 +01:00
|
|
|
|
Handle EPIPE more sanely when we close a pipe reading from a program.
Previously, any program launched by COPY TO/FROM PROGRAM inherited the
server's setting of SIGPIPE handling, i.e. SIG_IGN. Hence, if we were
doing COPY FROM PROGRAM and closed the pipe early, the child process
would see EPIPE on its output file and typically would treat that as
a fatal error, in turn causing the COPY to report error. Similarly,
one could get a failure report from a query that didn't read all of
the output from a contrib/file_fdw foreign table that uses file_fdw's
PROGRAM option.
To fix, ensure that child programs inherit SIG_DFL not SIG_IGN processing
of SIGPIPE. This seems like an all-around better situation since if
the called program wants some non-default treatment of SIGPIPE, it would
expect to have to set that up for itself. Then in COPY, if it's COPY
FROM PROGRAM and we stop reading short of detecting EOF, treat a SIGPIPE
exit from the called program as a non-error condition. This still allows
us to report an error for any case where the called program gets SIGPIPE
on some other file descriptor.
As coded, we won't report a SIGPIPE if we stop reading as a result of
seeing an in-band EOF marker (e.g. COPY BINARY EOF marker). It's
somewhat debatable whether we should complain if the called program
continues to transmit data after an EOF marker. However, it seems like
we should avoid throwing error in any questionable cases, especially in a
back-patched fix, and anyway it would take additional code to make such
an error get reported consistently.
Back-patch to v10. We could go further back, since COPY FROM PROGRAM
has been around awhile, but AFAICS the only way to reach this situation
using core or contrib is via file_fdw, which has only supported PROGRAM
sources since v10. The COPY statement per se has no feature whereby
it'd stop reading without having hit EOF or an error already. Therefore,
I don't see any upside to back-patching further that'd outweigh the
risk of complaints about behavioral change.
Per bug #15449 from Eric Cyr.
Patch by me, review by Etsuro Fujita and Kyotaro Horiguchi
Discussion: https://postgr.es/m/15449-1cf737dd5929450e@postgresql.org
2018-11-19 23:02:25 +01:00
|
|
|
cstate->is_copy_from = is_from;
|
|
|
|
|
2011-02-21 06:08:04 +01:00
|
|
|
cstate->file_encoding = -1;
|
|
|
|
|
2002-06-20 18:00:44 +02:00
|
|
|
/* Extract options from the statement node tree */
|
2011-02-16 03:19:11 +01:00
|
|
|
foreach(option, options)
|
2002-06-20 18:00:44 +02:00
|
|
|
{
|
Improve castNode notation by introducing list-extraction-specific variants.
This extends the castNode() notation introduced by commit 5bcab1114 to
provide, in one step, extraction of a list cell's pointer and coercion to
a concrete node type. For example, "lfirst_node(Foo, lc)" is the same
as "castNode(Foo, lfirst(lc))". Almost half of the uses of castNode
that have appeared so far include a list extraction call, so this is
pretty widely useful, and it saves a few more keystrokes compared to the
old way.
As with the previous patch, back-patch the addition of these macros to
pg_list.h, so that the notation will be available when back-patching.
Patch by me, after an idea of Andrew Gierth's.
Discussion: https://postgr.es/m/14197.1491841216@sss.pgh.pa.us
2017-04-10 19:51:29 +02:00
|
|
|
DefElem *defel = lfirst_node(DefElem, option);
|
2002-06-20 18:00:44 +02:00
|
|
|
|
2009-09-21 22:10:21 +02:00
|
|
|
if (strcmp(defel->defname, "format") == 0)
|
2002-06-20 18:00:44 +02:00
|
|
|
{
|
2010-02-26 03:01:40 +01:00
|
|
|
char *fmt = defGetString(defel);
|
2009-09-21 22:10:21 +02:00
|
|
|
|
|
|
|
if (format_specified)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("conflicting or redundant options"),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2009-09-21 22:10:21 +02:00
|
|
|
format_specified = true;
|
|
|
|
if (strcmp(fmt, "text") == 0)
|
2010-02-26 03:01:40 +01:00
|
|
|
/* default format */ ;
|
2009-09-21 22:10:21 +02:00
|
|
|
else if (strcmp(fmt, "csv") == 0)
|
|
|
|
cstate->csv_mode = true;
|
|
|
|
else if (strcmp(fmt, "binary") == 0)
|
|
|
|
cstate->binary = true;
|
|
|
|
else
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("COPY format \"%s\" not recognized", fmt),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2002-06-20 18:00:44 +02:00
|
|
|
}
|
2012-12-01 13:54:20 +01:00
|
|
|
else if (strcmp(defel->defname, "freeze") == 0)
|
|
|
|
{
|
|
|
|
if (cstate->freeze)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("conflicting or redundant options"),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2012-12-01 13:54:20 +01:00
|
|
|
cstate->freeze = defGetBoolean(defel);
|
|
|
|
}
|
2002-06-20 18:00:44 +02:00
|
|
|
else if (strcmp(defel->defname, "delimiter") == 0)
|
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->delim)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("conflicting or redundant options"),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2009-09-21 22:10:21 +02:00
|
|
|
cstate->delim = defGetString(defel);
|
2002-06-20 18:00:44 +02:00
|
|
|
}
|
|
|
|
else if (strcmp(defel->defname, "null") == 0)
|
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->null_print)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("conflicting or redundant options"),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2009-09-21 22:10:21 +02:00
|
|
|
cstate->null_print = defGetString(defel);
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
2005-05-07 04:22:49 +02:00
|
|
|
else if (strcmp(defel->defname, "header") == 0)
|
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->header_line)
|
2005-05-07 04:22:49 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("conflicting or redundant options"),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2009-09-21 22:10:21 +02:00
|
|
|
cstate->header_line = defGetBoolean(defel);
|
2005-05-07 04:22:49 +02:00
|
|
|
}
|
2004-04-19 19:22:31 +02:00
|
|
|
else if (strcmp(defel->defname, "quote") == 0)
|
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->quote)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("conflicting or redundant options"),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2009-09-21 22:10:21 +02:00
|
|
|
cstate->quote = defGetString(defel);
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
|
|
|
else if (strcmp(defel->defname, "escape") == 0)
|
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->escape)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("conflicting or redundant options"),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2009-09-21 22:10:21 +02:00
|
|
|
cstate->escape = defGetString(defel);
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
2004-04-21 02:34:18 +02:00
|
|
|
else if (strcmp(defel->defname, "force_quote") == 0)
|
2004-04-19 19:22:31 +02:00
|
|
|
{
|
2011-02-20 20:06:59 +01:00
|
|
|
if (cstate->force_quote || cstate->force_quote_all)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("conflicting or redundant options"),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2009-07-25 19:04:19 +02:00
|
|
|
if (defel->arg && IsA(defel->arg, A_Star))
|
2011-02-20 20:06:59 +01:00
|
|
|
cstate->force_quote_all = true;
|
2009-09-21 22:10:21 +02:00
|
|
|
else if (defel->arg && IsA(defel->arg, List))
|
2017-02-21 17:33:07 +01:00
|
|
|
cstate->force_quote = castNode(List, defel->arg);
|
2009-09-21 22:10:21 +02:00
|
|
|
else
|
|
|
|
ereport(ERROR,
|
2010-02-26 03:01:40 +01:00
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
|
|
|
errmsg("argument to option \"%s\" must be a list of column names",
|
2016-09-06 18:00:00 +02:00
|
|
|
defel->defname),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
2009-09-21 22:10:21 +02:00
|
|
|
else if (strcmp(defel->defname, "force_not_null") == 0)
|
2004-04-19 19:22:31 +02:00
|
|
|
{
|
2011-02-20 20:06:59 +01:00
|
|
|
if (cstate->force_notnull)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("conflicting or redundant options"),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2009-09-21 22:10:21 +02:00
|
|
|
if (defel->arg && IsA(defel->arg, List))
|
2017-01-27 01:47:03 +01:00
|
|
|
cstate->force_notnull = castNode(List, defel->arg);
|
2009-09-21 22:10:21 +02:00
|
|
|
else
|
|
|
|
ereport(ERROR,
|
2010-02-26 03:01:40 +01:00
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
|
|
|
errmsg("argument to option \"%s\" must be a list of column names",
|
2016-09-06 18:00:00 +02:00
|
|
|
defel->defname),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
2014-03-04 23:31:59 +01:00
|
|
|
else if (strcmp(defel->defname, "force_null") == 0)
|
|
|
|
{
|
|
|
|
if (cstate->force_null)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
errmsg("conflicting or redundant options")));
|
|
|
|
if (defel->arg && IsA(defel->arg, List))
|
2017-01-27 01:47:03 +01:00
|
|
|
cstate->force_null = castNode(List, defel->arg);
|
2014-03-04 23:31:59 +01:00
|
|
|
else
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
|
|
|
errmsg("argument to option \"%s\" must be a list of column names",
|
2016-09-06 18:00:00 +02:00
|
|
|
defel->defname),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2014-03-04 23:31:59 +01:00
|
|
|
}
|
2012-07-12 22:26:59 +02:00
|
|
|
else if (strcmp(defel->defname, "convert_selectively") == 0)
|
|
|
|
{
|
|
|
|
/*
|
2013-05-29 22:58:43 +02:00
|
|
|
* Undocumented, not-accessible-from-SQL option: convert only the
|
|
|
|
* named columns to binary form, storing the rest as NULLs. It's
|
|
|
|
* allowed for the column list to be NIL.
|
2012-07-12 22:26:59 +02:00
|
|
|
*/
|
|
|
|
if (cstate->convert_selectively)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("conflicting or redundant options"),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2012-07-12 22:26:59 +02:00
|
|
|
cstate->convert_selectively = true;
|
|
|
|
if (defel->arg == NULL || IsA(defel->arg, List))
|
2017-01-27 01:47:03 +01:00
|
|
|
cstate->convert_select = castNode(List, defel->arg);
|
2012-07-12 22:26:59 +02:00
|
|
|
else
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
|
|
|
errmsg("argument to option \"%s\" must be a list of column names",
|
2016-09-06 18:00:00 +02:00
|
|
|
defel->defname),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2012-07-12 22:26:59 +02:00
|
|
|
}
|
2011-02-21 06:08:04 +01:00
|
|
|
else if (strcmp(defel->defname, "encoding") == 0)
|
|
|
|
{
|
|
|
|
if (cstate->file_encoding >= 0)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
2016-09-06 18:00:00 +02:00
|
|
|
errmsg("conflicting or redundant options"),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2011-02-21 06:08:04 +01:00
|
|
|
cstate->file_encoding = pg_char_to_encoding(defGetString(defel));
|
|
|
|
if (cstate->file_encoding < 0)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
|
|
|
errmsg("argument to option \"%s\" must be a valid encoding name",
|
2016-09-06 18:00:00 +02:00
|
|
|
defel->defname),
|
|
|
|
parser_errposition(pstate, defel->location)));
|
2011-02-21 06:08:04 +01:00
|
|
|
}
|
2002-06-20 18:00:44 +02:00
|
|
|
else
|
2009-09-21 22:10:21 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
errmsg("option \"%s\" not recognized",
|
2016-09-06 18:00:00 +02:00
|
|
|
defel->defname),
|
2016-09-07 05:55:55 +02:00
|
|
|
parser_errposition(pstate, defel->location)));
|
2002-06-20 18:00:44 +02:00
|
|
|
}
|
|
|
|
|
2009-09-21 22:10:21 +02:00
|
|
|
/*
|
|
|
|
* Check for incompatible options (must do these two before inserting
|
|
|
|
* defaults)
|
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->binary && cstate->delim)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
errmsg("cannot specify DELIMITER in BINARY mode")));
|
2002-06-20 18:00:44 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->binary && cstate->null_print)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_SYNTAX_ERROR),
|
|
|
|
errmsg("cannot specify NULL in BINARY mode")));
|
2002-07-30 18:55:06 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* Set defaults for omitted options */
|
|
|
|
if (!cstate->delim)
|
|
|
|
cstate->delim = cstate->csv_mode ? "," : "\t";
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!cstate->null_print)
|
|
|
|
cstate->null_print = cstate->csv_mode ? "" : "\\N";
|
|
|
|
cstate->null_print_len = strlen(cstate->null_print);
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->csv_mode)
|
2004-04-19 19:22:31 +02:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!cstate->quote)
|
|
|
|
cstate->quote = "\"";
|
|
|
|
if (!cstate->escape)
|
|
|
|
cstate->escape = cstate->quote;
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2009-03-26 20:24:54 +01:00
|
|
|
/* Only single-byte delimiter strings are supported. */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (strlen(cstate->delim) != 1)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("COPY delimiter must be a single one-byte character")));
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2006-02-03 13:41:07 +01:00
|
|
|
/* Disallow end-of-line characters */
|
|
|
|
if (strchr(cstate->delim, '\r') != NULL ||
|
2006-10-04 02:30:14 +02:00
|
|
|
strchr(cstate->delim, '\n') != NULL)
|
2006-02-03 13:41:07 +01:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("COPY delimiter cannot be newline or carriage return")));
|
2006-02-03 13:41:07 +01:00
|
|
|
|
|
|
|
if (strchr(cstate->null_print, '\r') != NULL ||
|
|
|
|
strchr(cstate->null_print, '\n') != NULL)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
2006-10-06 19:14:01 +02:00
|
|
|
errmsg("COPY null representation cannot use newline or carriage return")));
|
2006-02-03 13:41:07 +01:00
|
|
|
|
Disallow digits and lower-case ASCII letters as the delimiter in non-CSV
COPY. We need a restriction here because when the delimiter occurs as a
data character, it is emitted with a backslash, and that will only work
as desired if CopyReadAttributesText() will interpret the backslash sequence
as representing the second character literally. This is currently untrue
for 'b', 'f', 'n', 'r', 't', 'v', 'x', and octal digits. For future-proofing
and simplicity of explanation, it seems best to disallow a-z and 0-9.
We must also disallow dot, since "\." by itself would look like copy EOF.
Note: "\N" is by default the null print string, so N would also cause a
problem, but that is already tested for.
2007-12-27 19:28:58 +01:00
|
|
|
/*
|
|
|
|
* Disallow unsafe delimiter characters in non-CSV mode. We can't allow
|
|
|
|
* backslash because it would be ambiguous. We can't allow the other
|
|
|
|
* cases because data characters matching the delimiter must be
|
|
|
|
* backslashed, and certain backslash combinations are interpreted
|
2009-06-11 16:49:15 +02:00
|
|
|
* non-literally by COPY IN. Disallowing all lower case ASCII letters is
|
|
|
|
* more than strictly necessary, but seems best for consistency and
|
Disallow digits and lower-case ASCII letters as the delimiter in non-CSV
COPY. We need a restriction here because when the delimiter occurs as a
data character, it is emitted with a backslash, and that will only work
as desired if CopyReadAttributesText() will interpret the backslash sequence
as representing the second character literally. This is currently untrue
for 'b', 'f', 'n', 'r', 't', 'v', 'x', and octal digits. For future-proofing
and simplicity of explanation, it seems best to disallow a-z and 0-9.
We must also disallow dot, since "\." by itself would look like copy EOF.
Note: "\N" is by default the null print string, so N would also cause a
problem, but that is already tested for.
2007-12-27 19:28:58 +01:00
|
|
|
* future-proofing. Likewise we disallow all digits though only octal
|
|
|
|
* digits are actually dangerous.
|
|
|
|
*/
|
|
|
|
if (!cstate->csv_mode &&
|
|
|
|
strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789",
|
|
|
|
cstate->delim[0]) != NULL)
|
2006-02-03 13:41:07 +01:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
Disallow digits and lower-case ASCII letters as the delimiter in non-CSV
COPY. We need a restriction here because when the delimiter occurs as a
data character, it is emitted with a backslash, and that will only work
as desired if CopyReadAttributesText() will interpret the backslash sequence
as representing the second character literally. This is currently untrue
for 'b', 'f', 'n', 'r', 't', 'v', 'x', and octal digits. For future-proofing
and simplicity of explanation, it seems best to disallow a-z and 0-9.
We must also disallow dot, since "\." by itself would look like copy EOF.
Note: "\N" is by default the null print string, so N would also cause a
problem, but that is already tested for.
2007-12-27 19:28:58 +01:00
|
|
|
errmsg("COPY delimiter cannot be \"%s\"", cstate->delim)));
|
2006-02-03 13:41:07 +01:00
|
|
|
|
2005-10-15 04:49:52 +02:00
|
|
|
/* Check header */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!cstate->csv_mode && cstate->header_line)
|
2005-05-07 04:22:49 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
|
|
|
errmsg("COPY HEADER available only in CSV mode")));
|
|
|
|
|
2005-05-06 04:56:42 +02:00
|
|
|
/* Check quote */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!cstate->csv_mode && cstate->quote != NULL)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
|
|
|
errmsg("COPY quote available only in CSV mode")));
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->csv_mode && strlen(cstate->quote) != 1)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
2009-03-26 20:24:54 +01:00
|
|
|
errmsg("COPY quote must be a single one-byte character")));
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2007-12-30 15:46:52 +01:00
|
|
|
if (cstate->csv_mode && cstate->delim[0] == cstate->quote[0])
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
|
|
|
|
errmsg("COPY delimiter and quote must be different")));
|
|
|
|
|
2005-05-06 04:56:42 +02:00
|
|
|
/* Check escape */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!cstate->csv_mode && cstate->escape != NULL)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
|
|
|
errmsg("COPY escape available only in CSV mode")));
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->csv_mode && strlen(cstate->escape) != 1)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
2009-03-26 20:24:54 +01:00
|
|
|
errmsg("COPY escape must be a single one-byte character")));
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-05-06 04:56:42 +02:00
|
|
|
/* Check force_quote */
|
2011-02-20 20:06:59 +01:00
|
|
|
if (!cstate->csv_mode && (cstate->force_quote || cstate->force_quote_all))
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
2004-04-21 02:34:18 +02:00
|
|
|
errmsg("COPY force quote available only in CSV mode")));
|
2011-02-20 20:06:59 +01:00
|
|
|
if ((cstate->force_quote || cstate->force_quote_all) && is_from)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
2005-10-15 04:49:52 +02:00
|
|
|
errmsg("COPY force quote only available using COPY TO")));
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-05-06 04:56:42 +02:00
|
|
|
/* Check force_notnull */
|
2011-02-20 20:06:59 +01:00
|
|
|
if (!cstate->csv_mode && cstate->force_notnull != NIL)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
2005-10-15 04:49:52 +02:00
|
|
|
errmsg("COPY force not null available only in CSV mode")));
|
2011-02-20 20:06:59 +01:00
|
|
|
if (cstate->force_notnull != NIL && !is_from)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("COPY force not null only available using COPY FROM")));
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2014-03-04 23:31:59 +01:00
|
|
|
/* Check force_null */
|
|
|
|
if (!cstate->csv_mode && cstate->force_null != NIL)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
|
|
|
errmsg("COPY force null available only in CSV mode")));
|
|
|
|
|
|
|
|
if (cstate->force_null != NIL && !is_from)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
2014-05-06 18:12:18 +02:00
|
|
|
errmsg("COPY force null only available using COPY FROM")));
|
2014-03-04 23:31:59 +01:00
|
|
|
|
2005-05-06 04:56:42 +02:00
|
|
|
/* Don't allow the delimiter to appear in the null string. */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (strchr(cstate->null_print, cstate->delim[0]) != NULL)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("COPY delimiter must not appear in the NULL specification")));
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* Don't allow the CSV quote char to appear in the null string. */
|
|
|
|
if (cstate->csv_mode &&
|
|
|
|
strchr(cstate->null_print, cstate->quote[0]) != NULL)
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
|
|
|
errmsg("CSV quote character must not appear in the NULL specification")));
|
2011-02-20 20:06:59 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Common setup routines used by BeginCopyFrom and BeginCopyTo.
|
|
|
|
*
|
|
|
|
* Iff <binary>, unload or reload in the binary format, as opposed to the
|
|
|
|
* more wasteful but more robust and portable text format.
|
|
|
|
*
|
|
|
|
* Iff <oids>, unload or reload the format that includes OID information.
|
|
|
|
* On input, we accept OIDs whether or not the table has an OID column,
|
|
|
|
* but silently drop them if it does not. On output, we report an error
|
|
|
|
* if the user asks for OIDs in a table that has none (not providing an
|
|
|
|
* OID column might seem friendlier, but could seriously confuse programs).
|
|
|
|
*
|
|
|
|
* If in the text format, delimit columns with delimiter <delim> and print
|
|
|
|
* NULL values as <null_print>.
|
|
|
|
*/
|
|
|
|
static CopyState
|
2016-09-06 18:00:00 +02:00
|
|
|
BeginCopy(ParseState *pstate,
|
|
|
|
bool is_from,
|
2011-02-20 20:06:59 +01:00
|
|
|
Relation rel,
|
Change representation of statement lists, and add statement location info.
This patch makes several changes that improve the consistency of
representation of lists of statements. It's always been the case
that the output of parse analysis is a list of Query nodes, whatever
the types of the individual statements in the list. This patch brings
similar consistency to the outputs of raw parsing and planning steps:
* The output of raw parsing is now always a list of RawStmt nodes;
the statement-type-dependent nodes are one level down from that.
* The output of pg_plan_queries() is now always a list of PlannedStmt
nodes, even for utility statements. In the case of a utility statement,
"planning" just consists of wrapping a CMD_UTILITY PlannedStmt around
the utility node. This list representation is now used in Portal and
CachedPlan plan lists, replacing the former convention of intermixing
PlannedStmts with bare utility-statement nodes.
Now, every list of statements has a consistent head-node type depending
on how far along it is in processing. This allows changing many places
that formerly used generic "Node *" pointers to use a more specific
pointer type, thus reducing the number of IsA() tests and casts needed,
as well as improving code clarity.
Also, the post-parse-analysis representation of DECLARE CURSOR is changed
so that it looks more like EXPLAIN, PREPARE, etc. That is, the contained
SELECT remains a child of the DeclareCursorStmt rather than getting flipped
around to be the other way. It's now true for both Query and PlannedStmt
that utilityStmt is non-null if and only if commandType is CMD_UTILITY.
That allows simplifying a lot of places that were testing both fields.
(I think some of those were just defensive programming, but in many places,
it was actually necessary to avoid confusing DECLARE CURSOR with SELECT.)
Because PlannedStmt carries a canSetTag field, we're also able to get rid
of some ad-hoc rules about how to reconstruct canSetTag for a bare utility
statement; specifically, the assumption that a utility is canSetTag if and
only if it's the only one in its list. While I see no near-term need for
relaxing that restriction, it's nice to get rid of the ad-hocery.
The API of ProcessUtility() is changed so that what it's passed is the
wrapper PlannedStmt not just the bare utility statement. This will affect
all users of ProcessUtility_hook, but the changes are pretty trivial; see
the affected contrib modules for examples of the minimum change needed.
(Most compilers should give pointer-type-mismatch warnings for uncorrected
code.)
There's also a change in the API of ExplainOneQuery_hook, to pass through
cursorOptions instead of expecting hook functions to know what to pick.
This is needed because of the DECLARE CURSOR changes, but really should
have been done in 9.6; it's unlikely that any extant hook functions
know about using CURSOR_OPT_PARALLEL_OK.
Finally, teach gram.y to save statement boundary locations in RawStmt
nodes, and pass those through to Query and PlannedStmt nodes. This allows
more intelligent handling of cases where a source query string contains
multiple statements. This patch doesn't actually do anything with the
information, but a follow-on patch will. (Passing this information through
cleanly is the true motivation for these changes; while I think this is all
good cleanup, it's unlikely we'd have bothered without this end goal.)
catversion bump because addition of location fields to struct Query
affects stored rules.
This patch is by me, but it owes a good deal to Fabien Coelho who did
a lot of preliminary work on the problem, and also reviewed the patch.
Discussion: https://postgr.es/m/alpine.DEB.2.20.1612200926310.29821@lancre
2017-01-14 22:02:35 +01:00
|
|
|
RawStmt *raw_query,
|
|
|
|
Oid queryRelId,
|
2011-02-20 20:06:59 +01:00
|
|
|
List *attnamelist,
|
|
|
|
List *options)
|
|
|
|
{
|
|
|
|
CopyState cstate;
|
|
|
|
TupleDesc tupDesc;
|
|
|
|
int num_phys_attrs;
|
|
|
|
MemoryContext oldcontext;
|
|
|
|
|
|
|
|
/* Allocate workspace and zero all fields */
|
|
|
|
cstate = (CopyStateData *) palloc0(sizeof(CopyStateData));
|
|
|
|
|
|
|
|
/*
|
2011-04-10 17:42:00 +02:00
|
|
|
* We allocate everything used by a cstate in a new memory context. This
|
|
|
|
* avoids memory leaks during repeated use of COPY in a query.
|
2011-02-20 20:06:59 +01:00
|
|
|
*/
|
|
|
|
cstate->copycontext = AllocSetContextCreate(CurrentMemoryContext,
|
|
|
|
"COPY",
|
Add macros to make AllocSetContextCreate() calls simpler and safer.
I found that half a dozen (nearly 5%) of our AllocSetContextCreate calls
had typos in the context-sizing parameters. While none of these led to
especially significant problems, they did create minor inefficiencies,
and it's now clear that expecting people to copy-and-paste those calls
accurately is not a great idea. Let's reduce the risk of future errors
by introducing single macros that encapsulate the common use-cases.
Three such macros are enough to cover all but two special-purpose contexts;
those two calls can be left as-is, I think.
While this patch doesn't in itself improve matters for third-party
extensions, it doesn't break anything for them either, and they can
gradually adopt the simplified notation over time.
In passing, change TopMemoryContext to use the default allocation
parameters. Formerly it could only be extended 8K at a time. That was
probably reasonable when this code was written; but nowadays we create
many more contexts than we did then, so that it's not unusual to have a
couple hundred K in TopMemoryContext, even without considering various
dubious code that sticks other things there. There seems no good reason
not to let it use growing blocks like most other contexts.
Back-patch to 9.6, mostly because that's still close enough to HEAD that
it's easy to do so, and keeping the branches in sync can be expected to
avoid some future back-patching pain. The bugs fixed by these changes
don't seem to be significant enough to justify fixing them further back.
Discussion: <21072.1472321324@sss.pgh.pa.us>
2016-08-27 23:50:38 +02:00
|
|
|
ALLOCSET_DEFAULT_SIZES);
|
2011-02-20 20:06:59 +01:00
|
|
|
|
|
|
|
oldcontext = MemoryContextSwitchTo(cstate->copycontext);
|
|
|
|
|
|
|
|
/* Extract options from the statement node tree */
|
2016-09-06 18:00:00 +02:00
|
|
|
ProcessCopyOptions(pstate, cstate, is_from, options);
|
2002-09-04 22:31:48 +02:00
|
|
|
|
2011-02-20 20:06:59 +01:00
|
|
|
/* Process the source/target relation or query */
|
2011-02-16 03:19:11 +01:00
|
|
|
if (rel)
|
2006-08-31 01:34:22 +02:00
|
|
|
{
|
2011-02-16 03:19:11 +01:00
|
|
|
Assert(!raw_query);
|
2006-08-31 01:34:22 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
cstate->rel = rel;
|
2006-08-31 01:34:22 +02:00
|
|
|
|
2009-02-06 22:15:12 +01:00
|
|
|
tupDesc = RelationGetDescr(cstate->rel);
|
2006-08-31 01:34:22 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
List *rewritten;
|
2007-03-13 01:33:44 +01:00
|
|
|
Query *query;
|
2007-02-20 18:32:18 +01:00
|
|
|
PlannedStmt *plan;
|
2006-08-31 01:34:22 +02:00
|
|
|
DestReceiver *dest;
|
|
|
|
|
|
|
|
Assert(!is_from);
|
|
|
|
cstate->rel = NULL;
|
|
|
|
|
|
|
|
/*
|
2014-05-06 18:12:18 +02:00
|
|
|
* Run parse analysis and rewrite. Note this also acquires sufficient
|
2007-03-13 01:33:44 +01:00
|
|
|
* locks on the source table(s).
|
2006-08-31 01:34:22 +02:00
|
|
|
*
|
2007-03-13 01:33:44 +01:00
|
|
|
* Because the parser and planner tend to scribble on their input, we
|
|
|
|
* make a preliminary copy of the source querytree. This prevents
|
2006-08-31 01:34:22 +02:00
|
|
|
* problems in the case that the COPY is in a portal or plpgsql
|
|
|
|
* function and is executed repeatedly. (See also the same hack in
|
2007-03-13 01:33:44 +01:00
|
|
|
* DECLARE CURSOR and PREPARE.) XXX FIXME someday.
|
2006-08-31 01:34:22 +02:00
|
|
|
*/
|
2017-03-09 21:18:59 +01:00
|
|
|
rewritten = pg_analyze_and_rewrite(copyObject(raw_query),
|
2017-04-01 06:17:18 +02:00
|
|
|
pstate->p_sourcetext, NULL, 0,
|
|
|
|
NULL);
|
2006-08-31 01:34:22 +02:00
|
|
|
|
2015-11-27 17:11:22 +01:00
|
|
|
/* check that we got back something we can work with */
|
|
|
|
if (rewritten == NIL)
|
|
|
|
{
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("DO INSTEAD NOTHING rules are not supported for COPY")));
|
2015-11-27 17:11:22 +01:00
|
|
|
}
|
|
|
|
else if (list_length(rewritten) > 1)
|
|
|
|
{
|
2016-06-10 00:02:36 +02:00
|
|
|
ListCell *lc;
|
2015-11-27 17:11:22 +01:00
|
|
|
|
|
|
|
/* examine queries to determine which error message to issue */
|
|
|
|
foreach(lc, rewritten)
|
|
|
|
{
|
Improve castNode notation by introducing list-extraction-specific variants.
This extends the castNode() notation introduced by commit 5bcab1114 to
provide, in one step, extraction of a list cell's pointer and coercion to
a concrete node type. For example, "lfirst_node(Foo, lc)" is the same
as "castNode(Foo, lfirst(lc))". Almost half of the uses of castNode
that have appeared so far include a list extraction call, so this is
pretty widely useful, and it saves a few more keystrokes compared to the
old way.
As with the previous patch, back-patch the addition of these macros to
pg_list.h, so that the notation will be available when back-patching.
Patch by me, after an idea of Andrew Gierth's.
Discussion: https://postgr.es/m/14197.1491841216@sss.pgh.pa.us
2017-04-10 19:51:29 +02:00
|
|
|
Query *q = lfirst_node(Query, lc);
|
2015-11-27 17:11:22 +01:00
|
|
|
|
|
|
|
if (q->querySource == QSRC_QUAL_INSTEAD_RULE)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
|
|
|
errmsg("conditional DO INSTEAD rules are not supported for COPY")));
|
|
|
|
if (q->querySource == QSRC_NON_INSTEAD_RULE)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("DO ALSO rules are not supported for the COPY")));
|
2015-11-27 17:11:22 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
|
|
|
errmsg("multi-statement DO INSTEAD rules are not supported for COPY")));
|
|
|
|
}
|
2006-08-31 01:34:22 +02:00
|
|
|
|
Improve castNode notation by introducing list-extraction-specific variants.
This extends the castNode() notation introduced by commit 5bcab1114 to
provide, in one step, extraction of a list cell's pointer and coercion to
a concrete node type. For example, "lfirst_node(Foo, lc)" is the same
as "castNode(Foo, lfirst(lc))". Almost half of the uses of castNode
that have appeared so far include a list extraction call, so this is
pretty widely useful, and it saves a few more keystrokes compared to the
old way.
As with the previous patch, back-patch the addition of these macros to
pg_list.h, so that the notation will be available when back-patching.
Patch by me, after an idea of Andrew Gierth's.
Discussion: https://postgr.es/m/14197.1491841216@sss.pgh.pa.us
2017-04-10 19:51:29 +02:00
|
|
|
query = linitial_node(Query, rewritten);
|
2006-08-31 01:34:22 +02:00
|
|
|
|
Restructure SELECT INTO's parsetree representation into CreateTableAsStmt.
Making this operation look like a utility statement seems generally a good
idea, and particularly so in light of the desire to provide command
triggers for utility statements. The original choice of representing it as
SELECT with an IntoClause appendage had metastasized into rather a lot of
places, unfortunately, so that this patch is a great deal more complicated
than one might at first expect.
In particular, keeping EXPLAIN working for SELECT INTO and CREATE TABLE AS
subcommands required restructuring some EXPLAIN-related APIs. Add-on code
that calls ExplainOnePlan or ExplainOneUtility, or uses
ExplainOneQuery_hook, will need adjustment.
Also, the cases PREPARE ... SELECT INTO and CREATE RULE ... SELECT INTO,
which formerly were accepted though undocumented, are no longer accepted.
The PREPARE case can be replaced with use of CREATE TABLE AS EXECUTE.
The CREATE RULE case doesn't seem to have much real-world use (since the
rule would work only once before failing with "table already exists"),
so we'll not bother with that one.
Both SELECT INTO and CREATE TABLE AS still return a command tag of
"SELECT nnnn". There was some discussion of returning "CREATE TABLE nnnn",
but for the moment backwards compatibility wins the day.
Andres Freund and Tom Lane
2012-03-20 02:37:19 +01:00
|
|
|
/* The grammar allows SELECT INTO, but we don't support that */
|
|
|
|
if (query->utilityStmt != NULL &&
|
|
|
|
IsA(query->utilityStmt, CreateTableAsStmt))
|
2007-03-13 01:33:44 +01:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
|
|
|
errmsg("COPY (SELECT INTO) is not supported")));
|
|
|
|
|
Restructure SELECT INTO's parsetree representation into CreateTableAsStmt.
Making this operation look like a utility statement seems generally a good
idea, and particularly so in light of the desire to provide command
triggers for utility statements. The original choice of representing it as
SELECT with an IntoClause appendage had metastasized into rather a lot of
places, unfortunately, so that this patch is a great deal more complicated
than one might at first expect.
In particular, keeping EXPLAIN working for SELECT INTO and CREATE TABLE AS
subcommands required restructuring some EXPLAIN-related APIs. Add-on code
that calls ExplainOnePlan or ExplainOneUtility, or uses
ExplainOneQuery_hook, will need adjustment.
Also, the cases PREPARE ... SELECT INTO and CREATE RULE ... SELECT INTO,
which formerly were accepted though undocumented, are no longer accepted.
The PREPARE case can be replaced with use of CREATE TABLE AS EXECUTE.
The CREATE RULE case doesn't seem to have much real-world use (since the
rule would work only once before failing with "table already exists"),
so we'll not bother with that one.
Both SELECT INTO and CREATE TABLE AS still return a command tag of
"SELECT nnnn". There was some discussion of returning "CREATE TABLE nnnn",
but for the moment backwards compatibility wins the day.
Andres Freund and Tom Lane
2012-03-20 02:37:19 +01:00
|
|
|
Assert(query->utilityStmt == NULL);
|
|
|
|
|
2015-11-27 17:11:22 +01:00
|
|
|
/*
|
|
|
|
* Similarly the grammar doesn't enforce the presence of a RETURNING
|
|
|
|
* clause, but this is required here.
|
|
|
|
*/
|
|
|
|
if (query->commandType != CMD_SELECT &&
|
|
|
|
query->returningList == NIL)
|
|
|
|
{
|
|
|
|
Assert(query->commandType == CMD_INSERT ||
|
|
|
|
query->commandType == CMD_UPDATE ||
|
|
|
|
query->commandType == CMD_DELETE);
|
|
|
|
|
|
|
|
ereport(ERROR,
|
2016-06-10 00:02:36 +02:00
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
|
|
|
errmsg("COPY query must have a RETURNING clause")));
|
2015-11-27 17:11:22 +01:00
|
|
|
}
|
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
/* plan the query */
|
2020-03-30 06:51:05 +02:00
|
|
|
plan = pg_plan_query(query, pstate->p_sourcetext,
|
|
|
|
CURSOR_OPT_PARALLEL_OK, NULL);
|
2006-08-31 01:34:22 +02:00
|
|
|
|
Rename pg_rowsecurity -> pg_policy and other fixes
As pointed out by Robert, we should really have named pg_rowsecurity
pg_policy, as the objects stored in that catalog are policies. This
patch fixes that and updates the column names to start with 'pol' to
match the new catalog name.
The security consideration for COPY with row level security, also
pointed out by Robert, has also been addressed by remembering and
re-checking the OID of the relation initially referenced during COPY
processing, to make sure it hasn't changed under us by the time we
finish planning out the query which has been built.
Robert and Alvaro also commented on missing OCLASS and OBJECT entries
for POLICY (formerly ROWSECURITY or POLICY, depending) in various
places. This patch fixes that too, which also happens to add the
ability to COMMENT on policies.
In passing, attempt to improve the consistency of messages, comments,
and documentation as well. This removes various incarnations of
'row-security', 'row-level security', 'Row-security', etc, in favor
of 'policy', 'row level security' or 'row_security' as appropriate.
Happy Thanksgiving!
2014-11-27 07:06:36 +01:00
|
|
|
/*
|
2015-07-27 22:48:26 +02:00
|
|
|
* With row level security and a user using "COPY relation TO", we
|
|
|
|
* have to convert the "COPY relation TO" to a query-based COPY (eg:
|
|
|
|
* "COPY (SELECT * FROM relation) TO"), to allow the rewriter to add
|
|
|
|
* in any RLS clauses.
|
|
|
|
*
|
|
|
|
* When this happens, we are passed in the relid of the originally
|
2015-08-03 05:49:19 +02:00
|
|
|
* found relation (which we have locked). As the planner will look up
|
|
|
|
* the relation again, we double-check here to make sure it found the
|
|
|
|
* same one that we have locked.
|
Rename pg_rowsecurity -> pg_policy and other fixes
As pointed out by Robert, we should really have named pg_rowsecurity
pg_policy, as the objects stored in that catalog are policies. This
patch fixes that and updates the column names to start with 'pol' to
match the new catalog name.
The security consideration for COPY with row level security, also
pointed out by Robert, has also been addressed by remembering and
re-checking the OID of the relation initially referenced during COPY
processing, to make sure it hasn't changed under us by the time we
finish planning out the query which has been built.
Robert and Alvaro also commented on missing OCLASS and OBJECT entries
for POLICY (formerly ROWSECURITY or POLICY, depending) in various
places. This patch fixes that too, which also happens to add the
ability to COMMENT on policies.
In passing, attempt to improve the consistency of messages, comments,
and documentation as well. This removes various incarnations of
'row-security', 'row-level security', 'Row-security', etc, in favor
of 'policy', 'row level security' or 'row_security' as appropriate.
Happy Thanksgiving!
2014-11-27 07:06:36 +01:00
|
|
|
*/
|
|
|
|
if (queryRelId != InvalidOid)
|
|
|
|
{
|
|
|
|
/*
|
2015-07-27 22:48:26 +02:00
|
|
|
* Note that with RLS involved there may be multiple relations,
|
|
|
|
* and while the one we need is almost certainly first, we don't
|
|
|
|
* make any guarantees of that in the planner, so check the whole
|
|
|
|
* list and make sure we find the original relation.
|
Rename pg_rowsecurity -> pg_policy and other fixes
As pointed out by Robert, we should really have named pg_rowsecurity
pg_policy, as the objects stored in that catalog are policies. This
patch fixes that and updates the column names to start with 'pol' to
match the new catalog name.
The security consideration for COPY with row level security, also
pointed out by Robert, has also been addressed by remembering and
re-checking the OID of the relation initially referenced during COPY
processing, to make sure it hasn't changed under us by the time we
finish planning out the query which has been built.
Robert and Alvaro also commented on missing OCLASS and OBJECT entries
for POLICY (formerly ROWSECURITY or POLICY, depending) in various
places. This patch fixes that too, which also happens to add the
ability to COMMENT on policies.
In passing, attempt to improve the consistency of messages, comments,
and documentation as well. This removes various incarnations of
'row-security', 'row-level security', 'Row-security', etc, in favor
of 'policy', 'row level security' or 'row_security' as appropriate.
Happy Thanksgiving!
2014-11-27 07:06:36 +01:00
|
|
|
*/
|
2015-07-27 22:48:26 +02:00
|
|
|
if (!list_member_oid(plan->relationOids, queryRelId))
|
Rename pg_rowsecurity -> pg_policy and other fixes
As pointed out by Robert, we should really have named pg_rowsecurity
pg_policy, as the objects stored in that catalog are policies. This
patch fixes that and updates the column names to start with 'pol' to
match the new catalog name.
The security consideration for COPY with row level security, also
pointed out by Robert, has also been addressed by remembering and
re-checking the OID of the relation initially referenced during COPY
processing, to make sure it hasn't changed under us by the time we
finish planning out the query which has been built.
Robert and Alvaro also commented on missing OCLASS and OBJECT entries
for POLICY (formerly ROWSECURITY or POLICY, depending) in various
places. This patch fixes that too, which also happens to add the
ability to COMMENT on policies.
In passing, attempt to improve the consistency of messages, comments,
and documentation as well. This removes various incarnations of
'row-security', 'row-level security', 'Row-security', etc, in favor
of 'policy', 'row level security' or 'row_security' as appropriate.
Happy Thanksgiving!
2014-11-27 07:06:36 +01:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("relation referenced by COPY statement has changed")));
|
Rename pg_rowsecurity -> pg_policy and other fixes
As pointed out by Robert, we should really have named pg_rowsecurity
pg_policy, as the objects stored in that catalog are policies. This
patch fixes that and updates the column names to start with 'pol' to
match the new catalog name.
The security consideration for COPY with row level security, also
pointed out by Robert, has also been addressed by remembering and
re-checking the OID of the relation initially referenced during COPY
processing, to make sure it hasn't changed under us by the time we
finish planning out the query which has been built.
Robert and Alvaro also commented on missing OCLASS and OBJECT entries
for POLICY (formerly ROWSECURITY or POLICY, depending) in various
places. This patch fixes that too, which also happens to add the
ability to COMMENT on policies.
In passing, attempt to improve the consistency of messages, comments,
and documentation as well. This removes various incarnations of
'row-security', 'row-level security', 'Row-security', etc, in favor
of 'policy', 'row level security' or 'row_security' as appropriate.
Happy Thanksgiving!
2014-11-27 07:06:36 +01:00
|
|
|
}
|
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
/*
|
2008-05-12 22:02:02 +02:00
|
|
|
* Use a snapshot with an updated command ID to ensure this query sees
|
|
|
|
* results of any previously executed queries.
|
2006-08-31 01:34:22 +02:00
|
|
|
*/
|
2011-03-01 05:27:18 +01:00
|
|
|
PushCopiedSnapshot(GetActiveSnapshot());
|
|
|
|
UpdateActiveSnapshotCommandId();
|
2006-08-31 01:34:22 +02:00
|
|
|
|
|
|
|
/* Create dest receiver for COPY OUT */
|
2008-11-30 21:51:25 +01:00
|
|
|
dest = CreateDestReceiver(DestCopyOut);
|
2006-08-31 01:34:22 +02:00
|
|
|
((DR_copy *) dest)->cstate = cstate;
|
|
|
|
|
|
|
|
/* Create a QueryDesc requesting no output */
|
2016-09-06 18:00:00 +02:00
|
|
|
cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext,
|
2009-01-02 21:42:00 +01:00
|
|
|
GetActiveSnapshot(),
|
2008-05-12 22:02:02 +02:00
|
|
|
InvalidSnapshot,
|
2017-04-01 06:17:18 +02:00
|
|
|
dest, NULL, NULL, 0);
|
2006-08-31 01:34:22 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Call ExecutorStart to prepare the plan for execution.
|
|
|
|
*
|
|
|
|
* ExecutorStart computes a result tupdesc for us
|
|
|
|
*/
|
|
|
|
ExecutorStart(cstate->queryDesc, 0);
|
|
|
|
|
|
|
|
tupDesc = cstate->queryDesc->tupDesc;
|
|
|
|
}
|
2002-07-30 18:55:06 +02:00
|
|
|
|
2005-05-06 04:56:42 +02:00
|
|
|
/* Generate or convert list of attributes to process */
|
2006-08-31 01:34:22 +02:00
|
|
|
cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist);
|
|
|
|
|
|
|
|
num_phys_attrs = tupDesc->natts;
|
2002-08-02 20:15:10 +02:00
|
|
|
|
2015-11-05 03:01:26 +01:00
|
|
|
/* Convert FORCE_QUOTE name list to per-column flags, check validity */
|
2006-08-31 01:34:22 +02:00
|
|
|
cstate->force_quote_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
|
2011-02-20 20:06:59 +01:00
|
|
|
if (cstate->force_quote_all)
|
2009-07-25 02:07:14 +02:00
|
|
|
{
|
2010-02-26 03:01:40 +01:00
|
|
|
int i;
|
2009-07-25 02:07:14 +02:00
|
|
|
|
|
|
|
for (i = 0; i < num_phys_attrs; i++)
|
|
|
|
cstate->force_quote_flags[i] = true;
|
|
|
|
}
|
2011-02-20 20:06:59 +01:00
|
|
|
else if (cstate->force_quote)
|
2004-04-19 19:22:31 +02:00
|
|
|
{
|
2006-08-31 01:34:22 +02:00
|
|
|
List *attnums;
|
2004-05-26 06:41:50 +02:00
|
|
|
ListCell *cur;
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2011-02-20 20:06:59 +01:00
|
|
|
attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_quote);
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
foreach(cur, attnums)
|
2004-04-19 19:22:31 +02:00
|
|
|
{
|
2004-05-26 06:41:50 +02:00
|
|
|
int attnum = lfirst_int(cur);
|
2017-08-20 20:19:07 +02:00
|
|
|
Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!list_member_int(cstate->attnumlist, attnum))
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("FORCE_QUOTE column \"%s\" not referenced by COPY",
|
2017-08-20 20:19:07 +02:00
|
|
|
NameStr(attr->attname))));
|
2006-08-31 01:34:22 +02:00
|
|
|
cstate->force_quote_flags[attnum - 1] = true;
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
|
|
|
}
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2015-11-05 03:01:26 +01:00
|
|
|
/* Convert FORCE_NOT_NULL name list to per-column flags, check validity */
|
2006-08-31 01:34:22 +02:00
|
|
|
cstate->force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
|
2011-02-20 20:06:59 +01:00
|
|
|
if (cstate->force_notnull)
|
2004-04-19 19:22:31 +02:00
|
|
|
{
|
2006-08-31 01:34:22 +02:00
|
|
|
List *attnums;
|
2005-08-06 22:41:58 +02:00
|
|
|
ListCell *cur;
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2011-02-20 20:06:59 +01:00
|
|
|
attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_notnull);
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
foreach(cur, attnums)
|
2004-04-19 19:22:31 +02:00
|
|
|
{
|
2004-05-26 06:41:50 +02:00
|
|
|
int attnum = lfirst_int(cur);
|
2017-08-20 20:19:07 +02:00
|
|
|
Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!list_member_int(cstate->attnumlist, attnum))
|
2004-04-19 19:22:31 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("FORCE_NOT_NULL column \"%s\" not referenced by COPY",
|
2017-08-20 20:19:07 +02:00
|
|
|
NameStr(attr->attname))));
|
2006-08-31 01:34:22 +02:00
|
|
|
cstate->force_notnull_flags[attnum - 1] = true;
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
|
|
|
}
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2015-11-05 03:01:26 +01:00
|
|
|
/* Convert FORCE_NULL name list to per-column flags, check validity */
|
2014-03-04 23:31:59 +01:00
|
|
|
cstate->force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
|
|
|
|
if (cstate->force_null)
|
|
|
|
{
|
|
|
|
List *attnums;
|
|
|
|
ListCell *cur;
|
|
|
|
|
|
|
|
attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->force_null);
|
|
|
|
|
|
|
|
foreach(cur, attnums)
|
|
|
|
{
|
|
|
|
int attnum = lfirst_int(cur);
|
2017-08-20 20:19:07 +02:00
|
|
|
Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
|
2014-03-04 23:31:59 +01:00
|
|
|
|
|
|
|
if (!list_member_int(cstate->attnumlist, attnum))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("FORCE_NULL column \"%s\" not referenced by COPY",
|
2017-08-20 20:19:07 +02:00
|
|
|
NameStr(attr->attname))));
|
2014-03-04 23:31:59 +01:00
|
|
|
cstate->force_null_flags[attnum - 1] = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-07-12 22:26:59 +02:00
|
|
|
/* Convert convert_selectively name list to per-column flags */
|
|
|
|
if (cstate->convert_selectively)
|
|
|
|
{
|
|
|
|
List *attnums;
|
|
|
|
ListCell *cur;
|
|
|
|
|
|
|
|
cstate->convert_select_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool));
|
|
|
|
|
|
|
|
attnums = CopyGetAttnums(tupDesc, cstate->rel, cstate->convert_select);
|
|
|
|
|
|
|
|
foreach(cur, attnums)
|
|
|
|
{
|
|
|
|
int attnum = lfirst_int(cur);
|
2017-08-20 20:19:07 +02:00
|
|
|
Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
|
2012-07-12 22:26:59 +02:00
|
|
|
|
|
|
|
if (!list_member_int(cstate->attnumlist, attnum))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
|
|
|
|
errmsg_internal("selected column \"%s\" not referenced by COPY",
|
2017-08-20 20:19:07 +02:00
|
|
|
NameStr(attr->attname))));
|
2012-07-12 22:26:59 +02:00
|
|
|
cstate->convert_select_flags[attnum - 1] = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-02-21 06:08:04 +01:00
|
|
|
/* Use client encoding when ENCODING option is not specified. */
|
|
|
|
if (cstate->file_encoding < 0)
|
|
|
|
cstate->file_encoding = pg_get_client_encoding();
|
|
|
|
|
2006-05-21 22:05:21 +02:00
|
|
|
/*
|
2011-04-10 17:42:00 +02:00
|
|
|
* Set up encoding conversion info. Even if the file and server encodings
|
|
|
|
* are the same, we must apply pg_any_to_server() to validate data in
|
|
|
|
* multibyte encodings.
|
2006-05-21 22:05:21 +02:00
|
|
|
*/
|
|
|
|
cstate->need_transcoding =
|
2011-02-21 06:08:04 +01:00
|
|
|
(cstate->file_encoding != GetDatabaseEncoding() ||
|
2006-05-21 22:05:21 +02:00
|
|
|
pg_database_encoding_max_length() > 1);
|
2005-12-27 19:10:48 +01:00
|
|
|
/* See Multibyte encoding comment above */
|
2011-02-21 06:08:04 +01:00
|
|
|
cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
|
2000-01-16 22:37:50 +01:00
|
|
|
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
cstate->copy_dest = COPY_FILE; /* default */
|
2003-04-19 02:02:30 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
MemoryContextSwitchTo(oldcontext);
|
2002-02-24 03:33:33 +01:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
return cstate;
|
|
|
|
}
|
2002-02-24 03:33:33 +01:00
|
|
|
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
/*
|
|
|
|
* Closes the pipe to an external program, checking the pclose() return code.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
ClosePipeToProgram(CopyState cstate)
|
|
|
|
{
|
2013-05-29 22:58:43 +02:00
|
|
|
int pclose_rc;
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
|
|
|
|
Assert(cstate->is_program);
|
|
|
|
|
|
|
|
pclose_rc = ClosePipeStream(cstate->copy_file);
|
|
|
|
if (pclose_rc == -1)
|
|
|
|
ereport(ERROR,
|
2015-08-03 05:49:19 +02:00
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not close pipe to external command: %m")));
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
else if (pclose_rc != 0)
|
Handle EPIPE more sanely when we close a pipe reading from a program.
Previously, any program launched by COPY TO/FROM PROGRAM inherited the
server's setting of SIGPIPE handling, i.e. SIG_IGN. Hence, if we were
doing COPY FROM PROGRAM and closed the pipe early, the child process
would see EPIPE on its output file and typically would treat that as
a fatal error, in turn causing the COPY to report error. Similarly,
one could get a failure report from a query that didn't read all of
the output from a contrib/file_fdw foreign table that uses file_fdw's
PROGRAM option.
To fix, ensure that child programs inherit SIG_DFL not SIG_IGN processing
of SIGPIPE. This seems like an all-around better situation since if
the called program wants some non-default treatment of SIGPIPE, it would
expect to have to set that up for itself. Then in COPY, if it's COPY
FROM PROGRAM and we stop reading short of detecting EOF, treat a SIGPIPE
exit from the called program as a non-error condition. This still allows
us to report an error for any case where the called program gets SIGPIPE
on some other file descriptor.
As coded, we won't report a SIGPIPE if we stop reading as a result of
seeing an in-band EOF marker (e.g. COPY BINARY EOF marker). It's
somewhat debatable whether we should complain if the called program
continues to transmit data after an EOF marker. However, it seems like
we should avoid throwing error in any questionable cases, especially in a
back-patched fix, and anyway it would take additional code to make such
an error get reported consistently.
Back-patch to v10. We could go further back, since COPY FROM PROGRAM
has been around awhile, but AFAICS the only way to reach this situation
using core or contrib is via file_fdw, which has only supported PROGRAM
sources since v10. The COPY statement per se has no feature whereby
it'd stop reading without having hit EOF or an error already. Therefore,
I don't see any upside to back-patching further that'd outweigh the
risk of complaints about behavioral change.
Per bug #15449 from Eric Cyr.
Patch by me, review by Etsuro Fujita and Kyotaro Horiguchi
Discussion: https://postgr.es/m/15449-1cf737dd5929450e@postgresql.org
2018-11-19 23:02:25 +01:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If we ended a COPY FROM PROGRAM before reaching EOF, then it's
|
|
|
|
* expectable for the called program to fail with SIGPIPE, and we
|
|
|
|
* should not report that as an error. Otherwise, SIGPIPE indicates a
|
|
|
|
* problem.
|
|
|
|
*/
|
|
|
|
if (cstate->is_copy_from && !cstate->reached_eof &&
|
Improve detection of child-process SIGPIPE failures.
Commit ffa4cbd62 added logic to detect SIGPIPE failure of a COPY child
process, but it only worked correctly if the SIGPIPE occurred in the
immediate child process. Depending on the shell in use and the
complexity of the shell command string, we might instead get back
an exit code of 128 + SIGPIPE, representing a shell error exit
reporting SIGPIPE in the child process.
We could just hack up ClosePipeToProgram() to add the extra case,
but it seems like this is a fairly general issue deserving a more
general and better-documented solution. I chose to add a couple
of functions in src/common/wait_error.c, which is a natural place
to know about wait-result encodings, that will test for either a
specific child-process signal type or any child-process signal failure.
Then, adjust other places that were doing ad-hoc tests of this type
to use the common functions.
In RestoreArchivedFile, this fixes a race condition affecting whether
the process will report an error or just silently proc_exit(1): before,
that depended on whether the intermediate shell got SIGTERM'd itself
or reported a child process failing on SIGTERM.
Like the previous patch, back-patch to v10; we could go further
but there seems no real need to.
Per report from Erik Rijkers.
Discussion: https://postgr.es/m/f3683f87ab1701bea5d86a7742b22432@xs4all.nl
2018-12-16 20:32:14 +01:00
|
|
|
wait_result_is_signal(pclose_rc, SIGPIPE))
|
Handle EPIPE more sanely when we close a pipe reading from a program.
Previously, any program launched by COPY TO/FROM PROGRAM inherited the
server's setting of SIGPIPE handling, i.e. SIG_IGN. Hence, if we were
doing COPY FROM PROGRAM and closed the pipe early, the child process
would see EPIPE on its output file and typically would treat that as
a fatal error, in turn causing the COPY to report error. Similarly,
one could get a failure report from a query that didn't read all of
the output from a contrib/file_fdw foreign table that uses file_fdw's
PROGRAM option.
To fix, ensure that child programs inherit SIG_DFL not SIG_IGN processing
of SIGPIPE. This seems like an all-around better situation since if
the called program wants some non-default treatment of SIGPIPE, it would
expect to have to set that up for itself. Then in COPY, if it's COPY
FROM PROGRAM and we stop reading short of detecting EOF, treat a SIGPIPE
exit from the called program as a non-error condition. This still allows
us to report an error for any case where the called program gets SIGPIPE
on some other file descriptor.
As coded, we won't report a SIGPIPE if we stop reading as a result of
seeing an in-band EOF marker (e.g. COPY BINARY EOF marker). It's
somewhat debatable whether we should complain if the called program
continues to transmit data after an EOF marker. However, it seems like
we should avoid throwing error in any questionable cases, especially in a
back-patched fix, and anyway it would take additional code to make such
an error get reported consistently.
Back-patch to v10. We could go further back, since COPY FROM PROGRAM
has been around awhile, but AFAICS the only way to reach this situation
using core or contrib is via file_fdw, which has only supported PROGRAM
sources since v10. The COPY statement per se has no feature whereby
it'd stop reading without having hit EOF or an error already. Therefore,
I don't see any upside to back-patching further that'd outweigh the
risk of complaints about behavioral change.
Per bug #15449 from Eric Cyr.
Patch by me, review by Etsuro Fujita and Kyotaro Horiguchi
Discussion: https://postgr.es/m/15449-1cf737dd5929450e@postgresql.org
2018-11-19 23:02:25 +01:00
|
|
|
return;
|
|
|
|
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
ereport(ERROR,
|
2015-08-03 05:49:19 +02:00
|
|
|
(errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
|
|
|
|
errmsg("program \"%s\" failed",
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
cstate->filename),
|
|
|
|
errdetail_internal("%s", wait_result_to_str(pclose_rc))));
|
Handle EPIPE more sanely when we close a pipe reading from a program.
Previously, any program launched by COPY TO/FROM PROGRAM inherited the
server's setting of SIGPIPE handling, i.e. SIG_IGN. Hence, if we were
doing COPY FROM PROGRAM and closed the pipe early, the child process
would see EPIPE on its output file and typically would treat that as
a fatal error, in turn causing the COPY to report error. Similarly,
one could get a failure report from a query that didn't read all of
the output from a contrib/file_fdw foreign table that uses file_fdw's
PROGRAM option.
To fix, ensure that child programs inherit SIG_DFL not SIG_IGN processing
of SIGPIPE. This seems like an all-around better situation since if
the called program wants some non-default treatment of SIGPIPE, it would
expect to have to set that up for itself. Then in COPY, if it's COPY
FROM PROGRAM and we stop reading short of detecting EOF, treat a SIGPIPE
exit from the called program as a non-error condition. This still allows
us to report an error for any case where the called program gets SIGPIPE
on some other file descriptor.
As coded, we won't report a SIGPIPE if we stop reading as a result of
seeing an in-band EOF marker (e.g. COPY BINARY EOF marker). It's
somewhat debatable whether we should complain if the called program
continues to transmit data after an EOF marker. However, it seems like
we should avoid throwing error in any questionable cases, especially in a
back-patched fix, and anyway it would take additional code to make such
an error get reported consistently.
Back-patch to v10. We could go further back, since COPY FROM PROGRAM
has been around awhile, but AFAICS the only way to reach this situation
using core or contrib is via file_fdw, which has only supported PROGRAM
sources since v10. The COPY statement per se has no feature whereby
it'd stop reading without having hit EOF or an error already. Therefore,
I don't see any upside to back-patching further that'd outweigh the
risk of complaints about behavioral change.
Per bug #15449 from Eric Cyr.
Patch by me, review by Etsuro Fujita and Kyotaro Horiguchi
Discussion: https://postgr.es/m/15449-1cf737dd5929450e@postgresql.org
2018-11-19 23:02:25 +01:00
|
|
|
}
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
}
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/*
|
|
|
|
* Release resources allocated in a cstate for COPY TO/FROM.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
EndCopy(CopyState cstate)
|
|
|
|
{
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
if (cstate->is_program)
|
|
|
|
{
|
|
|
|
ClosePipeToProgram(cstate);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
if (cstate->filename != NULL && FreeFile(cstate->copy_file))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not close file \"%s\": %m",
|
|
|
|
cstate->filename)));
|
|
|
|
}
|
2002-02-24 03:33:33 +01:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
MemoryContextDelete(cstate->copycontext);
|
2006-08-31 01:34:22 +02:00
|
|
|
pfree(cstate);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2011-02-16 03:19:11 +01:00
|
|
|
* Setup CopyState to read tuples from a table or a query for COPY TO.
|
2006-08-31 01:34:22 +02:00
|
|
|
*/
|
2011-02-16 03:19:11 +01:00
|
|
|
static CopyState
|
2016-09-06 18:00:00 +02:00
|
|
|
BeginCopyTo(ParseState *pstate,
|
|
|
|
Relation rel,
|
Change representation of statement lists, and add statement location info.
This patch makes several changes that improve the consistency of
representation of lists of statements. It's always been the case
that the output of parse analysis is a list of Query nodes, whatever
the types of the individual statements in the list. This patch brings
similar consistency to the outputs of raw parsing and planning steps:
* The output of raw parsing is now always a list of RawStmt nodes;
the statement-type-dependent nodes are one level down from that.
* The output of pg_plan_queries() is now always a list of PlannedStmt
nodes, even for utility statements. In the case of a utility statement,
"planning" just consists of wrapping a CMD_UTILITY PlannedStmt around
the utility node. This list representation is now used in Portal and
CachedPlan plan lists, replacing the former convention of intermixing
PlannedStmts with bare utility-statement nodes.
Now, every list of statements has a consistent head-node type depending
on how far along it is in processing. This allows changing many places
that formerly used generic "Node *" pointers to use a more specific
pointer type, thus reducing the number of IsA() tests and casts needed,
as well as improving code clarity.
Also, the post-parse-analysis representation of DECLARE CURSOR is changed
so that it looks more like EXPLAIN, PREPARE, etc. That is, the contained
SELECT remains a child of the DeclareCursorStmt rather than getting flipped
around to be the other way. It's now true for both Query and PlannedStmt
that utilityStmt is non-null if and only if commandType is CMD_UTILITY.
That allows simplifying a lot of places that were testing both fields.
(I think some of those were just defensive programming, but in many places,
it was actually necessary to avoid confusing DECLARE CURSOR with SELECT.)
Because PlannedStmt carries a canSetTag field, we're also able to get rid
of some ad-hoc rules about how to reconstruct canSetTag for a bare utility
statement; specifically, the assumption that a utility is canSetTag if and
only if it's the only one in its list. While I see no near-term need for
relaxing that restriction, it's nice to get rid of the ad-hocery.
The API of ProcessUtility() is changed so that what it's passed is the
wrapper PlannedStmt not just the bare utility statement. This will affect
all users of ProcessUtility_hook, but the changes are pretty trivial; see
the affected contrib modules for examples of the minimum change needed.
(Most compilers should give pointer-type-mismatch warnings for uncorrected
code.)
There's also a change in the API of ExplainOneQuery_hook, to pass through
cursorOptions instead of expecting hook functions to know what to pick.
This is needed because of the DECLARE CURSOR changes, but really should
have been done in 9.6; it's unlikely that any extant hook functions
know about using CURSOR_OPT_PARALLEL_OK.
Finally, teach gram.y to save statement boundary locations in RawStmt
nodes, and pass those through to Query and PlannedStmt nodes. This allows
more intelligent handling of cases where a source query string contains
multiple statements. This patch doesn't actually do anything with the
information, but a follow-on patch will. (Passing this information through
cleanly is the true motivation for these changes; while I think this is all
good cleanup, it's unlikely we'd have bothered without this end goal.)
catversion bump because addition of location fields to struct Query
affects stored rules.
This patch is by me, but it owes a good deal to Fabien Coelho who did
a lot of preliminary work on the problem, and also reviewed the patch.
Discussion: https://postgr.es/m/alpine.DEB.2.20.1612200926310.29821@lancre
2017-01-14 22:02:35 +01:00
|
|
|
RawStmt *query,
|
|
|
|
Oid queryRelId,
|
2011-02-16 03:19:11 +01:00
|
|
|
const char *filename,
|
2013-05-29 22:58:43 +02:00
|
|
|
bool is_program,
|
2011-02-16 03:19:11 +01:00
|
|
|
List *attnamelist,
|
|
|
|
List *options)
|
2006-08-31 01:34:22 +02:00
|
|
|
{
|
2011-02-16 03:19:11 +01:00
|
|
|
CopyState cstate;
|
|
|
|
bool pipe = (filename == NULL);
|
|
|
|
MemoryContext oldcontext;
|
2006-08-31 01:34:22 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
if (rel != NULL && rel->rd_rel->relkind != RELKIND_RELATION)
|
2006-08-31 01:34:22 +02:00
|
|
|
{
|
2011-02-16 03:19:11 +01:00
|
|
|
if (rel->rd_rel->relkind == RELKIND_VIEW)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("cannot copy from view \"%s\"",
|
|
|
|
RelationGetRelationName(rel)),
|
|
|
|
errhint("Try the COPY (SELECT ...) TO variant.")));
|
2013-03-04 01:23:31 +01:00
|
|
|
else if (rel->rd_rel->relkind == RELKIND_MATVIEW)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("cannot copy from materialized view \"%s\"",
|
|
|
|
RelationGetRelationName(rel)),
|
|
|
|
errhint("Try the COPY (SELECT ...) TO variant.")));
|
2011-02-16 03:19:11 +01:00
|
|
|
else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("cannot copy from foreign table \"%s\"",
|
|
|
|
RelationGetRelationName(rel)),
|
|
|
|
errhint("Try the COPY (SELECT ...) TO variant.")));
|
|
|
|
else if (rel->rd_rel->relkind == RELKIND_SEQUENCE)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("cannot copy from sequence \"%s\"",
|
|
|
|
RelationGetRelationName(rel))));
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("cannot copy from partitioned table \"%s\"",
|
|
|
|
RelationGetRelationName(rel)),
|
|
|
|
errhint("Try the COPY (SELECT ...) TO variant.")));
|
2011-02-16 03:19:11 +01:00
|
|
|
else
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("cannot copy from non-table relation \"%s\"",
|
|
|
|
RelationGetRelationName(rel))));
|
2000-01-16 22:37:50 +01:00
|
|
|
}
|
|
|
|
|
2016-09-06 18:00:00 +02:00
|
|
|
cstate = BeginCopy(pstate, false, rel, query, queryRelId, attnamelist,
|
Rename pg_rowsecurity -> pg_policy and other fixes
As pointed out by Robert, we should really have named pg_rowsecurity
pg_policy, as the objects stored in that catalog are policies. This
patch fixes that and updates the column names to start with 'pol' to
match the new catalog name.
The security consideration for COPY with row level security, also
pointed out by Robert, has also been addressed by remembering and
re-checking the OID of the relation initially referenced during COPY
processing, to make sure it hasn't changed under us by the time we
finish planning out the query which has been built.
Robert and Alvaro also commented on missing OCLASS and OBJECT entries
for POLICY (formerly ROWSECURITY or POLICY, depending) in various
places. This patch fixes that too, which also happens to add the
ability to COMMENT on policies.
In passing, attempt to improve the consistency of messages, comments,
and documentation as well. This removes various incarnations of
'row-security', 'row-level security', 'Row-security', etc, in favor
of 'policy', 'row level security' or 'row_security' as appropriate.
Happy Thanksgiving!
2014-11-27 07:06:36 +01:00
|
|
|
options);
|
2011-02-16 03:19:11 +01:00
|
|
|
oldcontext = MemoryContextSwitchTo(cstate->copycontext);
|
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
if (pipe)
|
2004-01-26 23:35:32 +01:00
|
|
|
{
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
Assert(!is_program); /* the grammar does not allow this */
|
2011-02-16 03:19:11 +01:00
|
|
|
if (whereToSendOutput != DestRemote)
|
2006-08-31 01:34:22 +02:00
|
|
|
cstate->copy_file = stdout;
|
2004-01-26 23:35:32 +01:00
|
|
|
}
|
2006-08-31 01:34:22 +02:00
|
|
|
else
|
|
|
|
{
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
cstate->filename = pstrdup(filename);
|
|
|
|
cstate->is_program = is_program;
|
1999-09-18 21:08:25 +02:00
|
|
|
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
if (is_program)
|
|
|
|
{
|
|
|
|
cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_W);
|
|
|
|
if (cstate->copy_file == NULL)
|
|
|
|
ereport(ERROR,
|
2015-08-03 05:49:19 +02:00
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not execute command \"%s\": %m",
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
cstate->filename)));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2013-05-29 22:58:43 +02:00
|
|
|
mode_t oumask; /* Pre-existing umask value */
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
struct stat st;
|
2006-03-03 20:54:10 +01:00
|
|
|
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
/*
|
|
|
|
* Prevent write to relative path ... too easy to shoot oneself in
|
|
|
|
* the foot by overwriting a database file ...
|
|
|
|
*/
|
|
|
|
if (!is_absolute_path(filename))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_NAME),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("relative path not allowed for COPY to file")));
|
2006-03-03 20:54:10 +01:00
|
|
|
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
oumask = umask(S_IWGRP | S_IWOTH);
|
2017-09-22 22:50:59 +02:00
|
|
|
PG_TRY();
|
|
|
|
{
|
|
|
|
cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_W);
|
|
|
|
}
|
2019-11-01 11:09:52 +01:00
|
|
|
PG_FINALLY();
|
2017-09-22 22:50:59 +02:00
|
|
|
{
|
|
|
|
umask(oumask);
|
|
|
|
}
|
|
|
|
PG_END_TRY();
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
if (cstate->copy_file == NULL)
|
2016-09-07 05:55:55 +02:00
|
|
|
{
|
|
|
|
/* copy errno because ereport subfunctions might change it */
|
|
|
|
int save_errno = errno;
|
|
|
|
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not open file \"%s\" for writing: %m",
|
2016-09-07 05:55:55 +02:00
|
|
|
cstate->filename),
|
|
|
|
(save_errno == ENOENT || save_errno == EACCES) ?
|
|
|
|
errhint("COPY TO instructs the PostgreSQL server process to write a file. "
|
|
|
|
"You may want a client-side facility such as psql's \\copy.") : 0));
|
|
|
|
}
|
1996-07-09 08:22:35 +02:00
|
|
|
|
2015-01-04 16:47:23 +01:00
|
|
|
if (fstat(fileno(cstate->copy_file), &st))
|
2015-08-03 05:49:19 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not stat file \"%s\": %m",
|
|
|
|
cstate->filename)));
|
2015-01-04 16:47:23 +01:00
|
|
|
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
if (S_ISDIR(st.st_mode))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("\"%s\" is a directory", cstate->filename)));
|
|
|
|
}
|
2006-08-31 01:34:22 +02:00
|
|
|
}
|
1996-11-02 03:01:48 +01:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
MemoryContextSwitchTo(oldcontext);
|
|
|
|
|
|
|
|
return cstate;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This intermediate routine exists mainly to localize the effects of setjmp
|
|
|
|
* so we don't need to plaster a lot of variables with "volatile".
|
|
|
|
*/
|
|
|
|
static uint64
|
|
|
|
DoCopyTo(CopyState cstate)
|
|
|
|
{
|
|
|
|
bool pipe = (cstate->filename == NULL);
|
|
|
|
bool fe_copy = (pipe && whereToSendOutput == DestRemote);
|
|
|
|
uint64 processed;
|
|
|
|
|
2004-07-31 02:45:57 +02:00
|
|
|
PG_TRY();
|
|
|
|
{
|
2011-02-16 03:19:11 +01:00
|
|
|
if (fe_copy)
|
2005-08-06 22:41:58 +02:00
|
|
|
SendCopyBegin(cstate);
|
2004-07-31 02:45:57 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
processed = CopyTo(cstate);
|
2004-07-31 02:45:57 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
if (fe_copy)
|
2005-08-06 22:41:58 +02:00
|
|
|
SendCopyEnd(cstate);
|
2004-07-31 02:45:57 +02:00
|
|
|
}
|
|
|
|
PG_CATCH();
|
|
|
|
{
|
|
|
|
/*
|
2004-08-29 07:07:03 +02:00
|
|
|
* Make sure we turn off old-style COPY OUT mode upon error. It is
|
2005-10-15 04:49:52 +02:00
|
|
|
* okay to do this in all cases, since it does nothing if the mode is
|
|
|
|
* not on.
|
2004-07-31 02:45:57 +02:00
|
|
|
*/
|
|
|
|
pq_endcopyout(true);
|
|
|
|
PG_RE_THROW();
|
|
|
|
}
|
|
|
|
PG_END_TRY();
|
2006-08-31 01:34:22 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
return processed;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Clean up storage and release resources for COPY TO.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
EndCopyTo(CopyState cstate)
|
|
|
|
{
|
|
|
|
if (cstate->queryDesc != NULL)
|
2006-08-31 01:34:22 +02:00
|
|
|
{
|
2011-02-16 03:19:11 +01:00
|
|
|
/* Close down the query and free resources. */
|
2011-02-27 19:43:29 +01:00
|
|
|
ExecutorFinish(cstate->queryDesc);
|
2011-02-16 03:19:11 +01:00
|
|
|
ExecutorEnd(cstate->queryDesc);
|
|
|
|
FreeQueryDesc(cstate->queryDesc);
|
|
|
|
PopActiveSnapshot();
|
2006-08-31 01:34:22 +02:00
|
|
|
}
|
2011-02-16 03:19:11 +01:00
|
|
|
|
|
|
|
/* Clean up storage */
|
|
|
|
EndCopy(cstate);
|
2004-07-31 02:45:57 +02:00
|
|
|
}
|
|
|
|
|
2000-12-02 21:49:24 +01:00
|
|
|
/*
|
2006-08-31 01:34:22 +02:00
|
|
|
* Copy from relation or query TO file.
|
2000-12-02 21:49:24 +01:00
|
|
|
*/
|
2011-02-16 03:19:11 +01:00
|
|
|
static uint64
|
2005-08-06 22:41:58 +02:00
|
|
|
CopyTo(CopyState cstate)
|
1996-07-09 08:22:35 +02:00
|
|
|
{
|
2000-12-02 21:49:24 +01:00
|
|
|
TupleDesc tupDesc;
|
2002-08-02 20:15:10 +02:00
|
|
|
int num_phys_attrs;
|
2004-05-26 06:41:50 +02:00
|
|
|
ListCell *cur;
|
2011-02-16 03:19:11 +01:00
|
|
|
uint64 processed;
|
2001-08-10 20:57:42 +02:00
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
if (cstate->rel)
|
|
|
|
tupDesc = RelationGetDescr(cstate->rel);
|
|
|
|
else
|
|
|
|
tupDesc = cstate->queryDesc->tupDesc;
|
2002-08-02 20:15:10 +02:00
|
|
|
num_phys_attrs = tupDesc->natts;
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
cstate->null_print_client = cstate->null_print; /* default */
|
2000-12-02 21:49:24 +01:00
|
|
|
|
2006-05-27 00:50:02 +02:00
|
|
|
/* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
|
2017-05-10 23:41:27 +02:00
|
|
|
cstate->fe_msgbuf = makeStringInfo();
|
2006-05-27 00:50:02 +02:00
|
|
|
|
2005-05-06 04:56:42 +02:00
|
|
|
/* Get info about the columns we need to process. */
|
2006-08-31 01:34:22 +02:00
|
|
|
cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
|
2005-08-06 22:41:58 +02:00
|
|
|
foreach(cur, cstate->attnumlist)
|
2000-12-02 21:49:24 +01:00
|
|
|
{
|
2004-05-26 06:41:50 +02:00
|
|
|
int attnum = lfirst_int(cur);
|
2000-12-02 21:49:24 +01:00
|
|
|
Oid out_func_oid;
|
2005-05-01 20:56:19 +02:00
|
|
|
bool isvarlena;
|
2017-08-20 20:19:07 +02:00
|
|
|
Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->binary)
|
2017-08-20 20:19:07 +02:00
|
|
|
getTypeBinaryOutputInfo(attr->atttypid,
|
2005-05-01 20:56:19 +02:00
|
|
|
&out_func_oid,
|
|
|
|
&isvarlena);
|
2003-05-09 23:19:50 +02:00
|
|
|
else
|
2017-08-20 20:19:07 +02:00
|
|
|
getTypeOutputInfo(attr->atttypid,
|
2005-05-01 20:56:19 +02:00
|
|
|
&out_func_oid,
|
|
|
|
&isvarlena);
|
2006-08-31 01:34:22 +02:00
|
|
|
fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
|
2000-12-02 21:49:24 +01:00
|
|
|
}
|
1997-09-07 07:04:48 +02:00
|
|
|
|
2002-12-01 19:14:22 +01:00
|
|
|
/*
|
2003-08-04 02:43:34 +02:00
|
|
|
* Create a temporary memory context that we can reset once per row to
|
2005-10-15 04:49:52 +02:00
|
|
|
* recover palloc'd memory. This avoids any problems with leaks inside
|
|
|
|
* datatype output routines, and should be faster than retail pfree's
|
2014-05-06 18:12:18 +02:00
|
|
|
* anyway. (We don't need a whole econtext as CopyFrom does.)
|
2002-12-01 19:14:22 +01:00
|
|
|
*/
|
2006-08-31 01:34:22 +02:00
|
|
|
cstate->rowcontext = AllocSetContextCreate(CurrentMemoryContext,
|
|
|
|
"COPY TO",
|
Add macros to make AllocSetContextCreate() calls simpler and safer.
I found that half a dozen (nearly 5%) of our AllocSetContextCreate calls
had typos in the context-sizing parameters. While none of these led to
especially significant problems, they did create minor inefficiencies,
and it's now clear that expecting people to copy-and-paste those calls
accurately is not a great idea. Let's reduce the risk of future errors
by introducing single macros that encapsulate the common use-cases.
Three such macros are enough to cover all but two special-purpose contexts;
those two calls can be left as-is, I think.
While this patch doesn't in itself improve matters for third-party
extensions, it doesn't break anything for them either, and they can
gradually adopt the simplified notation over time.
In passing, change TopMemoryContext to use the default allocation
parameters. Formerly it could only be extended 8K at a time. That was
probably reasonable when this code was written; but nowadays we create
many more contexts than we did then, so that it's not unusual to have a
couple hundred K in TopMemoryContext, even without considering various
dubious code that sticks other things there. There seems no good reason
not to let it use growing blocks like most other contexts.
Back-patch to 9.6, mostly because that's still close enough to HEAD that
it's easy to do so, and keeping the branches in sync can be expected to
avoid some future back-patching pain. The bugs fixed by these changes
don't seem to be significant enough to justify fixing them further back.
Discussion: <21072.1472321324@sss.pgh.pa.us>
2016-08-27 23:50:38 +02:00
|
|
|
ALLOCSET_DEFAULT_SIZES);
|
2002-12-01 19:14:22 +01:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->binary)
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
2001-01-03 21:04:10 +01:00
|
|
|
/* Generate header for a binary copy */
|
|
|
|
int32 tmp;
|
|
|
|
|
|
|
|
/* Signature */
|
2011-09-11 20:54:32 +02:00
|
|
|
CopySendData(cstate, BinarySignature, 11);
|
2001-01-03 21:04:10 +01:00
|
|
|
/* Flags field */
|
|
|
|
tmp = 0;
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendInt32(cstate, tmp);
|
2001-01-03 21:04:10 +01:00
|
|
|
/* No header extension */
|
|
|
|
tmp = 0;
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendInt32(cstate, tmp);
|
1997-09-07 07:04:48 +02:00
|
|
|
}
|
2003-10-06 04:38:53 +02:00
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
2011-02-21 06:08:04 +01:00
|
|
|
* For non-binary copy, we need to convert null_print to file
|
2003-10-06 04:38:53 +02:00
|
|
|
* encoding, because it will be sent directly with CopySendString.
|
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->need_transcoding)
|
2011-02-21 06:08:04 +01:00
|
|
|
cstate->null_print_client = pg_server_to_any(cstate->null_print,
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
cstate->null_print_len,
|
|
|
|
cstate->file_encoding);
|
2005-05-07 04:22:49 +02:00
|
|
|
|
|
|
|
/* if a header has been requested send the line */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->header_line)
|
2005-05-07 04:22:49 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
bool hdr_delim = false;
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
foreach(cur, cstate->attnumlist)
|
2005-05-07 04:22:49 +02:00
|
|
|
{
|
|
|
|
int attnum = lfirst_int(cur);
|
2005-10-15 04:49:52 +02:00
|
|
|
char *colname;
|
2005-05-07 04:22:49 +02:00
|
|
|
|
|
|
|
if (hdr_delim)
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendChar(cstate, cstate->delim[0]);
|
2005-05-07 04:22:49 +02:00
|
|
|
hdr_delim = true;
|
|
|
|
|
2017-08-20 20:19:07 +02:00
|
|
|
colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
|
2005-05-07 04:22:49 +02:00
|
|
|
|
2005-12-28 04:25:32 +01:00
|
|
|
CopyAttributeOutCSV(cstate, colname, false,
|
|
|
|
list_length(cstate->attnumlist) == 1);
|
2005-05-07 04:22:49 +02:00
|
|
|
}
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendEndOfRow(cstate);
|
2005-05-07 04:22:49 +02:00
|
|
|
}
|
2003-10-06 04:38:53 +02:00
|
|
|
}
|
2000-12-02 21:49:24 +01:00
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
if (cstate->rel)
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
TupleTableSlot *slot;
|
tableam: Add and use scan APIs.
Too allow table accesses to be not directly dependent on heap, several
new abstractions are needed. Specifically:
1) Heap scans need to be generalized into table scans. Do this by
introducing TableScanDesc, which will be the "base class" for
individual AMs. This contains the AM independent fields from
HeapScanDesc.
The previous heap_{beginscan,rescan,endscan} et al. have been
replaced with a table_ version.
There's no direct replacement for heap_getnext(), as that returned
a HeapTuple, which is undesirable for a other AMs. Instead there's
table_scan_getnextslot(). But note that heap_getnext() lives on,
it's still used widely to access catalog tables.
This is achieved by new scan_begin, scan_end, scan_rescan,
scan_getnextslot callbacks.
2) The portion of parallel scans that's shared between backends need
to be able to do so without the user doing per-AM work. To achieve
that new parallelscan_{estimate, initialize, reinitialize}
callbacks are introduced, which operate on a new
ParallelTableScanDesc, which again can be subclassed by AMs.
As it is likely that several AMs are going to be block oriented,
block oriented callbacks that can be shared between such AMs are
provided and used by heap. table_block_parallelscan_{estimate,
intiialize, reinitialize} as callbacks, and
table_block_parallelscan_{nextpage, init} for use in AMs. These
operate on a ParallelBlockTableScanDesc.
3) Index scans need to be able to access tables to return a tuple, and
there needs to be state across individual accesses to the heap to
store state like buffers. That's now handled by introducing a
sort-of-scan IndexFetchTable, which again is intended to be
subclassed by individual AMs (for heap IndexFetchHeap).
The relevant callbacks for an AM are index_fetch_{end, begin,
reset} to create the necessary state, and index_fetch_tuple to
retrieve an indexed tuple. Note that index_fetch_tuple
implementations need to be smarter than just blindly fetching the
tuples for AMs that have optimizations similar to heap's HOT - the
currently alive tuple in the update chain needs to be fetched if
appropriate.
Similar to table_scan_getnextslot(), it's undesirable to continue
to return HeapTuples. Thus index_fetch_heap (might want to rename
that later) now accepts a slot as an argument. Core code doesn't
have a lot of call sites performing index scans without going
through the systable_* API (in contrast to loads of heap_getnext
calls and working directly with HeapTuples).
Index scans now store the result of a search in
IndexScanDesc->xs_heaptid, rather than xs_ctup->t_self. As the
target is not generally a HeapTuple anymore that seems cleaner.
To be able to sensible adapt code to use the above, two further
callbacks have been introduced:
a) slot_callbacks returns a TupleTableSlotOps* suitable for creating
slots capable of holding a tuple of the AMs
type. table_slot_callbacks() and table_slot_create() are based
upon that, but have additional logic to deal with views, foreign
tables, etc.
While this change could have been done separately, nearly all the
call sites that needed to be adapted for the rest of this commit
also would have been needed to be adapted for
table_slot_callbacks(), making separation not worthwhile.
b) tuple_satisfies_snapshot checks whether the tuple in a slot is
currently visible according to a snapshot. That's required as a few
places now don't have a buffer + HeapTuple around, but a
slot (which in heap's case internally has that information).
Additionally a few infrastructure changes were needed:
I) SysScanDesc, as used by systable_{beginscan, getnext} et al. now
internally uses a slot to keep track of tuples. While
systable_getnext() still returns HeapTuples, and will so for the
foreseeable future, the index API (see 1) above) now only deals with
slots.
The remainder, and largest part, of this commit is then adjusting all
scans in postgres to use the new APIs.
Author: Andres Freund, Haribabu Kommi, Alvaro Herrera
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20160812231527.GA690404@alvherre.pgsql
2019-03-11 20:46:41 +01:00
|
|
|
TableScanDesc scandesc;
|
1997-09-07 07:04:48 +02:00
|
|
|
|
tableam: Add and use scan APIs.
Too allow table accesses to be not directly dependent on heap, several
new abstractions are needed. Specifically:
1) Heap scans need to be generalized into table scans. Do this by
introducing TableScanDesc, which will be the "base class" for
individual AMs. This contains the AM independent fields from
HeapScanDesc.
The previous heap_{beginscan,rescan,endscan} et al. have been
replaced with a table_ version.
There's no direct replacement for heap_getnext(), as that returned
a HeapTuple, which is undesirable for a other AMs. Instead there's
table_scan_getnextslot(). But note that heap_getnext() lives on,
it's still used widely to access catalog tables.
This is achieved by new scan_begin, scan_end, scan_rescan,
scan_getnextslot callbacks.
2) The portion of parallel scans that's shared between backends need
to be able to do so without the user doing per-AM work. To achieve
that new parallelscan_{estimate, initialize, reinitialize}
callbacks are introduced, which operate on a new
ParallelTableScanDesc, which again can be subclassed by AMs.
As it is likely that several AMs are going to be block oriented,
block oriented callbacks that can be shared between such AMs are
provided and used by heap. table_block_parallelscan_{estimate,
intiialize, reinitialize} as callbacks, and
table_block_parallelscan_{nextpage, init} for use in AMs. These
operate on a ParallelBlockTableScanDesc.
3) Index scans need to be able to access tables to return a tuple, and
there needs to be state across individual accesses to the heap to
store state like buffers. That's now handled by introducing a
sort-of-scan IndexFetchTable, which again is intended to be
subclassed by individual AMs (for heap IndexFetchHeap).
The relevant callbacks for an AM are index_fetch_{end, begin,
reset} to create the necessary state, and index_fetch_tuple to
retrieve an indexed tuple. Note that index_fetch_tuple
implementations need to be smarter than just blindly fetching the
tuples for AMs that have optimizations similar to heap's HOT - the
currently alive tuple in the update chain needs to be fetched if
appropriate.
Similar to table_scan_getnextslot(), it's undesirable to continue
to return HeapTuples. Thus index_fetch_heap (might want to rename
that later) now accepts a slot as an argument. Core code doesn't
have a lot of call sites performing index scans without going
through the systable_* API (in contrast to loads of heap_getnext
calls and working directly with HeapTuples).
Index scans now store the result of a search in
IndexScanDesc->xs_heaptid, rather than xs_ctup->t_self. As the
target is not generally a HeapTuple anymore that seems cleaner.
To be able to sensible adapt code to use the above, two further
callbacks have been introduced:
a) slot_callbacks returns a TupleTableSlotOps* suitable for creating
slots capable of holding a tuple of the AMs
type. table_slot_callbacks() and table_slot_create() are based
upon that, but have additional logic to deal with views, foreign
tables, etc.
While this change could have been done separately, nearly all the
call sites that needed to be adapted for the rest of this commit
also would have been needed to be adapted for
table_slot_callbacks(), making separation not worthwhile.
b) tuple_satisfies_snapshot checks whether the tuple in a slot is
currently visible according to a snapshot. That's required as a few
places now don't have a buffer + HeapTuple around, but a
slot (which in heap's case internally has that information).
Additionally a few infrastructure changes were needed:
I) SysScanDesc, as used by systable_{beginscan, getnext} et al. now
internally uses a slot to keep track of tuples. While
systable_getnext() still returns HeapTuples, and will so for the
foreseeable future, the index API (see 1) above) now only deals with
slots.
The remainder, and largest part, of this commit is then adjusting all
scans in postgres to use the new APIs.
Author: Andres Freund, Haribabu Kommi, Alvaro Herrera
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20160812231527.GA690404@alvherre.pgsql
2019-03-11 20:46:41 +01:00
|
|
|
scandesc = table_beginscan(cstate->rel, GetActiveSnapshot(), 0, NULL);
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
slot = table_slot_create(cstate->rel, NULL);
|
2002-12-01 19:14:22 +01:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
processed = 0;
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
while (table_scan_getnextslot(scandesc, ForwardScanDirection, slot))
|
2001-01-03 21:04:10 +01:00
|
|
|
{
|
2006-08-31 01:34:22 +02:00
|
|
|
CHECK_FOR_INTERRUPTS();
|
2002-09-02 03:05:06 +02:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* Deconstruct the tuple ... */
|
|
|
|
slot_getallattrs(slot);
|
2006-08-31 01:34:22 +02:00
|
|
|
|
|
|
|
/* Format and send the data */
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
CopyOneRowTo(cstate, slot);
|
2011-02-16 03:19:11 +01:00
|
|
|
processed++;
|
2001-01-03 21:04:10 +01:00
|
|
|
}
|
2006-08-31 01:34:22 +02:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
ExecDropSingleTupleTableSlot(slot);
|
tableam: Add and use scan APIs.
Too allow table accesses to be not directly dependent on heap, several
new abstractions are needed. Specifically:
1) Heap scans need to be generalized into table scans. Do this by
introducing TableScanDesc, which will be the "base class" for
individual AMs. This contains the AM independent fields from
HeapScanDesc.
The previous heap_{beginscan,rescan,endscan} et al. have been
replaced with a table_ version.
There's no direct replacement for heap_getnext(), as that returned
a HeapTuple, which is undesirable for a other AMs. Instead there's
table_scan_getnextslot(). But note that heap_getnext() lives on,
it's still used widely to access catalog tables.
This is achieved by new scan_begin, scan_end, scan_rescan,
scan_getnextslot callbacks.
2) The portion of parallel scans that's shared between backends need
to be able to do so without the user doing per-AM work. To achieve
that new parallelscan_{estimate, initialize, reinitialize}
callbacks are introduced, which operate on a new
ParallelTableScanDesc, which again can be subclassed by AMs.
As it is likely that several AMs are going to be block oriented,
block oriented callbacks that can be shared between such AMs are
provided and used by heap. table_block_parallelscan_{estimate,
intiialize, reinitialize} as callbacks, and
table_block_parallelscan_{nextpage, init} for use in AMs. These
operate on a ParallelBlockTableScanDesc.
3) Index scans need to be able to access tables to return a tuple, and
there needs to be state across individual accesses to the heap to
store state like buffers. That's now handled by introducing a
sort-of-scan IndexFetchTable, which again is intended to be
subclassed by individual AMs (for heap IndexFetchHeap).
The relevant callbacks for an AM are index_fetch_{end, begin,
reset} to create the necessary state, and index_fetch_tuple to
retrieve an indexed tuple. Note that index_fetch_tuple
implementations need to be smarter than just blindly fetching the
tuples for AMs that have optimizations similar to heap's HOT - the
currently alive tuple in the update chain needs to be fetched if
appropriate.
Similar to table_scan_getnextslot(), it's undesirable to continue
to return HeapTuples. Thus index_fetch_heap (might want to rename
that later) now accepts a slot as an argument. Core code doesn't
have a lot of call sites performing index scans without going
through the systable_* API (in contrast to loads of heap_getnext
calls and working directly with HeapTuples).
Index scans now store the result of a search in
IndexScanDesc->xs_heaptid, rather than xs_ctup->t_self. As the
target is not generally a HeapTuple anymore that seems cleaner.
To be able to sensible adapt code to use the above, two further
callbacks have been introduced:
a) slot_callbacks returns a TupleTableSlotOps* suitable for creating
slots capable of holding a tuple of the AMs
type. table_slot_callbacks() and table_slot_create() are based
upon that, but have additional logic to deal with views, foreign
tables, etc.
While this change could have been done separately, nearly all the
call sites that needed to be adapted for the rest of this commit
also would have been needed to be adapted for
table_slot_callbacks(), making separation not worthwhile.
b) tuple_satisfies_snapshot checks whether the tuple in a slot is
currently visible according to a snapshot. That's required as a few
places now don't have a buffer + HeapTuple around, but a
slot (which in heap's case internally has that information).
Additionally a few infrastructure changes were needed:
I) SysScanDesc, as used by systable_{beginscan, getnext} et al. now
internally uses a slot to keep track of tuples. While
systable_getnext() still returns HeapTuples, and will so for the
foreseeable future, the index API (see 1) above) now only deals with
slots.
The remainder, and largest part, of this commit is then adjusting all
scans in postgres to use the new APIs.
Author: Andres Freund, Haribabu Kommi, Alvaro Herrera
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20160812231527.GA690404@alvherre.pgsql
2019-03-11 20:46:41 +01:00
|
|
|
table_endscan(scandesc);
|
2006-08-31 01:34:22 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* run the plan --- the dest receiver will send tuples */
|
2017-03-23 18:05:48 +01:00
|
|
|
ExecutorRun(cstate->queryDesc, ForwardScanDirection, 0L, true);
|
2011-02-16 03:19:11 +01:00
|
|
|
processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
|
2006-08-31 01:34:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if (cstate->binary)
|
|
|
|
{
|
|
|
|
/* Generate trailer for a binary copy */
|
|
|
|
CopySendInt16(cstate, -1);
|
|
|
|
/* Need to flush out the trailer */
|
|
|
|
CopySendEndOfRow(cstate);
|
|
|
|
}
|
|
|
|
|
|
|
|
MemoryContextDelete(cstate->rowcontext);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
|
|
|
return processed;
|
2006-08-31 01:34:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Emit one row during CopyTo().
|
|
|
|
*/
|
|
|
|
static void
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
CopyOneRowTo(CopyState cstate, TupleTableSlot *slot)
|
2006-08-31 01:34:22 +02:00
|
|
|
{
|
|
|
|
bool need_delim = false;
|
|
|
|
FmgrInfo *out_functions = cstate->out_functions;
|
|
|
|
MemoryContext oldcontext;
|
|
|
|
ListCell *cur;
|
|
|
|
char *string;
|
|
|
|
|
|
|
|
MemoryContextReset(cstate->rowcontext);
|
|
|
|
oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
|
|
|
|
|
|
|
|
if (cstate->binary)
|
|
|
|
{
|
|
|
|
/* Binary per-tuple header */
|
|
|
|
CopySendInt16(cstate, list_length(cstate->attnumlist));
|
|
|
|
}
|
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* Make sure the tuple is fully deconstructed */
|
|
|
|
slot_getallattrs(slot);
|
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
foreach(cur, cstate->attnumlist)
|
|
|
|
{
|
|
|
|
int attnum = lfirst_int(cur);
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
Datum value = slot->tts_values[attnum - 1];
|
|
|
|
bool isnull = slot->tts_isnull[attnum - 1];
|
2000-12-02 21:49:24 +01:00
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
if (!cstate->binary)
|
|
|
|
{
|
|
|
|
if (need_delim)
|
|
|
|
CopySendChar(cstate, cstate->delim[0]);
|
|
|
|
need_delim = true;
|
|
|
|
}
|
2000-12-02 21:49:24 +01:00
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
if (isnull)
|
|
|
|
{
|
|
|
|
if (!cstate->binary)
|
|
|
|
CopySendString(cstate, cstate->null_print_client);
|
|
|
|
else
|
|
|
|
CopySendInt32(cstate, -1);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!cstate->binary)
|
2001-01-03 21:04:10 +01:00
|
|
|
{
|
2006-08-31 01:34:22 +02:00
|
|
|
string = OutputFunctionCall(&out_functions[attnum - 1],
|
|
|
|
value);
|
|
|
|
if (cstate->csv_mode)
|
|
|
|
CopyAttributeOutCSV(cstate, string,
|
|
|
|
cstate->force_quote_flags[attnum - 1],
|
|
|
|
list_length(cstate->attnumlist) == 1);
|
2000-12-02 21:49:24 +01:00
|
|
|
else
|
2006-08-31 01:34:22 +02:00
|
|
|
CopyAttributeOutText(cstate, string);
|
2000-12-02 21:49:24 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2006-08-31 01:34:22 +02:00
|
|
|
bytea *outputbytes;
|
2003-05-09 23:19:50 +02:00
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
outputbytes = SendFunctionCall(&out_functions[attnum - 1],
|
|
|
|
value);
|
|
|
|
CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
|
|
|
|
CopySendData(cstate, VARDATA(outputbytes),
|
|
|
|
VARSIZE(outputbytes) - VARHDRSZ);
|
2000-12-02 21:49:24 +01:00
|
|
|
}
|
1997-09-07 07:04:48 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
CopySendEndOfRow(cstate);
|
2002-12-01 19:14:22 +01:00
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
MemoryContextSwitchTo(oldcontext);
|
1996-07-09 08:22:35 +02:00
|
|
|
}
|
|
|
|
|
2000-12-02 21:49:24 +01:00
|
|
|
|
2003-04-24 23:16:45 +02:00
|
|
|
/*
|
|
|
|
* error context callback for COPY FROM
|
2011-02-16 03:19:11 +01:00
|
|
|
*
|
|
|
|
* The argument for the error context must be CopyState.
|
2003-04-24 23:16:45 +02:00
|
|
|
*/
|
2011-02-16 03:19:11 +01:00
|
|
|
void
|
|
|
|
CopyFromErrorCallback(void *arg)
|
2003-04-24 23:16:45 +02:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
CopyState cstate = (CopyState) arg;
|
2018-05-22 19:32:52 +02:00
|
|
|
char curlineno_str[32];
|
|
|
|
|
|
|
|
snprintf(curlineno_str, sizeof(curlineno_str), UINT64_FORMAT,
|
|
|
|
cstate->cur_lineno);
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
if (cstate->binary)
|
2003-09-30 00:06:40 +02:00
|
|
|
{
|
|
|
|
/* can't usefully display the data */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->cur_attname)
|
2018-05-22 19:32:52 +02:00
|
|
|
errcontext("COPY %s, line %s, column %s",
|
|
|
|
cstate->cur_relname, curlineno_str,
|
2005-08-06 22:41:58 +02:00
|
|
|
cstate->cur_attname);
|
2003-09-30 00:06:40 +02:00
|
|
|
else
|
2018-05-22 19:32:52 +02:00
|
|
|
errcontext("COPY %s, line %s",
|
|
|
|
cstate->cur_relname, curlineno_str);
|
2003-09-30 00:06:40 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->cur_attname && cstate->cur_attval)
|
2003-09-30 00:06:40 +02:00
|
|
|
{
|
|
|
|
/* error is relevant to a particular column */
|
2005-10-15 04:49:52 +02:00
|
|
|
char *attval;
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
attval = limit_printout_length(cstate->cur_attval);
|
2018-05-22 19:32:52 +02:00
|
|
|
errcontext("COPY %s, line %s, column %s: \"%s\"",
|
|
|
|
cstate->cur_relname, curlineno_str,
|
2005-08-06 22:41:58 +02:00
|
|
|
cstate->cur_attname, attval);
|
|
|
|
pfree(attval);
|
2003-09-30 00:06:40 +02:00
|
|
|
}
|
2006-04-04 21:35:37 +02:00
|
|
|
else if (cstate->cur_attname)
|
|
|
|
{
|
|
|
|
/* error is relevant to a particular column, value is NULL */
|
2018-05-22 19:32:52 +02:00
|
|
|
errcontext("COPY %s, line %s, column %s: null input",
|
|
|
|
cstate->cur_relname, curlineno_str,
|
2006-04-04 21:35:37 +02:00
|
|
|
cstate->cur_attname);
|
|
|
|
}
|
2003-09-30 00:06:40 +02:00
|
|
|
else
|
|
|
|
{
|
2013-05-23 13:49:59 +02:00
|
|
|
/*
|
|
|
|
* Error is relevant to a particular line.
|
|
|
|
*
|
|
|
|
* If line_buf still contains the correct line, and it's already
|
2013-05-29 22:58:43 +02:00
|
|
|
* transcoded, print it. If it's still in a foreign encoding, it's
|
|
|
|
* quite likely that the error is precisely a failure to do
|
2013-05-23 13:49:59 +02:00
|
|
|
* encoding conversion (ie, bad data). We dare not try to convert
|
|
|
|
* it, and at present there's no way to regurgitate it without
|
|
|
|
* conversion. So we have to punt and just report the line number.
|
|
|
|
*/
|
|
|
|
if (cstate->line_buf_valid &&
|
|
|
|
(cstate->line_buf_converted || !cstate->need_transcoding))
|
2003-09-30 00:06:40 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
char *lineval;
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
lineval = limit_printout_length(cstate->line_buf.data);
|
2018-05-22 19:32:52 +02:00
|
|
|
errcontext("COPY %s, line %s: \"%s\"",
|
|
|
|
cstate->cur_relname, curlineno_str, lineval);
|
2005-08-06 22:41:58 +02:00
|
|
|
pfree(lineval);
|
2004-10-29 21:18:22 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2018-05-22 19:32:52 +02:00
|
|
|
errcontext("COPY %s, line %s",
|
|
|
|
cstate->cur_relname, curlineno_str);
|
2003-09-30 00:06:40 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2003-04-24 23:16:45 +02:00
|
|
|
}
|
|
|
|
|
2004-01-18 03:15:29 +01:00
|
|
|
/*
|
|
|
|
* Make sure we don't print an unreasonable amount of COPY data in a message.
|
|
|
|
*
|
|
|
|
* It would seem a lot easier to just use the sprintf "precision" limit to
|
|
|
|
* truncate the string. However, some versions of glibc have a bug/misfeature
|
|
|
|
* that vsnprintf will always fail (return -1) if it is asked to truncate
|
|
|
|
* a string that contains invalid byte sequences for the current encoding.
|
2005-08-06 22:41:58 +02:00
|
|
|
* So, do our own truncation. We return a pstrdup'd copy of the input.
|
2004-01-18 03:15:29 +01:00
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
static char *
|
|
|
|
limit_printout_length(const char *str)
|
2004-01-18 03:15:29 +01:00
|
|
|
{
|
|
|
|
#define MAX_COPY_DATA_DISPLAY 100
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
int slen = strlen(str);
|
2004-08-29 07:07:03 +02:00
|
|
|
int len;
|
2005-08-06 22:41:58 +02:00
|
|
|
char *res;
|
2004-01-18 03:15:29 +01:00
|
|
|
|
|
|
|
/* Fast path if definitely okay */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (slen <= MAX_COPY_DATA_DISPLAY)
|
|
|
|
return pstrdup(str);
|
2004-01-18 03:15:29 +01:00
|
|
|
|
|
|
|
/* Apply encoding-dependent truncation */
|
2005-08-06 22:41:58 +02:00
|
|
|
len = pg_mbcliplen(str, slen, MAX_COPY_DATA_DISPLAY);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Truncate, and add "..." to show we truncated the input.
|
|
|
|
*/
|
|
|
|
res = (char *) palloc(len + 4);
|
|
|
|
memcpy(res, str, len);
|
|
|
|
strcpy(res + len, "...");
|
|
|
|
|
|
|
|
return res;
|
2004-01-18 03:15:29 +01:00
|
|
|
}
|
2003-04-24 23:16:45 +02:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/*
|
|
|
|
* Allocate memory and initialize a new CopyMultiInsertBuffer for this
|
|
|
|
* ResultRelInfo.
|
|
|
|
*/
|
|
|
|
static CopyMultiInsertBuffer *
|
|
|
|
CopyMultiInsertBufferInit(ResultRelInfo *rri)
|
|
|
|
{
|
|
|
|
CopyMultiInsertBuffer *buffer;
|
|
|
|
|
|
|
|
buffer = (CopyMultiInsertBuffer *) palloc(sizeof(CopyMultiInsertBuffer));
|
|
|
|
memset(buffer->slots, 0, sizeof(TupleTableSlot *) * MAX_BUFFERED_TUPLES);
|
|
|
|
buffer->resultRelInfo = rri;
|
|
|
|
buffer->bistate = GetBulkInsertState();
|
|
|
|
buffer->nused = 0;
|
|
|
|
|
|
|
|
return buffer;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Make a new buffer for this ResultRelInfo.
|
|
|
|
*/
|
|
|
|
static inline void
|
|
|
|
CopyMultiInsertInfoSetupBuffer(CopyMultiInsertInfo *miinfo,
|
|
|
|
ResultRelInfo *rri)
|
|
|
|
{
|
|
|
|
CopyMultiInsertBuffer *buffer;
|
|
|
|
|
|
|
|
buffer = CopyMultiInsertBufferInit(rri);
|
|
|
|
|
|
|
|
/* Setup back-link so we can easily find this buffer again */
|
|
|
|
rri->ri_CopyMultiInsertBuffer = buffer;
|
|
|
|
/* Record that we're tracking this buffer */
|
|
|
|
miinfo->multiInsertBuffers = lappend(miinfo->multiInsertBuffers, buffer);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize an already allocated CopyMultiInsertInfo.
|
|
|
|
*
|
|
|
|
* If rri is a non-partitioned table then a CopyMultiInsertBuffer is set up
|
|
|
|
* for that table.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
CopyMultiInsertInfoInit(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri,
|
|
|
|
CopyState cstate, EState *estate, CommandId mycid,
|
|
|
|
int ti_options)
|
|
|
|
{
|
|
|
|
miinfo->multiInsertBuffers = NIL;
|
|
|
|
miinfo->bufferedTuples = 0;
|
|
|
|
miinfo->bufferedBytes = 0;
|
|
|
|
miinfo->cstate = cstate;
|
|
|
|
miinfo->estate = estate;
|
|
|
|
miinfo->mycid = mycid;
|
|
|
|
miinfo->ti_options = ti_options;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Only setup the buffer when not dealing with a partitioned table.
|
|
|
|
* Buffers for partitioned tables will just be setup when we need to send
|
|
|
|
* tuples their way for the first time.
|
|
|
|
*/
|
|
|
|
if (rri->ri_RelationDesc->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
|
|
|
|
CopyMultiInsertInfoSetupBuffer(miinfo, rri);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Returns true if the buffers are full
|
|
|
|
*/
|
|
|
|
static inline bool
|
|
|
|
CopyMultiInsertInfoIsFull(CopyMultiInsertInfo *miinfo)
|
|
|
|
{
|
|
|
|
if (miinfo->bufferedTuples >= MAX_BUFFERED_TUPLES ||
|
|
|
|
miinfo->bufferedBytes >= MAX_BUFFERED_BYTES)
|
|
|
|
return true;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Returns true if we have no buffered tuples
|
|
|
|
*/
|
|
|
|
static inline bool
|
|
|
|
CopyMultiInsertInfoIsEmpty(CopyMultiInsertInfo *miinfo)
|
|
|
|
{
|
|
|
|
return miinfo->bufferedTuples == 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Write the tuples stored in 'buffer' out to the table.
|
|
|
|
*/
|
|
|
|
static inline void
|
|
|
|
CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
|
|
|
|
CopyMultiInsertBuffer *buffer)
|
|
|
|
{
|
|
|
|
MemoryContext oldcontext;
|
|
|
|
int i;
|
|
|
|
uint64 save_cur_lineno;
|
|
|
|
CopyState cstate = miinfo->cstate;
|
|
|
|
EState *estate = miinfo->estate;
|
|
|
|
CommandId mycid = miinfo->mycid;
|
|
|
|
int ti_options = miinfo->ti_options;
|
|
|
|
bool line_buf_valid = cstate->line_buf_valid;
|
|
|
|
int nused = buffer->nused;
|
|
|
|
ResultRelInfo *resultRelInfo = buffer->resultRelInfo;
|
|
|
|
TupleTableSlot **slots = buffer->slots;
|
|
|
|
|
2019-06-05 08:28:38 +02:00
|
|
|
/* Set es_result_relation_info to the ResultRelInfo we're flushing. */
|
|
|
|
estate->es_result_relation_info = resultRelInfo;
|
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/*
|
|
|
|
* Print error context information correctly, if one of the operations
|
|
|
|
* below fail.
|
|
|
|
*/
|
|
|
|
cstate->line_buf_valid = false;
|
|
|
|
save_cur_lineno = cstate->cur_lineno;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* table_multi_insert may leak memory, so switch to short-lived memory
|
|
|
|
* context before calling it.
|
|
|
|
*/
|
|
|
|
oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
|
|
|
|
table_multi_insert(resultRelInfo->ri_RelationDesc,
|
|
|
|
slots,
|
|
|
|
nused,
|
|
|
|
mycid,
|
|
|
|
ti_options,
|
|
|
|
buffer->bistate);
|
|
|
|
MemoryContextSwitchTo(oldcontext);
|
|
|
|
|
|
|
|
for (i = 0; i < nused; i++)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If there are any indexes, update them for all the inserted tuples,
|
|
|
|
* and run AFTER ROW INSERT triggers.
|
|
|
|
*/
|
|
|
|
if (resultRelInfo->ri_NumIndices > 0)
|
|
|
|
{
|
|
|
|
List *recheckIndexes;
|
|
|
|
|
|
|
|
cstate->cur_lineno = buffer->linenos[i];
|
|
|
|
recheckIndexes =
|
|
|
|
ExecInsertIndexTuples(buffer->slots[i], estate, false, NULL,
|
|
|
|
NIL);
|
|
|
|
ExecARInsertTriggers(estate, resultRelInfo,
|
|
|
|
slots[i], recheckIndexes,
|
|
|
|
cstate->transition_capture);
|
|
|
|
list_free(recheckIndexes);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* There's no indexes, but see if we need to run AFTER ROW INSERT
|
|
|
|
* triggers anyway.
|
|
|
|
*/
|
|
|
|
else if (resultRelInfo->ri_TrigDesc != NULL &&
|
|
|
|
(resultRelInfo->ri_TrigDesc->trig_insert_after_row ||
|
|
|
|
resultRelInfo->ri_TrigDesc->trig_insert_new_table))
|
|
|
|
{
|
|
|
|
cstate->cur_lineno = buffer->linenos[i];
|
|
|
|
ExecARInsertTriggers(estate, resultRelInfo,
|
|
|
|
slots[i], NIL, cstate->transition_capture);
|
|
|
|
}
|
|
|
|
|
|
|
|
ExecClearTuple(slots[i]);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Mark that all slots are free */
|
|
|
|
buffer->nused = 0;
|
|
|
|
|
|
|
|
/* reset cur_lineno and line_buf_valid to what they were */
|
|
|
|
cstate->line_buf_valid = line_buf_valid;
|
|
|
|
cstate->cur_lineno = save_cur_lineno;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Drop used slots and free member for this buffer.
|
|
|
|
*
|
|
|
|
* The buffer must be flushed before cleanup.
|
|
|
|
*/
|
|
|
|
static inline void
|
Fix missing calls to table_finish_bulk_insert during COPY, take 2
86b85044e abstracted calls to heap functions in COPY FROM to support a
generic table AM. However, when performing a copy into a partitioned
table, this commit neglected to call table_finish_bulk_insert for each
partition. Before 86b85044e, when we always called the heap functions,
there was no need to call heapam_finish_bulk_insert for partitions since
it only did any work when performing a copy without WAL. For partitioned
tables, this was unsupported anyway, so there was no issue. With
pluggable storage, we can't make any assumptions about what the table AM
might want to do in its equivalent function, so we'd better ensure we
always call table_finish_bulk_insert each partition that's received a row.
For now, we make the table_finish_bulk_insert call whenever we evict a
CopyMultiInsertBuffer out of the CopyMultiInsertInfo. This does mean
that it's possible that we call table_finish_bulk_insert multiple times
per partition, which is not a problem other than being an inefficiency.
Improving this requires a more invasive patch, so let's leave that for
another day.
This also changes things so that we no longer needlessly call
table_finish_bulk_insert when performing a COPY FROM for a non-partitioned
table when not using multi-inserts.
Reported-by: Robert Haas
Backpatch-through: 12
Discussion: https://postgr.es/m/CA+TgmoYK=6BpxiJ0tN-p9wtH0BTAfbdxzHhwou0mdud4+BkYuQ@mail.gmail.com
2019-07-10 06:03:04 +02:00
|
|
|
CopyMultiInsertBufferCleanup(CopyMultiInsertInfo *miinfo,
|
|
|
|
CopyMultiInsertBuffer *buffer)
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/* Ensure buffer was flushed */
|
|
|
|
Assert(buffer->nused == 0);
|
|
|
|
|
|
|
|
/* Remove back-link to ourself */
|
|
|
|
buffer->resultRelInfo->ri_CopyMultiInsertBuffer = NULL;
|
|
|
|
|
|
|
|
FreeBulkInsertState(buffer->bistate);
|
|
|
|
|
|
|
|
/* Since we only create slots on demand, just drop the non-null ones. */
|
|
|
|
for (i = 0; i < MAX_BUFFERED_TUPLES && buffer->slots[i] != NULL; i++)
|
|
|
|
ExecDropSingleTupleTableSlot(buffer->slots[i]);
|
|
|
|
|
Fix missing calls to table_finish_bulk_insert during COPY, take 2
86b85044e abstracted calls to heap functions in COPY FROM to support a
generic table AM. However, when performing a copy into a partitioned
table, this commit neglected to call table_finish_bulk_insert for each
partition. Before 86b85044e, when we always called the heap functions,
there was no need to call heapam_finish_bulk_insert for partitions since
it only did any work when performing a copy without WAL. For partitioned
tables, this was unsupported anyway, so there was no issue. With
pluggable storage, we can't make any assumptions about what the table AM
might want to do in its equivalent function, so we'd better ensure we
always call table_finish_bulk_insert each partition that's received a row.
For now, we make the table_finish_bulk_insert call whenever we evict a
CopyMultiInsertBuffer out of the CopyMultiInsertInfo. This does mean
that it's possible that we call table_finish_bulk_insert multiple times
per partition, which is not a problem other than being an inefficiency.
Improving this requires a more invasive patch, so let's leave that for
another day.
This also changes things so that we no longer needlessly call
table_finish_bulk_insert when performing a COPY FROM for a non-partitioned
table when not using multi-inserts.
Reported-by: Robert Haas
Backpatch-through: 12
Discussion: https://postgr.es/m/CA+TgmoYK=6BpxiJ0tN-p9wtH0BTAfbdxzHhwou0mdud4+BkYuQ@mail.gmail.com
2019-07-10 06:03:04 +02:00
|
|
|
table_finish_bulk_insert(buffer->resultRelInfo->ri_RelationDesc,
|
|
|
|
miinfo->ti_options);
|
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
pfree(buffer);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Write out all stored tuples in all buffers out to the tables.
|
|
|
|
*
|
|
|
|
* Once flushed we also trim the tracked buffers list down to size by removing
|
|
|
|
* the buffers created earliest first.
|
|
|
|
*
|
|
|
|
* Callers should pass 'curr_rri' is the ResultRelInfo that's currently being
|
|
|
|
* used. When cleaning up old buffers we'll never remove the one for
|
|
|
|
* 'curr_rri'.
|
|
|
|
*/
|
|
|
|
static inline void
|
|
|
|
CopyMultiInsertInfoFlush(CopyMultiInsertInfo *miinfo, ResultRelInfo *curr_rri)
|
|
|
|
{
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
foreach(lc, miinfo->multiInsertBuffers)
|
|
|
|
{
|
|
|
|
CopyMultiInsertBuffer *buffer = (CopyMultiInsertBuffer *) lfirst(lc);
|
|
|
|
|
|
|
|
CopyMultiInsertBufferFlush(miinfo, buffer);
|
|
|
|
}
|
|
|
|
|
|
|
|
miinfo->bufferedTuples = 0;
|
|
|
|
miinfo->bufferedBytes = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Trim the list of tracked buffers down if it exceeds the limit. Here we
|
|
|
|
* remove buffers starting with the ones we created first. It seems more
|
|
|
|
* likely that these older ones are less likely to be needed than ones
|
|
|
|
* that were just created.
|
|
|
|
*/
|
|
|
|
while (list_length(miinfo->multiInsertBuffers) > MAX_PARTITION_BUFFERS)
|
|
|
|
{
|
|
|
|
CopyMultiInsertBuffer *buffer;
|
|
|
|
|
|
|
|
buffer = (CopyMultiInsertBuffer *) linitial(miinfo->multiInsertBuffers);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We never want to remove the buffer that's currently being used, so
|
|
|
|
* if we happen to find that then move it to the end of the list.
|
|
|
|
*/
|
|
|
|
if (buffer->resultRelInfo == curr_rri)
|
|
|
|
{
|
|
|
|
miinfo->multiInsertBuffers = list_delete_first(miinfo->multiInsertBuffers);
|
|
|
|
miinfo->multiInsertBuffers = lappend(miinfo->multiInsertBuffers, buffer);
|
|
|
|
buffer = (CopyMultiInsertBuffer *) linitial(miinfo->multiInsertBuffers);
|
|
|
|
}
|
|
|
|
|
Fix missing calls to table_finish_bulk_insert during COPY, take 2
86b85044e abstracted calls to heap functions in COPY FROM to support a
generic table AM. However, when performing a copy into a partitioned
table, this commit neglected to call table_finish_bulk_insert for each
partition. Before 86b85044e, when we always called the heap functions,
there was no need to call heapam_finish_bulk_insert for partitions since
it only did any work when performing a copy without WAL. For partitioned
tables, this was unsupported anyway, so there was no issue. With
pluggable storage, we can't make any assumptions about what the table AM
might want to do in its equivalent function, so we'd better ensure we
always call table_finish_bulk_insert each partition that's received a row.
For now, we make the table_finish_bulk_insert call whenever we evict a
CopyMultiInsertBuffer out of the CopyMultiInsertInfo. This does mean
that it's possible that we call table_finish_bulk_insert multiple times
per partition, which is not a problem other than being an inefficiency.
Improving this requires a more invasive patch, so let's leave that for
another day.
This also changes things so that we no longer needlessly call
table_finish_bulk_insert when performing a COPY FROM for a non-partitioned
table when not using multi-inserts.
Reported-by: Robert Haas
Backpatch-through: 12
Discussion: https://postgr.es/m/CA+TgmoYK=6BpxiJ0tN-p9wtH0BTAfbdxzHhwou0mdud4+BkYuQ@mail.gmail.com
2019-07-10 06:03:04 +02:00
|
|
|
CopyMultiInsertBufferCleanup(miinfo, buffer);
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
miinfo->multiInsertBuffers = list_delete_first(miinfo->multiInsertBuffers);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Cleanup allocated buffers and free memory
|
|
|
|
*/
|
|
|
|
static inline void
|
|
|
|
CopyMultiInsertInfoCleanup(CopyMultiInsertInfo *miinfo)
|
|
|
|
{
|
|
|
|
ListCell *lc;
|
|
|
|
|
|
|
|
foreach(lc, miinfo->multiInsertBuffers)
|
Fix missing calls to table_finish_bulk_insert during COPY, take 2
86b85044e abstracted calls to heap functions in COPY FROM to support a
generic table AM. However, when performing a copy into a partitioned
table, this commit neglected to call table_finish_bulk_insert for each
partition. Before 86b85044e, when we always called the heap functions,
there was no need to call heapam_finish_bulk_insert for partitions since
it only did any work when performing a copy without WAL. For partitioned
tables, this was unsupported anyway, so there was no issue. With
pluggable storage, we can't make any assumptions about what the table AM
might want to do in its equivalent function, so we'd better ensure we
always call table_finish_bulk_insert each partition that's received a row.
For now, we make the table_finish_bulk_insert call whenever we evict a
CopyMultiInsertBuffer out of the CopyMultiInsertInfo. This does mean
that it's possible that we call table_finish_bulk_insert multiple times
per partition, which is not a problem other than being an inefficiency.
Improving this requires a more invasive patch, so let's leave that for
another day.
This also changes things so that we no longer needlessly call
table_finish_bulk_insert when performing a COPY FROM for a non-partitioned
table when not using multi-inserts.
Reported-by: Robert Haas
Backpatch-through: 12
Discussion: https://postgr.es/m/CA+TgmoYK=6BpxiJ0tN-p9wtH0BTAfbdxzHhwou0mdud4+BkYuQ@mail.gmail.com
2019-07-10 06:03:04 +02:00
|
|
|
CopyMultiInsertBufferCleanup(miinfo, lfirst(lc));
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
|
|
|
|
list_free(miinfo->multiInsertBuffers);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get the next TupleTableSlot that the next tuple should be stored in.
|
|
|
|
*
|
|
|
|
* Callers must ensure that the buffer is not full.
|
|
|
|
*/
|
|
|
|
static inline TupleTableSlot *
|
|
|
|
CopyMultiInsertInfoNextFreeSlot(CopyMultiInsertInfo *miinfo,
|
|
|
|
ResultRelInfo *rri)
|
|
|
|
{
|
|
|
|
CopyMultiInsertBuffer *buffer = rri->ri_CopyMultiInsertBuffer;
|
|
|
|
int nused = buffer->nused;
|
|
|
|
|
|
|
|
Assert(buffer != NULL);
|
|
|
|
Assert(nused < MAX_BUFFERED_TUPLES);
|
|
|
|
|
|
|
|
if (buffer->slots[nused] == NULL)
|
|
|
|
buffer->slots[nused] = table_slot_create(rri->ri_RelationDesc, NULL);
|
|
|
|
return buffer->slots[nused];
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Record the previously reserved TupleTableSlot that was reserved by
|
2019-06-14 02:34:34 +02:00
|
|
|
* CopyMultiInsertInfoNextFreeSlot as being consumed.
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
*/
|
|
|
|
static inline void
|
|
|
|
CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri,
|
|
|
|
TupleTableSlot *slot, int tuplen, uint64 lineno)
|
|
|
|
{
|
|
|
|
CopyMultiInsertBuffer *buffer = rri->ri_CopyMultiInsertBuffer;
|
|
|
|
|
|
|
|
Assert(buffer != NULL);
|
|
|
|
Assert(slot == buffer->slots[buffer->nused]);
|
|
|
|
|
|
|
|
/* Store the line number so we can properly report any errors later */
|
|
|
|
buffer->linenos[buffer->nused] = lineno;
|
|
|
|
|
|
|
|
/* Record this slot as being used */
|
|
|
|
buffer->nused++;
|
|
|
|
|
|
|
|
/* Update how many tuples are stored and their size */
|
|
|
|
miinfo->bufferedTuples++;
|
|
|
|
miinfo->bufferedBytes += tuplen;
|
|
|
|
}
|
|
|
|
|
2000-12-02 21:49:24 +01:00
|
|
|
/*
|
|
|
|
* Copy FROM file to relation.
|
|
|
|
*/
|
2017-03-23 13:36:36 +01:00
|
|
|
uint64
|
2005-08-06 22:41:58 +02:00
|
|
|
CopyFrom(CopyState cstate)
|
1996-07-09 08:22:35 +02:00
|
|
|
{
|
2000-11-12 01:37:02 +01:00
|
|
|
ResultRelInfo *resultRelInfo;
|
2018-08-01 10:23:09 +02:00
|
|
|
ResultRelInfo *target_resultRelInfo;
|
2018-11-16 18:54:15 +01:00
|
|
|
ResultRelInfo *prevResultRelInfo = NULL;
|
2001-03-22 05:01:46 +01:00
|
|
|
EState *estate = CreateExecutorState(); /* for ExecConstraints() */
|
2018-04-07 01:16:11 +02:00
|
|
|
ModifyTableState *mtstate;
|
2011-02-16 03:19:11 +01:00
|
|
|
ExprContext *econtext;
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
TupleTableSlot *singleslot = NULL;
|
2002-08-02 20:15:10 +02:00
|
|
|
MemoryContext oldcontext = CurrentMemoryContext;
|
2011-11-09 09:54:41 +01:00
|
|
|
|
2018-08-01 10:23:09 +02:00
|
|
|
PartitionTupleRouting *proute = NULL;
|
2012-11-12 14:10:24 +01:00
|
|
|
ErrorContextCallback errcallback;
|
2007-11-30 22:22:54 +01:00
|
|
|
CommandId mycid = GetCurrentCommandId(true);
|
2019-05-24 01:25:48 +02:00
|
|
|
int ti_options = 0; /* start with default options for insert */
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
BulkInsertState bistate = NULL;
|
2018-08-01 10:23:09 +02:00
|
|
|
CopyInsertMethod insertMethod;
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
CopyMultiInsertInfo multiInsertInfo = {0}; /* pacify compiler */
|
2011-02-16 03:19:11 +01:00
|
|
|
uint64 processed = 0;
|
2018-08-01 10:23:09 +02:00
|
|
|
bool has_before_insert_row_trig;
|
|
|
|
bool has_instead_insert_row_trig;
|
|
|
|
bool leafpart_use_multi_insert = false;
|
2012-06-10 21:20:04 +02:00
|
|
|
|
2006-08-31 01:34:22 +02:00
|
|
|
Assert(cstate->rel);
|
|
|
|
|
2016-11-10 20:13:43 +01:00
|
|
|
/*
|
2018-04-07 01:16:11 +02:00
|
|
|
* The target must be a plain, foreign, or partitioned relation, or have
|
|
|
|
* an INSTEAD OF INSERT row trigger. (Currently, such triggers are only
|
|
|
|
* allowed on views, so we only hint about them in the view case.)
|
2016-11-10 20:13:43 +01:00
|
|
|
*/
|
|
|
|
if (cstate->rel->rd_rel->relkind != RELKIND_RELATION &&
|
2018-04-07 01:16:11 +02:00
|
|
|
cstate->rel->rd_rel->relkind != RELKIND_FOREIGN_TABLE &&
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
cstate->rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE &&
|
2016-11-10 20:13:43 +01:00
|
|
|
!(cstate->rel->trigdesc &&
|
|
|
|
cstate->rel->trigdesc->trig_insert_instead_row))
|
2006-08-31 01:34:22 +02:00
|
|
|
{
|
|
|
|
if (cstate->rel->rd_rel->relkind == RELKIND_VIEW)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("cannot copy to view \"%s\"",
|
2016-11-10 20:13:43 +01:00
|
|
|
RelationGetRelationName(cstate->rel)),
|
|
|
|
errhint("To enable copying to a view, provide an INSTEAD OF INSERT trigger.")));
|
2013-03-04 01:23:31 +01:00
|
|
|
else if (cstate->rel->rd_rel->relkind == RELKIND_MATVIEW)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("cannot copy to materialized view \"%s\"",
|
|
|
|
RelationGetRelationName(cstate->rel))));
|
2006-08-31 01:34:22 +02:00
|
|
|
else if (cstate->rel->rd_rel->relkind == RELKIND_SEQUENCE)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("cannot copy to sequence \"%s\"",
|
|
|
|
RelationGetRelationName(cstate->rel))));
|
|
|
|
else
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("cannot copy to non-table relation \"%s\"",
|
|
|
|
RelationGetRelationName(cstate->rel))));
|
|
|
|
}
|
|
|
|
|
Skip WAL for new relfilenodes, under wal_level=minimal.
Until now, only selected bulk operations (e.g. COPY) did this. If a
given relfilenode received both a WAL-skipping COPY and a WAL-logged
operation (e.g. INSERT), recovery could lose tuples from the COPY. See
src/backend/access/transam/README section "Skipping WAL for New
RelFileNode" for the new coding rules. Maintainers of table access
methods should examine that section.
To maintain data durability, just before commit, we choose between an
fsync of the relfilenode and copying its contents to WAL. A new GUC,
wal_skip_threshold, guides that choice. If this change slows a workload
that creates small, permanent relfilenodes under wal_level=minimal, try
adjusting wal_skip_threshold. Users setting a timeout on COMMIT may
need to adjust that timeout, and log_min_duration_statement analysis
will reflect time consumption moving to COMMIT from commands like COPY.
Internally, this requires a reliable determination of whether
RollbackAndReleaseCurrentSubTransaction() would unlink a relation's
current relfilenode. Introduce rd_firstRelfilenodeSubid. Amend the
specification of rd_createSubid such that the field is zero when a new
rel has an old rd_node. Make relcache.c retain entries for certain
dropped relations until end of transaction.
Bump XLOG_PAGE_MAGIC, since this introduces XLOG_GIST_ASSIGN_LSN.
Future servers accept older WAL, so this bump is discretionary.
Kyotaro Horiguchi, reviewed (in earlier, similar versions) by Robert
Haas. Heikki Linnakangas and Michael Paquier implemented earlier
designs that materially clarified the problem. Reviewed, in earlier
designs, by Andrew Dunstan, Andres Freund, Alvaro Herrera, Tom Lane,
Fujii Masao, and Simon Riggs. Reported by Martijn van Oosterhout.
Discussion: https://postgr.es/m/20150702220524.GA9392@svana.org
2020-04-04 21:25:34 +02:00
|
|
|
/*
|
|
|
|
* If the target file is new-in-transaction, we assume that checking FSM
|
|
|
|
* for free space is a waste of time. This could possibly be wrong, but
|
|
|
|
* it's unlikely.
|
2007-03-29 02:15:39 +02:00
|
|
|
*/
|
2019-01-04 18:34:18 +01:00
|
|
|
if (RELKIND_HAS_STORAGE(cstate->rel->rd_rel->relkind) &&
|
2018-11-19 15:16:28 +01:00
|
|
|
(cstate->rel->rd_createSubid != InvalidSubTransactionId ||
|
Skip WAL for new relfilenodes, under wal_level=minimal.
Until now, only selected bulk operations (e.g. COPY) did this. If a
given relfilenode received both a WAL-skipping COPY and a WAL-logged
operation (e.g. INSERT), recovery could lose tuples from the COPY. See
src/backend/access/transam/README section "Skipping WAL for New
RelFileNode" for the new coding rules. Maintainers of table access
methods should examine that section.
To maintain data durability, just before commit, we choose between an
fsync of the relfilenode and copying its contents to WAL. A new GUC,
wal_skip_threshold, guides that choice. If this change slows a workload
that creates small, permanent relfilenodes under wal_level=minimal, try
adjusting wal_skip_threshold. Users setting a timeout on COMMIT may
need to adjust that timeout, and log_min_duration_statement analysis
will reflect time consumption moving to COMMIT from commands like COPY.
Internally, this requires a reliable determination of whether
RollbackAndReleaseCurrentSubTransaction() would unlink a relation's
current relfilenode. Introduce rd_firstRelfilenodeSubid. Amend the
specification of rd_createSubid such that the field is zero when a new
rel has an old rd_node. Make relcache.c retain entries for certain
dropped relations until end of transaction.
Bump XLOG_PAGE_MAGIC, since this introduces XLOG_GIST_ASSIGN_LSN.
Future servers accept older WAL, so this bump is discretionary.
Kyotaro Horiguchi, reviewed (in earlier, similar versions) by Robert
Haas. Heikki Linnakangas and Michael Paquier implemented earlier
designs that materially clarified the problem. Reviewed, in earlier
designs, by Andrew Dunstan, Andres Freund, Alvaro Herrera, Tom Lane,
Fujii Masao, and Simon Riggs. Reported by Martijn van Oosterhout.
Discussion: https://postgr.es/m/20150702220524.GA9392@svana.org
2020-04-04 21:25:34 +02:00
|
|
|
cstate->rel->rd_firstRelfilenodeSubid != InvalidSubTransactionId))
|
2019-04-01 23:41:42 +02:00
|
|
|
ti_options |= TABLE_INSERT_SKIP_FSM;
|
2012-12-01 13:54:20 +01:00
|
|
|
|
2013-02-02 18:56:14 +01:00
|
|
|
/*
|
2013-05-29 22:58:43 +02:00
|
|
|
* Optimize if new relfilenode was created in this subxact or one of its
|
|
|
|
* committed children and we won't see those rows later as part of an
|
2017-11-05 18:25:52 +01:00
|
|
|
* earlier scan or command. The subxact test ensures that if this subxact
|
|
|
|
* aborts then the frozen rows won't be visible after xact cleanup. Note
|
2013-05-29 22:58:43 +02:00
|
|
|
* that the stronger test of exactly which subtransaction created it is
|
2017-11-05 18:25:52 +01:00
|
|
|
* crucial for correctness of this optimization. The test for an earlier
|
|
|
|
* scan or command tolerates false negatives. FREEZE causes other sessions
|
|
|
|
* to see rows they would not see under MVCC, and a false negative merely
|
|
|
|
* spreads that anomaly to the current session.
|
2013-02-02 18:56:14 +01:00
|
|
|
*/
|
|
|
|
if (cstate->freeze)
|
|
|
|
{
|
2018-11-19 15:16:28 +01:00
|
|
|
/*
|
|
|
|
* We currently disallow COPY FREEZE on partitioned tables. The
|
|
|
|
* reason for this is that we've simply not yet opened the partitions
|
|
|
|
* to determine if the optimization can be applied to them. We could
|
|
|
|
* go and open them all here, but doing so may be quite a costly
|
|
|
|
* overhead for small copies. In any case, we may just end up routing
|
|
|
|
* tuples to a small number of partitions. It seems better just to
|
|
|
|
* raise an ERROR for partitioned tables.
|
|
|
|
*/
|
|
|
|
if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
|
|
|
|
{
|
|
|
|
ereport(ERROR,
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
|
2019-05-17 00:50:56 +02:00
|
|
|
errmsg("cannot perform COPY FREEZE on a partitioned table")));
|
2018-11-19 15:16:28 +01:00
|
|
|
}
|
|
|
|
|
2017-11-05 18:25:52 +01:00
|
|
|
/*
|
|
|
|
* Tolerate one registration for the benefit of FirstXactSnapshot.
|
|
|
|
* Scan-bearing queries generally create at least two registrations,
|
|
|
|
* though relying on that is fragile, as is ignoring ActiveSnapshot.
|
|
|
|
* Clear CatalogSnapshot to avoid counting its registration. We'll
|
|
|
|
* still detect ongoing catalog scans, each of which separately
|
|
|
|
* registers the snapshot it uses.
|
|
|
|
*/
|
|
|
|
InvalidateCatalogSnapshot();
|
2013-02-02 18:56:14 +01:00
|
|
|
if (!ThereAreNoPriorRegisteredSnapshots() || !ThereAreNoReadyPortals())
|
|
|
|
ereport(ERROR,
|
2015-08-03 05:49:19 +02:00
|
|
|
(errcode(ERRCODE_INVALID_TRANSACTION_STATE),
|
2019-05-17 00:50:56 +02:00
|
|
|
errmsg("cannot perform COPY FREEZE because of prior transaction activity")));
|
2013-01-26 19:33:24 +01:00
|
|
|
|
2013-02-02 18:56:14 +01:00
|
|
|
if (cstate->rel->rd_createSubid != GetCurrentSubTransactionId() &&
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
cstate->rel->rd_newRelfilenodeSubid != GetCurrentSubTransactionId())
|
2013-02-02 18:56:14 +01:00
|
|
|
ereport(ERROR,
|
2015-08-03 05:49:19 +02:00
|
|
|
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
|
2019-05-17 00:50:56 +02:00
|
|
|
errmsg("cannot perform COPY FREEZE because the table was not created or truncated in the current subtransaction")));
|
2013-02-02 18:56:14 +01:00
|
|
|
|
2019-04-01 23:41:42 +02:00
|
|
|
ti_options |= TABLE_INSERT_FROZEN;
|
2007-03-29 02:15:39 +02:00
|
|
|
}
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/*
|
|
|
|
* We need a ResultRelInfo so we can use the regular executor's
|
|
|
|
* index-entry-making machinery. (There used to be a huge amount of code
|
|
|
|
* here that basically duplicated execUtils.c ...)
|
|
|
|
*/
|
|
|
|
resultRelInfo = makeNode(ResultRelInfo);
|
2013-03-10 19:14:53 +01:00
|
|
|
InitResultRelInfo(resultRelInfo,
|
|
|
|
cstate->rel,
|
2018-10-04 20:03:37 +02:00
|
|
|
1, /* must match rel's position in range_table */
|
2017-01-04 20:36:34 +01:00
|
|
|
NULL,
|
2013-03-10 19:14:53 +01:00
|
|
|
0);
|
2018-08-01 10:23:09 +02:00
|
|
|
target_resultRelInfo = resultRelInfo;
|
1997-09-07 07:04:48 +02:00
|
|
|
|
2018-04-07 01:16:11 +02:00
|
|
|
/* Verify the named relation is a valid target for INSERT */
|
|
|
|
CheckValidResultRel(resultRelInfo, CMD_INSERT);
|
|
|
|
|
Add support for INSERT ... ON CONFLICT DO NOTHING/UPDATE.
The newly added ON CONFLICT clause allows to specify an alternative to
raising a unique or exclusion constraint violation error when inserting.
ON CONFLICT refers to constraints that can either be specified using a
inference clause (by specifying the columns of a unique constraint) or
by naming a unique or exclusion constraint. DO NOTHING avoids the
constraint violation, without touching the pre-existing row. DO UPDATE
SET ... [WHERE ...] updates the pre-existing tuple, and has access to
both the tuple proposed for insertion and the existing tuple; the
optional WHERE clause can be used to prevent an update from being
executed. The UPDATE SET and WHERE clauses have access to the tuple
proposed for insertion using the "magic" EXCLUDED alias, and to the
pre-existing tuple using the table name or its alias.
This feature is often referred to as upsert.
This is implemented using a new infrastructure called "speculative
insertion". It is an optimistic variant of regular insertion that first
does a pre-check for existing tuples and then attempts an insert. If a
violating tuple was inserted concurrently, the speculatively inserted
tuple is deleted and a new attempt is made. If the pre-check finds a
matching tuple the alternative DO NOTHING or DO UPDATE action is taken.
If the insertion succeeds without detecting a conflict, the tuple is
deemed inserted.
To handle the possible ambiguity between the excluded alias and a table
named excluded, and for convenience with long relation names, INSERT
INTO now can alias its target table.
Bumps catversion as stored rules change.
Author: Peter Geoghegan, with significant contributions from Heikki
Linnakangas and Andres Freund. Testing infrastructure by Jeff Janes.
Reviewed-By: Heikki Linnakangas, Andres Freund, Robert Haas, Simon Riggs,
Dean Rasheed, Stephen Frost and many others.
2015-05-08 05:31:36 +02:00
|
|
|
ExecOpenIndices(resultRelInfo, false);
|
2000-07-15 00:18:02 +02:00
|
|
|
|
2000-11-12 01:37:02 +01:00
|
|
|
estate->es_result_relations = resultRelInfo;
|
|
|
|
estate->es_num_result_relations = 1;
|
|
|
|
estate->es_result_relation_info = resultRelInfo;
|
2018-10-04 21:48:17 +02:00
|
|
|
|
|
|
|
ExecInitRangeTable(estate, cstate->range_table);
|
2000-07-15 00:18:02 +02:00
|
|
|
|
2018-04-07 01:16:11 +02:00
|
|
|
/*
|
|
|
|
* Set up a ModifyTableState so we can let FDW(s) init themselves for
|
|
|
|
* foreign-table result relation(s).
|
|
|
|
*/
|
|
|
|
mtstate = makeNode(ModifyTableState);
|
|
|
|
mtstate->ps.plan = NULL;
|
|
|
|
mtstate->ps.state = estate;
|
|
|
|
mtstate->operation = CMD_INSERT;
|
|
|
|
mtstate->resultRelInfo = estate->es_result_relations;
|
|
|
|
|
|
|
|
if (resultRelInfo->ri_FdwRoutine != NULL &&
|
|
|
|
resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
|
|
|
|
resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
|
|
|
|
resultRelInfo);
|
|
|
|
|
Fix SQL-spec incompatibilities in new transition table feature.
The standard says that all changes of the same kind (insert, update, or
delete) caused in one table by a single SQL statement should be reported
in a single transition table; and by that, they mean to include foreign key
enforcement actions cascading from the statement's direct effects. It's
also reasonable to conclude that if the standard had wCTEs, they would say
that effects of wCTEs applying to the same table as each other or the outer
statement should be merged into one transition table. We weren't doing it
like that.
Hence, arrange to merge tuples from multiple update actions into a single
transition table as much as we can. There is a problem, which is that if
the firing of FK enforcement triggers and after-row triggers with
transition tables is interspersed, we might need to report more tuples
after some triggers have already seen the transition table. It seems like
a bad idea for the transition table to be mutable between trigger calls.
There's no good way around this without a major redesign of the FK logic,
so for now, resolve it by opening a new transition table each time this
happens.
Also, ensure that AFTER STATEMENT triggers fire just once per statement,
or once per transition table when we're forced to make more than one.
Previous versions of Postgres have allowed each FK enforcement query
to cause an additional firing of the AFTER STATEMENT triggers for the
referencing table, but that's certainly not per spec. (We're still
doing multiple firings of BEFORE STATEMENT triggers, though; is that
something worth changing?)
Also, forbid using transition tables with column-specific UPDATE triggers.
The spec requires such transition tables to show only the tuples for which
the UPDATE trigger would have fired, which means maintaining multiple
transition tables or else somehow filtering the contents at readout.
Maybe someday we'll bother to support that option, but it looks like a
lot of trouble for a marginal feature.
The transition tables are now managed by the AfterTriggers data structures,
rather than being directly the responsibility of ModifyTable nodes. This
removes a subtransaction-lifespan memory leak introduced by my previous
band-aid patch 3c4359521.
In passing, refactor the AfterTriggers data structures to reduce the
management overhead for them, by using arrays of structs rather than
several parallel arrays for per-query-level and per-subtransaction state.
I failed to resist the temptation to do some copy-editing on the SGML
docs about triggers, above and beyond merely documenting the effects
of this patch.
Back-patch to v10, because we don't want the semantics of transition
tables to change post-release.
Patch by me, with help and review from Thomas Munro.
Discussion: https://postgr.es/m/20170909064853.25630.12825@wrigleys.postgresql.org
2017-09-16 19:20:32 +02:00
|
|
|
/* Prepare to catch AFTER triggers. */
|
|
|
|
AfterTriggerBeginQuery();
|
|
|
|
|
2017-08-18 19:01:05 +02:00
|
|
|
/*
|
|
|
|
* If there are any triggers with transition tables on the named relation,
|
|
|
|
* we need to be prepared to capture transition tuples.
|
2018-11-16 18:54:15 +01:00
|
|
|
*
|
|
|
|
* Because partition tuple routing would like to know about whether
|
|
|
|
* transition capture is active, we also set it in mtstate, which is
|
|
|
|
* passed to ExecFindPartition() below.
|
2017-08-18 19:01:05 +02:00
|
|
|
*/
|
2018-11-16 18:54:15 +01:00
|
|
|
cstate->transition_capture = mtstate->mt_transition_capture =
|
Fix SQL-spec incompatibilities in new transition table feature.
The standard says that all changes of the same kind (insert, update, or
delete) caused in one table by a single SQL statement should be reported
in a single transition table; and by that, they mean to include foreign key
enforcement actions cascading from the statement's direct effects. It's
also reasonable to conclude that if the standard had wCTEs, they would say
that effects of wCTEs applying to the same table as each other or the outer
statement should be merged into one transition table. We weren't doing it
like that.
Hence, arrange to merge tuples from multiple update actions into a single
transition table as much as we can. There is a problem, which is that if
the firing of FK enforcement triggers and after-row triggers with
transition tables is interspersed, we might need to report more tuples
after some triggers have already seen the transition table. It seems like
a bad idea for the transition table to be mutable between trigger calls.
There's no good way around this without a major redesign of the FK logic,
so for now, resolve it by opening a new transition table each time this
happens.
Also, ensure that AFTER STATEMENT triggers fire just once per statement,
or once per transition table when we're forced to make more than one.
Previous versions of Postgres have allowed each FK enforcement query
to cause an additional firing of the AFTER STATEMENT triggers for the
referencing table, but that's certainly not per spec. (We're still
doing multiple firings of BEFORE STATEMENT triggers, though; is that
something worth changing?)
Also, forbid using transition tables with column-specific UPDATE triggers.
The spec requires such transition tables to show only the tuples for which
the UPDATE trigger would have fired, which means maintaining multiple
transition tables or else somehow filtering the contents at readout.
Maybe someday we'll bother to support that option, but it looks like a
lot of trouble for a marginal feature.
The transition tables are now managed by the AfterTriggers data structures,
rather than being directly the responsibility of ModifyTable nodes. This
removes a subtransaction-lifespan memory leak introduced by my previous
band-aid patch 3c4359521.
In passing, refactor the AfterTriggers data structures to reduce the
management overhead for them, by using arrays of structs rather than
several parallel arrays for per-query-level and per-subtransaction state.
I failed to resist the temptation to do some copy-editing on the SGML
docs about triggers, above and beyond merely documenting the effects
of this patch.
Back-patch to v10, because we don't want the semantics of transition
tables to change post-release.
Patch by me, with help and review from Thomas Munro.
Discussion: https://postgr.es/m/20170909064853.25630.12825@wrigleys.postgresql.org
2017-09-16 19:20:32 +02:00
|
|
|
MakeTransitionCaptureState(cstate->rel->trigdesc,
|
|
|
|
RelationGetRelid(cstate->rel),
|
|
|
|
CMD_INSERT);
|
2017-08-18 19:01:05 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If the named relation is a partitioned table, initialize state for
|
|
|
|
* CopyFrom tuple routing.
|
|
|
|
*/
|
|
|
|
if (cstate->rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
|
Allow ATTACH PARTITION with only ShareUpdateExclusiveLock.
We still require AccessExclusiveLock on the partition itself, because
otherwise an insert that violates the newly-imposed partition
constraint could be in progress at the same time that we're changing
that constraint; only the lock level on the parent relation is
weakened.
To make this safe, we have to cope with (at least) three separate
problems. First, relevant DDL might commit while we're in the process
of building a PartitionDesc. If so, find_inheritance_children() might
see a new partition while the RELOID system cache still has the old
partition bound cached, and even before invalidation messages have
been queued. To fix that, if we see that the pg_class tuple seems to
be missing or to have a null relpartbound, refetch the value directly
from the table. We can't get the wrong value, because DETACH PARTITION
still requires AccessExclusiveLock throughout; if we ever want to
change that, this will need more thought. In testing, I found it quite
difficult to hit even the null-relpartbound case; the race condition
is extremely tight, but the theoretical risk is there.
Second, successive calls to RelationGetPartitionDesc might not return
the same answer. The query planner will get confused if lookup up the
PartitionDesc for a particular relation does not return a consistent
answer for the entire duration of query planning. Likewise, query
execution will get confused if the same relation seems to have a
different PartitionDesc at different times. Invent a new
PartitionDirectory concept and use it to ensure consistency. This
ensures that a single invocation of either the planner or the executor
sees the same view of the PartitionDesc from beginning to end, but it
does not guarantee that the planner and the executor see the same
view. Since this allows pointers to old PartitionDesc entries to
survive even after a relcache rebuild, also postpone removing the old
PartitionDesc entry until we're certain no one is using it.
For the most part, it seems to be OK for the planner and executor to
have different views of the PartitionDesc, because the executor will
just ignore any concurrently added partitions which were unknown at
plan time; those partitions won't be part of the inheritance
expansion, but invalidation messages will trigger replanning at some
point. Normally, this happens by the time the very next command is
executed, but if the next command acquires no locks and executes a
prepared query, it can manage not to notice until a new transaction is
started. We might want to tighten that up, but it's material for a
separate patch. There would still be a small window where a query
that started just after an ATTACH PARTITION command committed might
fail to notice its results -- but only if the command starts before
the commit has been acknowledged to the user. All in all, the warts
here around serializability seem small enough to be worth accepting
for the considerable advantage of being able to add partitions without
a full table lock.
Although in general the consequences of new partitions showing up
between planning and execution are limited to the query not noticing
the new partitions, run-time partition pruning will get confused in
that case, so that's the third problem that this patch fixes.
Run-time partition pruning assumes that indexes into the PartitionDesc
are stable between planning and execution. So, add code so that if
new partitions are added between plan time and execution time, the
indexes stored in the subplan_map[] and subpart_map[] arrays within
the plan's PartitionedRelPruneInfo get adjusted accordingly. There
does not seem to be a simple way to generalize this scheme to cope
with partitions that are removed, mostly because they could then get
added back again with different bounds, but it works OK for added
partitions.
This code does not try to ensure that every backend participating in
a parallel query sees the same view of the PartitionDesc. That
currently doesn't matter, because we never pass PartitionDesc
indexes between backends. Each backend will ignore the concurrently
added partitions which it notices, and it doesn't matter if different
backends are ignoring different sets of concurrently added partitions.
If in the future that matters, for example because we allow writes in
parallel query and want all participants to do tuple routing to the same
set of partitions, the PartitionDirectory concept could be improved to
share PartitionDescs across backends. There is a draft patch to
serialize and restore PartitionDescs on the thread where this patch
was discussed, which may be a useful place to start.
Patch by me. Thanks to Alvaro Herrera, David Rowley, Simon Riggs,
Amit Langote, and Michael Paquier for discussion, and to Alvaro
Herrera for some review.
Discussion: http://postgr.es/m/CA+Tgmobt2upbSocvvDej3yzokd7AkiT+PvgFH+a9-5VV1oJNSQ@mail.gmail.com
Discussion: http://postgr.es/m/CA+TgmoZE0r9-cyA-aY6f8WFEROaDLLL7Vf81kZ8MtFCkxpeQSw@mail.gmail.com
Discussion: http://postgr.es/m/CA+TgmoY13KQZF-=HNTrt9UYWYx3_oYOQpu9ioNT49jGgiDpUEA@mail.gmail.com
2019-03-07 17:13:12 +01:00
|
|
|
proute = ExecSetupPartitionTupleRouting(estate, NULL, cstate->rel);
|
2017-08-18 19:01:05 +02:00
|
|
|
|
2019-01-19 23:48:16 +01:00
|
|
|
if (cstate->whereClause)
|
|
|
|
cstate->qualexpr = ExecInitQual(castNode(List, cstate->whereClause),
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
&mtstate->ps);
|
2019-01-19 23:48:16 +01:00
|
|
|
|
2011-11-09 09:54:41 +01:00
|
|
|
/*
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
* It's generally more efficient to prepare a bunch of tuples for
|
|
|
|
* insertion, and insert them in one table_multi_insert() call, than call
|
2019-05-24 01:25:48 +02:00
|
|
|
* table_tuple_insert() separately for every tuple. However, there are a
|
|
|
|
* number of reasons why we might not be able to do this. These are
|
|
|
|
* explained below.
|
2011-11-09 09:54:41 +01:00
|
|
|
*/
|
2018-08-01 10:23:09 +02:00
|
|
|
if (resultRelInfo->ri_TrigDesc != NULL &&
|
|
|
|
(resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
|
|
|
|
resultRelInfo->ri_TrigDesc->trig_insert_instead_row))
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Can't support multi-inserts when there are any BEFORE/INSTEAD OF
|
|
|
|
* triggers on the table. Such triggers might query the table we're
|
|
|
|
* inserting into and act differently if the tuples that have already
|
|
|
|
* been processed and prepared for insertion are not there.
|
|
|
|
*/
|
|
|
|
insertMethod = CIM_SINGLE;
|
|
|
|
}
|
|
|
|
else if (proute != NULL && resultRelInfo->ri_TrigDesc != NULL &&
|
|
|
|
resultRelInfo->ri_TrigDesc->trig_insert_new_table)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* For partitioned tables we can't support multi-inserts when there
|
|
|
|
* are any statement level insert triggers. It might be possible to
|
|
|
|
* allow partitioned tables with such triggers in the future, but for
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
* now, CopyMultiInsertInfoFlush expects that any before row insert
|
|
|
|
* and statement level insert triggers are on the same relation.
|
2018-08-01 10:23:09 +02:00
|
|
|
*/
|
|
|
|
insertMethod = CIM_SINGLE;
|
|
|
|
}
|
|
|
|
else if (resultRelInfo->ri_FdwRoutine != NULL ||
|
|
|
|
cstate->volatile_defexprs)
|
2011-11-09 09:54:41 +01:00
|
|
|
{
|
2018-08-01 10:23:09 +02:00
|
|
|
/*
|
|
|
|
* Can't support multi-inserts to foreign tables or if there are any
|
|
|
|
* volatile default expressions in the table. Similarly to the
|
|
|
|
* trigger case above, such expressions may query the table we're
|
|
|
|
* inserting into.
|
|
|
|
*
|
|
|
|
* Note: It does not matter if any partitions have any volatile
|
|
|
|
* default expressions as we use the defaults from the target of the
|
|
|
|
* COPY command.
|
|
|
|
*/
|
|
|
|
insertMethod = CIM_SINGLE;
|
2011-11-09 09:54:41 +01:00
|
|
|
}
|
2019-01-22 23:11:17 +01:00
|
|
|
else if (contain_volatile_functions(cstate->whereClause))
|
2019-01-19 23:48:16 +01:00
|
|
|
{
|
|
|
|
/*
|
2019-05-26 14:58:18 +02:00
|
|
|
* Can't support multi-inserts if there are any volatile function
|
2019-01-19 23:48:16 +01:00
|
|
|
* expressions in WHERE clause. Similarly to the trigger case above,
|
|
|
|
* such expressions may query the table we're inserting into.
|
|
|
|
*/
|
|
|
|
insertMethod = CIM_SINGLE;
|
|
|
|
}
|
2011-11-09 09:54:41 +01:00
|
|
|
else
|
|
|
|
{
|
2018-08-01 10:23:09 +02:00
|
|
|
/*
|
|
|
|
* For partitioned tables, we may still be able to perform bulk
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
* inserts. However, the possibility of this depends on which types
|
2018-08-01 10:23:09 +02:00
|
|
|
* of triggers exist on the partition. We must disable bulk inserts
|
|
|
|
* if the partition is a foreign table or it has any before row insert
|
|
|
|
* or insert instead triggers (same as we checked above for the parent
|
|
|
|
* table). Since the partition's resultRelInfos are initialized only
|
|
|
|
* when we actually need to insert the first tuple into them, we must
|
|
|
|
* have the intermediate insert method of CIM_MULTI_CONDITIONAL to
|
|
|
|
* flag that we must later determine if we can use bulk-inserts for
|
|
|
|
* the partition being inserted into.
|
|
|
|
*/
|
|
|
|
if (proute)
|
|
|
|
insertMethod = CIM_MULTI_CONDITIONAL;
|
|
|
|
else
|
|
|
|
insertMethod = CIM_MULTI;
|
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
CopyMultiInsertInfoInit(&multiInsertInfo, resultRelInfo, cstate,
|
|
|
|
estate, mycid, ti_options);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If not using batch mode (which allocates slots as needed) set up a
|
|
|
|
* tuple slot too. When inserting into a partitioned table, we also need
|
|
|
|
* one, even if we might batch insert, to read the tuple in the root
|
|
|
|
* partition's form.
|
|
|
|
*/
|
|
|
|
if (insertMethod == CIM_SINGLE || insertMethod == CIM_MULTI_CONDITIONAL)
|
|
|
|
{
|
|
|
|
singleslot = table_slot_create(resultRelInfo->ri_RelationDesc,
|
|
|
|
&estate->es_tupleTable);
|
|
|
|
bistate = GetBulkInsertState();
|
2011-11-09 09:54:41 +01:00
|
|
|
}
|
|
|
|
|
2018-08-01 10:23:09 +02:00
|
|
|
has_before_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
|
|
|
|
resultRelInfo->ri_TrigDesc->trig_insert_before_row);
|
|
|
|
|
|
|
|
has_instead_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
|
|
|
|
resultRelInfo->ri_TrigDesc->trig_insert_instead_row);
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/*
|
2012-04-24 04:43:09 +02:00
|
|
|
* Check BEFORE STATEMENT insertion triggers. It's debatable whether we
|
2011-02-16 03:19:11 +01:00
|
|
|
* should do this for COPY, since it's not really an "INSERT" statement as
|
|
|
|
* such. However, executing these triggers maintains consistency with the
|
|
|
|
* EACH ROW triggers that we already fire on COPY.
|
|
|
|
*/
|
|
|
|
ExecBSInsertTriggers(estate, resultRelInfo);
|
|
|
|
|
2002-12-15 17:17:59 +01:00
|
|
|
econtext = GetPerTupleExprContext(estate);
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/* Set up callback to identify error line number */
|
2012-11-12 14:10:24 +01:00
|
|
|
errcallback.callback = CopyFromErrorCallback;
|
|
|
|
errcallback.arg = (void *) cstate;
|
|
|
|
errcallback.previous = error_context_stack;
|
|
|
|
error_context_stack = &errcallback;
|
2011-02-16 03:19:11 +01:00
|
|
|
|
|
|
|
for (;;)
|
|
|
|
{
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
TupleTableSlot *myslot;
|
2011-02-16 03:19:11 +01:00
|
|
|
bool skip_tuple;
|
|
|
|
|
|
|
|
CHECK_FOR_INTERRUPTS();
|
|
|
|
|
Separate per-batch and per-tuple memory contexts in COPY
In batching mode, COPY was using the same (per-tuple) memory context for
allocations with longer lifetime. This was confusing but harmless, until
commit 31f3817402 added COPY FROM ... WHERE feature, introducing a risk
of memory leak.
The "per-tuple" memory context was reset only when starting new batch,
but as the rows may be filtered out by the WHERE clauses, that may not
happen at all. The WHERE clause however has to be evaluated for all
rows, before filtering them out.
This commit separates the per-tuple and per-batch contexts, removing the
ambiguity. Expressions (both defaults and WHERE clause) are evaluated
in the per-tuple context, while tuples are formed in the batch context.
This allows resetting the contexts at appropriate times.
The main complexity is related to partitioning, in which case we need to
reset the batch context after forming the tuple (which happens before
routing to leaf partition). Instead of switching between two contexts
as before, we simply copy the last tuple aside, reset the context and
then copy the tuple back. The performance impact is negligible, and
juggling with two contexts is not free either.
Discussion: https://www.postgresql.org/message-id/flat/CALAY4q_DdpWDuB5-Zyi-oTtO2uSk8pmy+dupiRe3AvAc++1imA@mail.gmail.com
2019-01-29 00:00:47 +01:00
|
|
|
/*
|
|
|
|
* Reset the per-tuple exprcontext. We do this after every tuple, to
|
|
|
|
* clean-up after expression evaluations etc.
|
|
|
|
*/
|
|
|
|
ResetPerTupleExprContext(estate);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* select slot to (initially) load row into */
|
|
|
|
if (insertMethod == CIM_SINGLE || proute)
|
|
|
|
{
|
|
|
|
myslot = singleslot;
|
|
|
|
Assert(myslot != NULL);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
Assert(resultRelInfo == target_resultRelInfo);
|
|
|
|
Assert(insertMethod == CIM_MULTI);
|
|
|
|
|
|
|
|
myslot = CopyMultiInsertInfoNextFreeSlot(&multiInsertInfo,
|
|
|
|
resultRelInfo);
|
|
|
|
}
|
|
|
|
|
Separate per-batch and per-tuple memory contexts in COPY
In batching mode, COPY was using the same (per-tuple) memory context for
allocations with longer lifetime. This was confusing but harmless, until
commit 31f3817402 added COPY FROM ... WHERE feature, introducing a risk
of memory leak.
The "per-tuple" memory context was reset only when starting new batch,
but as the rows may be filtered out by the WHERE clauses, that may not
happen at all. The WHERE clause however has to be evaluated for all
rows, before filtering them out.
This commit separates the per-tuple and per-batch contexts, removing the
ambiguity. Expressions (both defaults and WHERE clause) are evaluated
in the per-tuple context, while tuples are formed in the batch context.
This allows resetting the contexts at appropriate times.
The main complexity is related to partitioning, in which case we need to
reset the batch context after forming the tuple (which happens before
routing to leaf partition). Instead of switching between two contexts
as before, we simply copy the last tuple aside, reset the context and
then copy the tuple back. The performance impact is negligible, and
juggling with two contexts is not free either.
Discussion: https://www.postgresql.org/message-id/flat/CALAY4q_DdpWDuB5-Zyi-oTtO2uSk8pmy+dupiRe3AvAc++1imA@mail.gmail.com
2019-01-29 00:00:47 +01:00
|
|
|
/*
|
|
|
|
* Switch to per-tuple context before calling NextCopyFrom, which does
|
|
|
|
* evaluate default expressions etc. and requires per-tuple context.
|
|
|
|
*/
|
2011-02-16 03:19:11 +01:00
|
|
|
MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
|
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
ExecClearTuple(myslot);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* Directly store the values/nulls array in the slot */
|
|
|
|
if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull))
|
|
|
|
break;
|
Separate per-batch and per-tuple memory contexts in COPY
In batching mode, COPY was using the same (per-tuple) memory context for
allocations with longer lifetime. This was confusing but harmless, until
commit 31f3817402 added COPY FROM ... WHERE feature, introducing a risk
of memory leak.
The "per-tuple" memory context was reset only when starting new batch,
but as the rows may be filtered out by the WHERE clauses, that may not
happen at all. The WHERE clause however has to be evaluated for all
rows, before filtering them out.
This commit separates the per-tuple and per-batch contexts, removing the
ambiguity. Expressions (both defaults and WHERE clause) are evaluated
in the per-tuple context, while tuples are formed in the batch context.
This allows resetting the contexts at appropriate times.
The main complexity is related to partitioning, in which case we need to
reset the batch context after forming the tuple (which happens before
routing to leaf partition). Instead of switching between two contexts
as before, we simply copy the last tuple aside, reset the context and
then copy the tuple back. The performance impact is negligible, and
juggling with two contexts is not free either.
Discussion: https://www.postgresql.org/message-id/flat/CALAY4q_DdpWDuB5-Zyi-oTtO2uSk8pmy+dupiRe3AvAc++1imA@mail.gmail.com
2019-01-29 00:00:47 +01:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
ExecStoreVirtualTuple(myslot);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
Don't allow system columns in CHECK constraints, except tableoid.
Previously, arbitray system columns could be mentioned in table
constraints, but they were not correctly checked at runtime, because
the values weren't actually set correctly in the tuple. Since it
seems easy enough to initialize the table OID properly, do that,
and continue allowing that column, but disallow the rest unless and
until someone figures out a way to make them work properly.
No back-patch, because this doesn't seem important enough to take the
risk of destabilizing the back branches. In fact, this will pose a
dump-and-reload hazard for those upgrading from previous versions:
constraints that were accepted before but were not correctly enforced
will now either be enforced correctly or not accepted at all. Either
could result in restore failures, but in practice I think very few
users will notice the difference, since the use case is pretty
marginal anyway and few users will be relying on features that have
not historically worked.
Amit Kapila, reviewed by Rushabh Lathia, with doc changes by me.
2013-09-23 19:31:22 +02:00
|
|
|
/*
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
* Constraints and where clause might reference the tableoid column,
|
|
|
|
* so (re-)initialize tts_tableOid before evaluating them.
|
Don't allow system columns in CHECK constraints, except tableoid.
Previously, arbitray system columns could be mentioned in table
constraints, but they were not correctly checked at runtime, because
the values weren't actually set correctly in the tuple. Since it
seems easy enough to initialize the table OID properly, do that,
and continue allowing that column, but disallow the rest unless and
until someone figures out a way to make them work properly.
No back-patch, because this doesn't seem important enough to take the
risk of destabilizing the back branches. In fact, this will pose a
dump-and-reload hazard for those upgrading from previous versions:
constraints that were accepted before but were not correctly enforced
will now either be enforced correctly or not accepted at all. Either
could result in restore failures, but in practice I think very few
users will notice the difference, since the use case is pretty
marginal anyway and few users will be relying on features that have
not historically worked.
Amit Kapila, reviewed by Rushabh Lathia, with doc changes by me.
2013-09-23 19:31:22 +02:00
|
|
|
*/
|
Store table oid and tuple's tid in tuple slots directly.
After the introduction of tuple table slots all table AMs need to
support returning the table oid of the tuple stored in a slot created
by said AM. It does not make sense to re-implement that in every AM,
therefore move handling of table OIDs into the TupleTableSlot
structure itself. It's possible that we, at a later date, might want
to get rid of HeapTupleData.t_tableOid entirely, but doing so before
the abstractions for table AMs are integrated turns out to be too
hard, so delay that for now.
Similarly, every AM needs to support the concept of a tuple
identifier (tid / item pointer) for its tuples. It's quite possible
that we'll generalize the exact form of a tid at a future point (to
allow for things like index organized tables), but for now many parts
of the code know about tids, so there's not much point in abstracting
tids away. Therefore also move into slot (rather than providing API to
set/get the tid associated with the tuple in a slot).
Once table AM includes insert/updating/deleting tuples, the
responsibility to set the correct tid after such an action will move
into that. After that change, code doing such modifications, should
not have to deal with HeapTuples directly anymore.
Author: Andres Freund, Haribabu Kommi and Ashutosh Bapat
Discussion: https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
2019-02-27 03:21:44 +01:00
|
|
|
myslot->tts_tableOid = RelationGetRelid(target_resultRelInfo->ri_RelationDesc);
|
Don't allow system columns in CHECK constraints, except tableoid.
Previously, arbitray system columns could be mentioned in table
constraints, but they were not correctly checked at runtime, because
the values weren't actually set correctly in the tuple. Since it
seems easy enough to initialize the table OID properly, do that,
and continue allowing that column, but disallow the rest unless and
until someone figures out a way to make them work properly.
No back-patch, because this doesn't seem important enough to take the
risk of destabilizing the back branches. In fact, this will pose a
dump-and-reload hazard for those upgrading from previous versions:
constraints that were accepted before but were not correctly enforced
will now either be enforced correctly or not accepted at all. Either
could result in restore failures, but in practice I think very few
users will notice the difference, since the use case is pretty
marginal anyway and few users will be relying on features that have
not historically worked.
Amit Kapila, reviewed by Rushabh Lathia, with doc changes by me.
2013-09-23 19:31:22 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/* Triggers and stuff need to be invoked in query context. */
|
|
|
|
MemoryContextSwitchTo(oldcontext);
|
|
|
|
|
2019-01-19 23:48:16 +01:00
|
|
|
if (cstate->whereClause)
|
|
|
|
{
|
|
|
|
econtext->ecxt_scantuple = myslot;
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* Skip items that don't match COPY's WHERE clause */
|
2019-01-19 23:48:16 +01:00
|
|
|
if (!ExecQual(cstate->qualexpr, econtext))
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* Determine the partition to insert the tuple into */
|
2018-08-01 10:23:09 +02:00
|
|
|
if (proute)
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
{
|
Use slots more widely in tuple mapping code and make naming more consistent.
It's inefficient to use a single slot for mapping between tuple
descriptors for multiple tuples, as previously done when using
ConvertPartitionTupleSlot(), as that means the slot's tuple descriptors
change for every tuple.
Previously we also, via ConvertPartitionTupleSlot(), built new tuples
after the mapping even in cases where we, immediately afterwards,
access individual columns again.
Refactor the code so one slot, on demand, is used for each
partition. That avoids having to change the descriptor (and allows to
use the more efficient "fixed" tuple slots). Then use slot->slot
mapping, to avoid unnecessarily forming a tuple.
As the naming between the tuple and slot mapping functions wasn't
consistent, rename them to execute_attr_map_{tuple,slot}. It's likely
that we'll also rename convert_tuples_by_* to denote that these
functions "only" build a map, but that's left for later.
Author: Amit Khandekar and Amit Langote, editorialized by me
Reviewed-By: Amit Langote, Amit Khandekar, Andres Freund
Discussion:
https://postgr.es/m/CAJ3gD9fR0wRNeAE8VqffNTyONS_UfFPRpqxhnD9Q42vZB+Jvpg@mail.gmail.com
https://postgr.es/m/e4f9d743-cd4b-efb0-7574-da21d86a7f36%40lab.ntt.co.jp
Backpatch: -
2018-10-02 20:14:26 +02:00
|
|
|
TupleConversionMap *map;
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
|
|
|
|
/*
|
2018-11-16 18:54:15 +01:00
|
|
|
* Attempt to find a partition suitable for this tuple.
|
|
|
|
* ExecFindPartition() will raise an error if none can be found or
|
|
|
|
* if the found partition is not suitable for INSERTs.
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
*/
|
2018-11-16 18:54:15 +01:00
|
|
|
resultRelInfo = ExecFindPartition(mtstate, target_resultRelInfo,
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
proute, myslot, estate);
|
2018-11-16 18:54:15 +01:00
|
|
|
|
|
|
|
if (prevResultRelInfo != resultRelInfo)
|
2017-01-24 14:50:16 +01:00
|
|
|
{
|
2018-08-01 10:23:09 +02:00
|
|
|
/* Determine which triggers exist on this partition */
|
|
|
|
has_before_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
|
|
|
|
resultRelInfo->ri_TrigDesc->trig_insert_before_row);
|
|
|
|
|
|
|
|
has_instead_insert_row_trig = (resultRelInfo->ri_TrigDesc &&
|
|
|
|
resultRelInfo->ri_TrigDesc->trig_insert_instead_row);
|
|
|
|
|
2018-10-17 20:31:20 +02:00
|
|
|
/*
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
* Disable multi-inserts when the partition has BEFORE/INSTEAD
|
|
|
|
* OF triggers, or if the partition is a foreign partition.
|
2018-10-17 20:31:20 +02:00
|
|
|
*/
|
|
|
|
leafpart_use_multi_insert = insertMethod == CIM_MULTI_CONDITIONAL &&
|
|
|
|
!has_before_insert_row_trig &&
|
|
|
|
!has_instead_insert_row_trig &&
|
|
|
|
resultRelInfo->ri_FdwRoutine == NULL;
|
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* Set the multi-insert buffer to use for this partition. */
|
|
|
|
if (leafpart_use_multi_insert)
|
|
|
|
{
|
|
|
|
if (resultRelInfo->ri_CopyMultiInsertBuffer == NULL)
|
|
|
|
CopyMultiInsertInfoSetupBuffer(&multiInsertInfo,
|
|
|
|
resultRelInfo);
|
|
|
|
}
|
|
|
|
else if (insertMethod == CIM_MULTI_CONDITIONAL &&
|
|
|
|
!CopyMultiInsertInfoIsEmpty(&multiInsertInfo))
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Flush pending inserts if this partition can't use
|
|
|
|
* batching, so rows are visible to triggers etc.
|
|
|
|
*/
|
|
|
|
CopyMultiInsertInfoFlush(&multiInsertInfo, resultRelInfo);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (bistate != NULL)
|
|
|
|
ReleaseBulkInsertStatePin(bistate);
|
2018-11-16 18:54:15 +01:00
|
|
|
prevResultRelInfo = resultRelInfo;
|
2017-01-24 14:50:16 +01:00
|
|
|
}
|
|
|
|
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
/*
|
|
|
|
* For ExecInsertIndexTuples() to work on the partition's indexes
|
|
|
|
*/
|
|
|
|
estate->es_result_relation_info = resultRelInfo;
|
|
|
|
|
2017-06-28 19:55:03 +02:00
|
|
|
/*
|
|
|
|
* If we're capturing transition tuples, we might need to convert
|
2018-11-16 18:54:15 +01:00
|
|
|
* from the partition rowtype to root rowtype.
|
2017-06-28 19:55:03 +02:00
|
|
|
*/
|
|
|
|
if (cstate->transition_capture != NULL)
|
|
|
|
{
|
2018-08-01 10:23:09 +02:00
|
|
|
if (has_before_insert_row_trig)
|
2017-06-28 19:55:03 +02:00
|
|
|
{
|
|
|
|
/*
|
2018-03-19 21:43:57 +01:00
|
|
|
* If there are any BEFORE triggers on the partition,
|
|
|
|
* we'll have to be ready to convert their result back to
|
|
|
|
* tuplestore format.
|
2017-06-28 19:55:03 +02:00
|
|
|
*/
|
|
|
|
cstate->transition_capture->tcs_original_insert_tuple = NULL;
|
|
|
|
cstate->transition_capture->tcs_map =
|
2018-11-16 18:54:15 +01:00
|
|
|
resultRelInfo->ri_PartitionInfo->pi_PartitionToRootMap;
|
2017-06-28 19:55:03 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Otherwise, just remember the original unconverted
|
|
|
|
* tuple, to avoid a needless round trip conversion.
|
|
|
|
*/
|
2019-02-27 05:30:28 +01:00
|
|
|
cstate->transition_capture->tcs_original_insert_tuple = myslot;
|
2017-06-28 19:55:03 +02:00
|
|
|
cstate->transition_capture->tcs_map = NULL;
|
|
|
|
}
|
|
|
|
}
|
2017-08-14 23:29:33 +02:00
|
|
|
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
/*
|
2018-11-16 18:54:15 +01:00
|
|
|
* We might need to convert from the root rowtype to the partition
|
|
|
|
* rowtype.
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
*/
|
2018-11-16 18:54:15 +01:00
|
|
|
map = resultRelInfo->ri_PartitionInfo->pi_RootToPartitionMap;
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
if (insertMethod == CIM_SINGLE || !leafpart_use_multi_insert)
|
Use slots more widely in tuple mapping code and make naming more consistent.
It's inefficient to use a single slot for mapping between tuple
descriptors for multiple tuples, as previously done when using
ConvertPartitionTupleSlot(), as that means the slot's tuple descriptors
change for every tuple.
Previously we also, via ConvertPartitionTupleSlot(), built new tuples
after the mapping even in cases where we, immediately afterwards,
access individual columns again.
Refactor the code so one slot, on demand, is used for each
partition. That avoids having to change the descriptor (and allows to
use the more efficient "fixed" tuple slots). Then use slot->slot
mapping, to avoid unnecessarily forming a tuple.
As the naming between the tuple and slot mapping functions wasn't
consistent, rename them to execute_attr_map_{tuple,slot}. It's likely
that we'll also rename convert_tuples_by_* to denote that these
functions "only" build a map, but that's left for later.
Author: Amit Khandekar and Amit Langote, editorialized by me
Reviewed-By: Amit Langote, Amit Khandekar, Andres Freund
Discussion:
https://postgr.es/m/CAJ3gD9fR0wRNeAE8VqffNTyONS_UfFPRpqxhnD9Q42vZB+Jvpg@mail.gmail.com
https://postgr.es/m/e4f9d743-cd4b-efb0-7574-da21d86a7f36%40lab.ntt.co.jp
Backpatch: -
2018-10-02 20:14:26 +02:00
|
|
|
{
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* non batch insert */
|
|
|
|
if (map != NULL)
|
|
|
|
{
|
|
|
|
TupleTableSlot *new_slot;
|
Use slots more widely in tuple mapping code and make naming more consistent.
It's inefficient to use a single slot for mapping between tuple
descriptors for multiple tuples, as previously done when using
ConvertPartitionTupleSlot(), as that means the slot's tuple descriptors
change for every tuple.
Previously we also, via ConvertPartitionTupleSlot(), built new tuples
after the mapping even in cases where we, immediately afterwards,
access individual columns again.
Refactor the code so one slot, on demand, is used for each
partition. That avoids having to change the descriptor (and allows to
use the more efficient "fixed" tuple slots). Then use slot->slot
mapping, to avoid unnecessarily forming a tuple.
As the naming between the tuple and slot mapping functions wasn't
consistent, rename them to execute_attr_map_{tuple,slot}. It's likely
that we'll also rename convert_tuples_by_* to denote that these
functions "only" build a map, but that's left for later.
Author: Amit Khandekar and Amit Langote, editorialized by me
Reviewed-By: Amit Langote, Amit Khandekar, Andres Freund
Discussion:
https://postgr.es/m/CAJ3gD9fR0wRNeAE8VqffNTyONS_UfFPRpqxhnD9Q42vZB+Jvpg@mail.gmail.com
https://postgr.es/m/e4f9d743-cd4b-efb0-7574-da21d86a7f36%40lab.ntt.co.jp
Backpatch: -
2018-10-02 20:14:26 +02:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
new_slot = resultRelInfo->ri_PartitionInfo->pi_PartitionTupleSlot;
|
|
|
|
myslot = execute_attr_map_slot(map->attrMap, myslot, new_slot);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
Use slots more widely in tuple mapping code and make naming more consistent.
It's inefficient to use a single slot for mapping between tuple
descriptors for multiple tuples, as previously done when using
ConvertPartitionTupleSlot(), as that means the slot's tuple descriptors
change for every tuple.
Previously we also, via ConvertPartitionTupleSlot(), built new tuples
after the mapping even in cases where we, immediately afterwards,
access individual columns again.
Refactor the code so one slot, on demand, is used for each
partition. That avoids having to change the descriptor (and allows to
use the more efficient "fixed" tuple slots). Then use slot->slot
mapping, to avoid unnecessarily forming a tuple.
As the naming between the tuple and slot mapping functions wasn't
consistent, rename them to execute_attr_map_{tuple,slot}. It's likely
that we'll also rename convert_tuples_by_* to denote that these
functions "only" build a map, but that's left for later.
Author: Amit Khandekar and Amit Langote, editorialized by me
Reviewed-By: Amit Langote, Amit Khandekar, Andres Freund
Discussion:
https://postgr.es/m/CAJ3gD9fR0wRNeAE8VqffNTyONS_UfFPRpqxhnD9Q42vZB+Jvpg@mail.gmail.com
https://postgr.es/m/e4f9d743-cd4b-efb0-7574-da21d86a7f36%40lab.ntt.co.jp
Backpatch: -
2018-10-02 20:14:26 +02:00
|
|
|
/*
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
* Prepare to queue up tuple for later batch insert into
|
|
|
|
* current partition.
|
Use slots more widely in tuple mapping code and make naming more consistent.
It's inefficient to use a single slot for mapping between tuple
descriptors for multiple tuples, as previously done when using
ConvertPartitionTupleSlot(), as that means the slot's tuple descriptors
change for every tuple.
Previously we also, via ConvertPartitionTupleSlot(), built new tuples
after the mapping even in cases where we, immediately afterwards,
access individual columns again.
Refactor the code so one slot, on demand, is used for each
partition. That avoids having to change the descriptor (and allows to
use the more efficient "fixed" tuple slots). Then use slot->slot
mapping, to avoid unnecessarily forming a tuple.
As the naming between the tuple and slot mapping functions wasn't
consistent, rename them to execute_attr_map_{tuple,slot}. It's likely
that we'll also rename convert_tuples_by_* to denote that these
functions "only" build a map, but that's left for later.
Author: Amit Khandekar and Amit Langote, editorialized by me
Reviewed-By: Amit Langote, Amit Khandekar, Andres Freund
Discussion:
https://postgr.es/m/CAJ3gD9fR0wRNeAE8VqffNTyONS_UfFPRpqxhnD9Q42vZB+Jvpg@mail.gmail.com
https://postgr.es/m/e4f9d743-cd4b-efb0-7574-da21d86a7f36%40lab.ntt.co.jp
Backpatch: -
2018-10-02 20:14:26 +02:00
|
|
|
*/
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
TupleTableSlot *batchslot;
|
|
|
|
|
|
|
|
/* no other path available for partitioned table */
|
|
|
|
Assert(insertMethod == CIM_MULTI_CONDITIONAL);
|
|
|
|
|
|
|
|
batchslot = CopyMultiInsertInfoNextFreeSlot(&multiInsertInfo,
|
|
|
|
resultRelInfo);
|
|
|
|
|
|
|
|
if (map != NULL)
|
|
|
|
myslot = execute_attr_map_slot(map->attrMap, myslot,
|
|
|
|
batchslot);
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* This looks more expensive than it is (Believe me, I
|
|
|
|
* optimized it away. Twice.). The input is in virtual
|
|
|
|
* form, and we'll materialize the slot below - for most
|
|
|
|
* slot types the copy performs the work materialization
|
|
|
|
* would later require anyway.
|
|
|
|
*/
|
|
|
|
ExecCopySlot(batchslot, myslot);
|
|
|
|
myslot = batchslot;
|
|
|
|
}
|
Use slots more widely in tuple mapping code and make naming more consistent.
It's inefficient to use a single slot for mapping between tuple
descriptors for multiple tuples, as previously done when using
ConvertPartitionTupleSlot(), as that means the slot's tuple descriptors
change for every tuple.
Previously we also, via ConvertPartitionTupleSlot(), built new tuples
after the mapping even in cases where we, immediately afterwards,
access individual columns again.
Refactor the code so one slot, on demand, is used for each
partition. That avoids having to change the descriptor (and allows to
use the more efficient "fixed" tuple slots). Then use slot->slot
mapping, to avoid unnecessarily forming a tuple.
As the naming between the tuple and slot mapping functions wasn't
consistent, rename them to execute_attr_map_{tuple,slot}. It's likely
that we'll also rename convert_tuples_by_* to denote that these
functions "only" build a map, but that's left for later.
Author: Amit Khandekar and Amit Langote, editorialized by me
Reviewed-By: Amit Langote, Amit Khandekar, Andres Freund
Discussion:
https://postgr.es/m/CAJ3gD9fR0wRNeAE8VqffNTyONS_UfFPRpqxhnD9Q42vZB+Jvpg@mail.gmail.com
https://postgr.es/m/e4f9d743-cd4b-efb0-7574-da21d86a7f36%40lab.ntt.co.jp
Backpatch: -
2018-10-02 20:14:26 +02:00
|
|
|
}
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* ensure that triggers etc see the right relation */
|
|
|
|
myslot->tts_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
}
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
skip_tuple = false;
|
|
|
|
|
|
|
|
/* BEFORE ROW INSERT Triggers */
|
2018-08-01 10:23:09 +02:00
|
|
|
if (has_before_insert_row_trig)
|
2011-02-16 03:19:11 +01:00
|
|
|
{
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
if (!ExecBRInsertTriggers(estate, resultRelInfo, myslot))
|
2019-02-27 05:30:28 +01:00
|
|
|
skip_tuple = true; /* "do nothing" */
|
2011-02-16 03:19:11 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!skip_tuple)
|
|
|
|
{
|
2019-03-14 22:44:21 +01:00
|
|
|
/*
|
|
|
|
* If there is an INSTEAD OF INSERT ROW trigger, let it handle the
|
|
|
|
* tuple. Otherwise, proceed with inserting the tuple into the
|
|
|
|
* table or foreign table.
|
|
|
|
*/
|
2018-08-01 10:23:09 +02:00
|
|
|
if (has_instead_insert_row_trig)
|
2011-11-09 09:54:41 +01:00
|
|
|
{
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
ExecIRInsertTriggers(estate, resultRelInfo, myslot);
|
2011-11-09 09:54:41 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2019-05-20 05:47:54 +02:00
|
|
|
/* Compute stored generated columns */
|
2019-03-30 08:13:09 +01:00
|
|
|
if (resultRelInfo->ri_RelationDesc->rd_att->constr &&
|
|
|
|
resultRelInfo->ri_RelationDesc->rd_att->constr->has_generated_stored)
|
2020-02-17 15:19:58 +01:00
|
|
|
ExecComputeStoredGenerated(estate, myslot, CMD_INSERT);
|
2019-03-30 08:13:09 +01:00
|
|
|
|
2018-04-07 01:16:11 +02:00
|
|
|
/*
|
|
|
|
* If the target is a plain table, check the constraints of
|
|
|
|
* the tuple.
|
|
|
|
*/
|
|
|
|
if (resultRelInfo->ri_FdwRoutine == NULL &&
|
2018-06-11 22:53:33 +02:00
|
|
|
resultRelInfo->ri_RelationDesc->rd_att->constr)
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
ExecConstraints(resultRelInfo, myslot, estate);
|
2018-06-11 22:53:33 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Also check the tuple against the partition constraint, if
|
|
|
|
* there is one; except that if we got here via tuple-routing,
|
|
|
|
* we don't need to if there's no BR trigger defined on the
|
|
|
|
* partition.
|
|
|
|
*/
|
|
|
|
if (resultRelInfo->ri_PartitionCheck &&
|
2018-08-01 10:23:09 +02:00
|
|
|
(proute == NULL || has_before_insert_row_trig))
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
ExecPartitionCheck(resultRelInfo, myslot, estate, true);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* Store the slot in the multi-insert buffer, when enabled. */
|
2018-08-01 10:23:09 +02:00
|
|
|
if (insertMethod == CIM_MULTI || leafpart_use_multi_insert)
|
2016-11-10 20:13:43 +01:00
|
|
|
{
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/*
|
|
|
|
* The slot previously might point into the per-tuple
|
|
|
|
* context. For batching it needs to be longer lived.
|
|
|
|
*/
|
|
|
|
ExecMaterializeSlot(myslot);
|
|
|
|
|
2016-11-10 20:13:43 +01:00
|
|
|
/* Add this tuple to the tuple buffer */
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
CopyMultiInsertInfoStore(&multiInsertInfo,
|
|
|
|
resultRelInfo, myslot,
|
|
|
|
cstate->line_buf.len,
|
|
|
|
cstate->cur_lineno);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
2016-11-10 20:13:43 +01:00
|
|
|
/*
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
* If enough inserts have queued up, then flush all
|
|
|
|
* buffers out to their tables.
|
2016-11-10 20:13:43 +01:00
|
|
|
*/
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
if (CopyMultiInsertInfoIsFull(&multiInsertInfo))
|
|
|
|
CopyMultiInsertInfoFlush(&multiInsertInfo, resultRelInfo);
|
2016-11-10 20:13:43 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
List *recheckIndexes = NIL;
|
|
|
|
|
2018-04-07 01:16:11 +02:00
|
|
|
/* OK, store the tuple */
|
|
|
|
if (resultRelInfo->ri_FdwRoutine != NULL)
|
|
|
|
{
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
myslot = resultRelInfo->ri_FdwRoutine->ExecForeignInsert(estate,
|
|
|
|
resultRelInfo,
|
|
|
|
myslot,
|
|
|
|
NULL);
|
2018-04-07 01:16:11 +02:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
if (myslot == NULL) /* "do nothing" */
|
2018-08-01 10:23:09 +02:00
|
|
|
continue; /* next tuple please */
|
2018-04-07 01:16:11 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* AFTER ROW Triggers might reference the tableoid
|
Store table oid and tuple's tid in tuple slots directly.
After the introduction of tuple table slots all table AMs need to
support returning the table oid of the tuple stored in a slot created
by said AM. It does not make sense to re-implement that in every AM,
therefore move handling of table OIDs into the TupleTableSlot
structure itself. It's possible that we, at a later date, might want
to get rid of HeapTupleData.t_tableOid entirely, but doing so before
the abstractions for table AMs are integrated turns out to be too
hard, so delay that for now.
Similarly, every AM needs to support the concept of a tuple
identifier (tid / item pointer) for its tuples. It's quite possible
that we'll generalize the exact form of a tid at a future point (to
allow for things like index organized tables), but for now many parts
of the code know about tids, so there's not much point in abstracting
tids away. Therefore also move into slot (rather than providing API to
set/get the tid associated with the tuple in a slot).
Once table AM includes insert/updating/deleting tuples, the
responsibility to set the correct tid after such an action will move
into that. After that change, code doing such modifications, should
not have to deal with HeapTuples directly anymore.
Author: Andres Freund, Haribabu Kommi and Ashutosh Bapat
Discussion: https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
2019-02-27 03:21:44 +01:00
|
|
|
* column, so (re-)initialize tts_tableOid before
|
|
|
|
* evaluating them.
|
2018-04-07 01:16:11 +02:00
|
|
|
*/
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
myslot->tts_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
|
2018-04-07 01:16:11 +02:00
|
|
|
}
|
|
|
|
else
|
Store table oid and tuple's tid in tuple slots directly.
After the introduction of tuple table slots all table AMs need to
support returning the table oid of the tuple stored in a slot created
by said AM. It does not make sense to re-implement that in every AM,
therefore move handling of table OIDs into the TupleTableSlot
structure itself. It's possible that we, at a later date, might want
to get rid of HeapTupleData.t_tableOid entirely, but doing so before
the abstractions for table AMs are integrated turns out to be too
hard, so delay that for now.
Similarly, every AM needs to support the concept of a tuple
identifier (tid / item pointer) for its tuples. It's quite possible
that we'll generalize the exact form of a tid at a future point (to
allow for things like index organized tables), but for now many parts
of the code know about tids, so there's not much point in abstracting
tids away. Therefore also move into slot (rather than providing API to
set/get the tid associated with the tuple in a slot).
Once table AM includes insert/updating/deleting tuples, the
responsibility to set the correct tid after such an action will move
into that. After that change, code doing such modifications, should
not have to deal with HeapTuples directly anymore.
Author: Andres Freund, Haribabu Kommi and Ashutosh Bapat
Discussion: https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
2019-02-27 03:21:44 +01:00
|
|
|
{
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* OK, store the tuple and create index entries for it */
|
2019-05-24 01:25:48 +02:00
|
|
|
table_tuple_insert(resultRelInfo->ri_RelationDesc,
|
|
|
|
myslot, mycid, ti_options, bistate);
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
|
|
|
|
if (resultRelInfo->ri_NumIndices > 0)
|
|
|
|
recheckIndexes = ExecInsertIndexTuples(myslot,
|
|
|
|
estate,
|
|
|
|
false,
|
|
|
|
NULL,
|
|
|
|
NIL);
|
Store table oid and tuple's tid in tuple slots directly.
After the introduction of tuple table slots all table AMs need to
support returning the table oid of the tuple stored in a slot created
by said AM. It does not make sense to re-implement that in every AM,
therefore move handling of table OIDs into the TupleTableSlot
structure itself. It's possible that we, at a later date, might want
to get rid of HeapTupleData.t_tableOid entirely, but doing so before
the abstractions for table AMs are integrated turns out to be too
hard, so delay that for now.
Similarly, every AM needs to support the concept of a tuple
identifier (tid / item pointer) for its tuples. It's quite possible
that we'll generalize the exact form of a tid at a future point (to
allow for things like index organized tables), but for now many parts
of the code know about tids, so there's not much point in abstracting
tids away. Therefore also move into slot (rather than providing API to
set/get the tid associated with the tuple in a slot).
Once table AM includes insert/updating/deleting tuples, the
responsibility to set the correct tid after such an action will move
into that. After that change, code doing such modifications, should
not have to deal with HeapTuples directly anymore.
Author: Andres Freund, Haribabu Kommi and Ashutosh Bapat
Discussion: https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
2019-02-27 03:21:44 +01:00
|
|
|
}
|
2018-04-07 01:16:11 +02:00
|
|
|
|
2016-11-10 20:13:43 +01:00
|
|
|
/* AFTER ROW INSERT Triggers */
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
ExecARInsertTriggers(estate, resultRelInfo, myslot,
|
2017-06-28 19:55:03 +02:00
|
|
|
recheckIndexes, cstate->transition_capture);
|
2016-11-10 20:13:43 +01:00
|
|
|
|
|
|
|
list_free(recheckIndexes);
|
|
|
|
}
|
2011-11-09 09:54:41 +01:00
|
|
|
}
|
2011-02-16 03:19:11 +01:00
|
|
|
|
|
|
|
/*
|
2018-04-07 01:16:11 +02:00
|
|
|
* We count only tuples not suppressed by a BEFORE INSERT trigger
|
|
|
|
* or FDW; this is the same definition used by nodeModifyTable.c
|
|
|
|
* for counting tuples inserted by an INSERT command.
|
2011-02-16 03:19:11 +01:00
|
|
|
*/
|
|
|
|
processed++;
|
2018-03-19 21:43:57 +01:00
|
|
|
}
|
2011-02-16 03:19:11 +01:00
|
|
|
}
|
|
|
|
|
2011-11-09 09:54:41 +01:00
|
|
|
/* Flush any remaining buffered tuples */
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
if (insertMethod != CIM_SINGLE)
|
2018-08-01 10:23:09 +02:00
|
|
|
{
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
if (!CopyMultiInsertInfoIsEmpty(&multiInsertInfo))
|
|
|
|
CopyMultiInsertInfoFlush(&multiInsertInfo, NULL);
|
2018-08-01 10:23:09 +02:00
|
|
|
}
|
2011-11-09 09:54:41 +01:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/* Done, clean up */
|
2012-11-12 14:10:24 +01:00
|
|
|
error_context_stack = errcallback.previous;
|
2011-02-16 03:19:11 +01:00
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
if (bistate != NULL)
|
|
|
|
FreeBulkInsertState(bistate);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
|
|
|
MemoryContextSwitchTo(oldcontext);
|
|
|
|
|
Be more careful to not lose sync in the FE/BE protocol.
If any error occurred while we were in the middle of reading a protocol
message from the client, we could lose sync, and incorrectly try to
interpret a part of another message as a new protocol message. That will
usually lead to an "invalid frontend message" error that terminates the
connection. However, this is a security issue because an attacker might
be able to deliberately cause an error, inject a Query message in what's
supposed to be just user data, and have the server execute it.
We were quite careful to not have CHECK_FOR_INTERRUPTS() calls or other
operations that could ereport(ERROR) in the middle of processing a message,
but a query cancel interrupt or statement timeout could nevertheless cause
it to happen. Also, the V2 fastpath and COPY handling were not so careful.
It's very difficult to recover in the V2 COPY protocol, so we will just
terminate the connection on error. In practice, that's what happened
previously anyway, as we lost protocol sync.
To fix, add a new variable in pqcomm.c, PqCommReadingMsg, that is set
whenever we're in the middle of reading a message. When it's set, we cannot
safely ERROR out and continue running, because we might've read only part
of a message. PqCommReadingMsg acts somewhat similarly to critical sections
in that if an error occurs while it's set, the error handler will force the
connection to be terminated, as if the error was FATAL. It's not
implemented by promoting ERROR to FATAL in elog.c, like ERROR is promoted
to PANIC in critical sections, because we want to be able to use
PG_TRY/CATCH to recover and regain protocol sync. pq_getmessage() takes
advantage of that to prevent an OOM error from terminating the connection.
To prevent unnecessary connection terminations, add a holdoff mechanism
similar to HOLD/RESUME_INTERRUPTS() that can be used hold off query cancel
interrupts, but still allow die interrupts. The rules on which interrupts
are processed when are now a bit more complicated, so refactor
ProcessInterrupts() and the calls to it in signal handlers so that the
signal handlers always call it if ImmediateInterruptOK is set, and
ProcessInterrupts() can decide to not do anything if the other conditions
are not met.
Reported by Emil Lenngren. Patch reviewed by Noah Misch and Andres Freund.
Backpatch to all supported versions.
Security: CVE-2015-0244
2015-02-02 16:08:45 +01:00
|
|
|
/*
|
|
|
|
* In the old protocol, tell pqcomm that we can process normal protocol
|
|
|
|
* messages again.
|
|
|
|
*/
|
|
|
|
if (cstate->copy_dest == COPY_OLD_FE)
|
|
|
|
pq_endmsgread();
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/* Execute AFTER STATEMENT insertion triggers */
|
2018-08-01 10:23:09 +02:00
|
|
|
ExecASInsertTriggers(estate, target_resultRelInfo, cstate->transition_capture);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
|
|
|
/* Handle queued AFTER triggers */
|
|
|
|
AfterTriggerEndQuery(estate);
|
|
|
|
|
|
|
|
ExecResetTupleTable(estate->es_tupleTable, false);
|
|
|
|
|
2018-04-07 01:16:11 +02:00
|
|
|
/* Allow the FDW to shut down */
|
2018-08-01 10:23:09 +02:00
|
|
|
if (target_resultRelInfo->ri_FdwRoutine != NULL &&
|
|
|
|
target_resultRelInfo->ri_FdwRoutine->EndForeignInsert != NULL)
|
|
|
|
target_resultRelInfo->ri_FdwRoutine->EndForeignInsert(estate,
|
|
|
|
target_resultRelInfo);
|
2018-04-07 01:16:11 +02:00
|
|
|
|
Fix missing calls to table_finish_bulk_insert during COPY, take 2
86b85044e abstracted calls to heap functions in COPY FROM to support a
generic table AM. However, when performing a copy into a partitioned
table, this commit neglected to call table_finish_bulk_insert for each
partition. Before 86b85044e, when we always called the heap functions,
there was no need to call heapam_finish_bulk_insert for partitions since
it only did any work when performing a copy without WAL. For partitioned
tables, this was unsupported anyway, so there was no issue. With
pluggable storage, we can't make any assumptions about what the table AM
might want to do in its equivalent function, so we'd better ensure we
always call table_finish_bulk_insert each partition that's received a row.
For now, we make the table_finish_bulk_insert call whenever we evict a
CopyMultiInsertBuffer out of the CopyMultiInsertInfo. This does mean
that it's possible that we call table_finish_bulk_insert multiple times
per partition, which is not a problem other than being an inefficiency.
Improving this requires a more invasive patch, so let's leave that for
another day.
This also changes things so that we no longer needlessly call
table_finish_bulk_insert when performing a COPY FROM for a non-partitioned
table when not using multi-inserts.
Reported-by: Robert Haas
Backpatch-through: 12
Discussion: https://postgr.es/m/CA+TgmoYK=6BpxiJ0tN-p9wtH0BTAfbdxzHhwou0mdud4+BkYuQ@mail.gmail.com
2019-07-10 06:03:04 +02:00
|
|
|
/* Tear down the multi-insert buffer data */
|
|
|
|
if (insertMethod != CIM_SINGLE)
|
|
|
|
CopyMultiInsertInfoCleanup(&multiInsertInfo);
|
|
|
|
|
2018-08-01 10:23:09 +02:00
|
|
|
ExecCloseIndices(target_resultRelInfo);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
/* Close all the partitioned tables, leaf partitions, and their indices */
|
2018-08-01 10:23:09 +02:00
|
|
|
if (proute)
|
|
|
|
ExecCleanupTupleRouting(mtstate, proute);
|
Implement table partitioning.
Table partitioning is like table inheritance and reuses much of the
existing infrastructure, but there are some important differences.
The parent is called a partitioned table and is always empty; it may
not have indexes or non-inherited constraints, since those make no
sense for a relation with no data of its own. The children are called
partitions and contain all of the actual data. Each partition has an
implicit partitioning constraint. Multiple inheritance is not
allowed, and partitioning and inheritance can't be mixed. Partitions
can't have extra columns and may not allow nulls unless the parent
does. Tuples inserted into the parent are automatically routed to the
correct partition, so tuple-routing ON INSERT triggers are not needed.
Tuple routing isn't yet supported for partitions which are foreign
tables, and it doesn't handle updates that cross partition boundaries.
Currently, tables can be range-partitioned or list-partitioned. List
partitioning is limited to a single column, but range partitioning can
involve multiple columns. A partitioning "column" can be an
expression.
Because table partitioning is less general than table inheritance, it
is hoped that it will be easier to reason about properties of
partitions, and therefore that this will serve as a better foundation
for a variety of possible optimizations, including query planner
optimizations. The tuple routing based which this patch does based on
the implicit partitioning constraints is an example of this, but it
seems likely that many other useful optimizations are also possible.
Amit Langote, reviewed and tested by Robert Haas, Ashutosh Bapat,
Amit Kapila, Rajkumar Raghuwanshi, Corey Huinker, Jaime Casanova,
Rushabh Lathia, Erik Rijkers, among others. Minor revisions by me.
2016-12-07 19:17:43 +01:00
|
|
|
|
2017-05-16 18:46:32 +02:00
|
|
|
/* Close any trigger target relations */
|
|
|
|
ExecCleanUpTriggerState(estate);
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
FreeExecutorState(estate);
|
|
|
|
|
|
|
|
return processed;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Setup to read tuples from a file for COPY FROM.
|
|
|
|
*
|
|
|
|
* 'rel': Used as a template for the tuples
|
|
|
|
* 'filename': Name of server-local file to read
|
|
|
|
* 'attnamelist': List of char *, columns to include. NIL selects all cols.
|
|
|
|
* 'options': List of DefElem. See copy_opt_item in gram.y for selections.
|
|
|
|
*
|
|
|
|
* Returns a CopyState, to be passed to NextCopyFrom and related functions.
|
|
|
|
*/
|
|
|
|
CopyState
|
2016-09-06 18:00:00 +02:00
|
|
|
BeginCopyFrom(ParseState *pstate,
|
|
|
|
Relation rel,
|
2011-02-16 03:19:11 +01:00
|
|
|
const char *filename,
|
2013-05-29 22:58:43 +02:00
|
|
|
bool is_program,
|
2017-03-23 13:36:36 +01:00
|
|
|
copy_data_source_cb data_source_cb,
|
2011-02-16 03:19:11 +01:00
|
|
|
List *attnamelist,
|
|
|
|
List *options)
|
|
|
|
{
|
|
|
|
CopyState cstate;
|
|
|
|
bool pipe = (filename == NULL);
|
|
|
|
TupleDesc tupDesc;
|
|
|
|
AttrNumber num_phys_attrs,
|
|
|
|
num_defaults;
|
|
|
|
FmgrInfo *in_functions;
|
|
|
|
Oid *typioparams;
|
|
|
|
int attnum;
|
|
|
|
Oid in_func_oid;
|
|
|
|
int *defmap;
|
|
|
|
ExprState **defexprs;
|
|
|
|
MemoryContext oldcontext;
|
2011-11-09 09:54:41 +01:00
|
|
|
bool volatile_defexprs;
|
2011-02-16 03:19:11 +01:00
|
|
|
|
2016-09-06 18:00:00 +02:00
|
|
|
cstate = BeginCopy(pstate, true, rel, NULL, InvalidOid, attnamelist, options);
|
2011-02-16 03:19:11 +01:00
|
|
|
oldcontext = MemoryContextSwitchTo(cstate->copycontext);
|
|
|
|
|
|
|
|
/* Initialize state variables */
|
Handle EPIPE more sanely when we close a pipe reading from a program.
Previously, any program launched by COPY TO/FROM PROGRAM inherited the
server's setting of SIGPIPE handling, i.e. SIG_IGN. Hence, if we were
doing COPY FROM PROGRAM and closed the pipe early, the child process
would see EPIPE on its output file and typically would treat that as
a fatal error, in turn causing the COPY to report error. Similarly,
one could get a failure report from a query that didn't read all of
the output from a contrib/file_fdw foreign table that uses file_fdw's
PROGRAM option.
To fix, ensure that child programs inherit SIG_DFL not SIG_IGN processing
of SIGPIPE. This seems like an all-around better situation since if
the called program wants some non-default treatment of SIGPIPE, it would
expect to have to set that up for itself. Then in COPY, if it's COPY
FROM PROGRAM and we stop reading short of detecting EOF, treat a SIGPIPE
exit from the called program as a non-error condition. This still allows
us to report an error for any case where the called program gets SIGPIPE
on some other file descriptor.
As coded, we won't report a SIGPIPE if we stop reading as a result of
seeing an in-band EOF marker (e.g. COPY BINARY EOF marker). It's
somewhat debatable whether we should complain if the called program
continues to transmit data after an EOF marker. However, it seems like
we should avoid throwing error in any questionable cases, especially in a
back-patched fix, and anyway it would take additional code to make such
an error get reported consistently.
Back-patch to v10. We could go further back, since COPY FROM PROGRAM
has been around awhile, but AFAICS the only way to reach this situation
using core or contrib is via file_fdw, which has only supported PROGRAM
sources since v10. The COPY statement per se has no feature whereby
it'd stop reading without having hit EOF or an error already. Therefore,
I don't see any upside to back-patching further that'd outweigh the
risk of complaints about behavioral change.
Per bug #15449 from Eric Cyr.
Patch by me, review by Etsuro Fujita and Kyotaro Horiguchi
Discussion: https://postgr.es/m/15449-1cf737dd5929450e@postgresql.org
2018-11-19 23:02:25 +01:00
|
|
|
cstate->reached_eof = false;
|
2011-02-16 03:19:11 +01:00
|
|
|
cstate->eol_type = EOL_UNKNOWN;
|
|
|
|
cstate->cur_relname = RelationGetRelationName(cstate->rel);
|
|
|
|
cstate->cur_lineno = 0;
|
|
|
|
cstate->cur_attname = NULL;
|
|
|
|
cstate->cur_attval = NULL;
|
|
|
|
|
|
|
|
/* Set up variables to avoid per-attribute overhead. */
|
2017-05-10 23:41:27 +02:00
|
|
|
initStringInfo(&cstate->attribute_buf);
|
|
|
|
initStringInfo(&cstate->line_buf);
|
2011-02-16 03:19:11 +01:00
|
|
|
cstate->line_buf_converted = false;
|
|
|
|
cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
|
|
|
|
cstate->raw_buf_index = cstate->raw_buf_len = 0;
|
|
|
|
|
2017-04-18 05:22:04 +02:00
|
|
|
/* Assign range table, we'll need it in CopyFrom. */
|
|
|
|
if (pstate)
|
|
|
|
cstate->range_table = pstate->p_rtable;
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
tupDesc = RelationGetDescr(cstate->rel);
|
|
|
|
num_phys_attrs = tupDesc->natts;
|
|
|
|
num_defaults = 0;
|
2011-11-09 09:54:41 +01:00
|
|
|
volatile_defexprs = false;
|
2011-02-16 03:19:11 +01:00
|
|
|
|
2002-08-02 20:15:10 +02:00
|
|
|
/*
|
2002-09-20 18:56:02 +02:00
|
|
|
* Pick up the required catalog information for each attribute in the
|
2005-10-15 04:49:52 +02:00
|
|
|
* relation, including the input function, the element type (to pass to
|
|
|
|
* the input function), and info about defaults and constraints. (Which
|
|
|
|
* input function we use depends on text/binary format choice.)
|
2002-08-02 20:15:10 +02:00
|
|
|
*/
|
2004-06-05 21:48:09 +02:00
|
|
|
in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
|
2004-06-06 02:41:28 +02:00
|
|
|
typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
|
2004-06-05 21:48:09 +02:00
|
|
|
defmap = (int *) palloc(num_phys_attrs * sizeof(int));
|
|
|
|
defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
|
2002-08-02 20:15:10 +02:00
|
|
|
|
2004-04-16 00:36:03 +02:00
|
|
|
for (attnum = 1; attnum <= num_phys_attrs; attnum++)
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
2017-08-20 20:19:07 +02:00
|
|
|
Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1);
|
|
|
|
|
2002-08-02 20:15:10 +02:00
|
|
|
/* We don't need info for dropped attributes */
|
2017-08-20 20:19:07 +02:00
|
|
|
if (att->attisdropped)
|
2002-08-02 20:15:10 +02:00
|
|
|
continue;
|
2002-07-30 18:55:06 +02:00
|
|
|
|
2004-06-06 02:41:28 +02:00
|
|
|
/* Fetch the input function and typioparam info */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->binary)
|
2017-08-20 20:19:07 +02:00
|
|
|
getTypeBinaryInputInfo(att->atttypid,
|
2005-10-15 04:49:52 +02:00
|
|
|
&in_func_oid, &typioparams[attnum - 1]);
|
2003-05-09 23:19:50 +02:00
|
|
|
else
|
2017-08-20 20:19:07 +02:00
|
|
|
getTypeInputInfo(att->atttypid,
|
2004-06-06 02:41:28 +02:00
|
|
|
&in_func_oid, &typioparams[attnum - 1]);
|
2004-04-16 00:36:03 +02:00
|
|
|
fmgr_info(in_func_oid, &in_functions[attnum - 1]);
|
2002-07-30 18:55:06 +02:00
|
|
|
|
2002-09-20 18:56:02 +02:00
|
|
|
/* Get default info if needed */
|
2019-03-30 08:13:09 +01:00
|
|
|
if (!list_member_int(cstate->attnumlist, attnum) && !att->attgenerated)
|
2002-08-24 17:00:47 +02:00
|
|
|
{
|
2003-05-09 23:19:50 +02:00
|
|
|
/* attribute is NOT to be copied from input */
|
2002-08-24 17:00:47 +02:00
|
|
|
/* use default value if one exists */
|
2018-02-02 20:20:50 +01:00
|
|
|
Expr *defexpr = (Expr *) build_column_default(cstate->rel,
|
|
|
|
attnum);
|
2002-12-13 20:46:01 +01:00
|
|
|
|
|
|
|
if (defexpr != NULL)
|
2002-07-30 18:55:06 +02:00
|
|
|
{
|
2013-11-08 14:59:39 +01:00
|
|
|
/* Run the expression through planner */
|
|
|
|
defexpr = expression_planner(defexpr);
|
|
|
|
|
|
|
|
/* Initialize executable expression in copycontext */
|
|
|
|
defexprs[num_defaults] = ExecInitExpr(defexpr, NULL);
|
2004-04-16 00:36:03 +02:00
|
|
|
defmap[num_defaults] = attnum - 1;
|
2002-08-02 20:15:10 +02:00
|
|
|
num_defaults++;
|
2011-11-09 09:54:41 +01:00
|
|
|
|
2014-01-20 18:22:38 +01:00
|
|
|
/*
|
2014-05-06 18:12:18 +02:00
|
|
|
* If a default expression looks at the table being loaded,
|
|
|
|
* then it could give the wrong answer when using
|
|
|
|
* multi-insert. Since database access can be dynamic this is
|
|
|
|
* hard to test for exactly, so we use the much wider test of
|
|
|
|
* whether the default expression is volatile. We allow for
|
|
|
|
* the special case of when the default expression is the
|
|
|
|
* nextval() of a sequence which in this specific case is
|
|
|
|
* known to be safe for use with the multi-insert
|
2017-03-14 16:38:30 +01:00
|
|
|
* optimization. Hence we use this special case function
|
2014-05-06 18:12:18 +02:00
|
|
|
* checker rather than the standard check for
|
2014-01-20 18:22:38 +01:00
|
|
|
* contain_volatile_functions().
|
|
|
|
*/
|
2011-11-09 09:54:41 +01:00
|
|
|
if (!volatile_defexprs)
|
2014-05-06 18:12:18 +02:00
|
|
|
volatile_defexprs = contain_volatile_functions_not_nextval((Node *) defexpr);
|
2002-07-18 06:43:51 +02:00
|
|
|
}
|
1997-09-07 07:04:48 +02:00
|
|
|
}
|
2002-08-02 20:15:10 +02:00
|
|
|
}
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/* We keep those variables in cstate. */
|
|
|
|
cstate->in_functions = in_functions;
|
|
|
|
cstate->typioparams = typioparams;
|
|
|
|
cstate->defmap = defmap;
|
|
|
|
cstate->defexprs = defexprs;
|
2011-11-09 09:54:41 +01:00
|
|
|
cstate->volatile_defexprs = volatile_defexprs;
|
2011-02-16 03:19:11 +01:00
|
|
|
cstate->num_defaults = num_defaults;
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
cstate->is_program = is_program;
|
2004-09-10 20:40:09 +02:00
|
|
|
|
2017-03-23 13:36:36 +01:00
|
|
|
if (data_source_cb)
|
|
|
|
{
|
|
|
|
cstate->copy_dest = COPY_CALLBACK;
|
|
|
|
cstate->data_source_cb = data_source_cb;
|
|
|
|
}
|
|
|
|
else if (pipe)
|
2011-02-16 03:19:11 +01:00
|
|
|
{
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
Assert(!is_program); /* the grammar does not allow this */
|
2011-02-16 03:19:11 +01:00
|
|
|
if (whereToSendOutput == DestRemote)
|
|
|
|
ReceiveCopyBegin(cstate);
|
|
|
|
else
|
|
|
|
cstate->copy_file = stdin;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
cstate->filename = pstrdup(filename);
|
|
|
|
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
if (cstate->is_program)
|
|
|
|
{
|
|
|
|
cstate->copy_file = OpenPipeStream(cstate->filename, PG_BINARY_R);
|
|
|
|
if (cstate->copy_file == NULL)
|
|
|
|
ereport(ERROR,
|
2015-08-03 05:49:19 +02:00
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not execute command \"%s\": %m",
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
cstate->filename)));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
struct stat st;
|
2011-02-16 03:19:11 +01:00
|
|
|
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
cstate->copy_file = AllocateFile(cstate->filename, PG_BINARY_R);
|
|
|
|
if (cstate->copy_file == NULL)
|
2016-09-07 05:55:55 +02:00
|
|
|
{
|
|
|
|
/* copy errno because ereport subfunctions might change it */
|
|
|
|
int save_errno = errno;
|
|
|
|
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not open file \"%s\" for reading: %m",
|
2016-09-07 05:55:55 +02:00
|
|
|
cstate->filename),
|
|
|
|
(save_errno == ENOENT || save_errno == EACCES) ?
|
|
|
|
errhint("COPY FROM instructs the PostgreSQL server process to read a file. "
|
|
|
|
"You may want a client-side facility such as psql's \\copy.") : 0));
|
|
|
|
}
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
|
2015-01-04 16:47:23 +01:00
|
|
|
if (fstat(fileno(cstate->copy_file), &st))
|
2015-08-03 05:49:19 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode_for_file_access(),
|
|
|
|
errmsg("could not stat file \"%s\": %m",
|
|
|
|
cstate->filename)));
|
2015-01-04 16:47:23 +01:00
|
|
|
|
Add support for piping COPY to/from an external program.
This includes backend "COPY TO/FROM PROGRAM '...'" syntax, and corresponding
psql \copy syntax. Like with reading/writing files, the backend version is
superuser-only, and in the psql version, the program is run in the client.
In the passing, the psql \copy STDIN/STDOUT syntax is subtly changed: if you
the stdin/stdout is quoted, it's now interpreted as a filename. For example,
"\copy foo from 'stdin'" now reads from a file called 'stdin', not from
standard input. Before this, there was no way to specify a filename called
stdin, stdout, pstdin or pstdout.
This creates a new function in pgport, wait_result_to_str(), which can
be used to convert the exit status of a process, as returned by wait(3),
to a human-readable string.
Etsuro Fujita, reviewed by Amit Kapila.
2013-02-27 17:17:21 +01:00
|
|
|
if (S_ISDIR(st.st_mode))
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
|
|
|
errmsg("\"%s\" is a directory", cstate->filename)));
|
|
|
|
}
|
2011-02-16 03:19:11 +01:00
|
|
|
}
|
2002-11-23 04:59:09 +01:00
|
|
|
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
if (cstate->binary)
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
2001-01-03 21:04:10 +01:00
|
|
|
/* Read and verify binary header */
|
2003-05-09 23:19:50 +02:00
|
|
|
char readSig[11];
|
2001-01-03 21:04:10 +01:00
|
|
|
int32 tmp;
|
|
|
|
|
|
|
|
/* Signature */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (CopyGetData(cstate, readSig, 11, 11) != 11 ||
|
|
|
|
memcmp(readSig, BinarySignature, 11) != 0)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("COPY file signature not recognized")));
|
2001-01-03 21:04:10 +01:00
|
|
|
/* Flags field */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!CopyGetInt32(cstate, &tmp))
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("invalid COPY file header (missing flags)")));
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
if ((tmp & (1 << 16)) != 0)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("invalid COPY file header (WITH OIDS)")));
|
2001-03-22 05:01:46 +01:00
|
|
|
tmp &= ~(1 << 16);
|
2001-01-03 21:04:10 +01:00
|
|
|
if ((tmp >> 16) != 0)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("unrecognized critical flags in COPY file header")));
|
2001-01-03 21:04:10 +01:00
|
|
|
/* Header extension length */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!CopyGetInt32(cstate, &tmp) ||
|
|
|
|
tmp < 0)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
2005-10-15 04:49:52 +02:00
|
|
|
errmsg("invalid COPY file header (missing length)")));
|
2001-01-03 21:04:10 +01:00
|
|
|
/* Skip extension header, if present */
|
|
|
|
while (tmp-- > 0)
|
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
if (CopyGetData(cstate, readSig, 1, 1) != 1)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
2005-10-15 04:49:52 +02:00
|
|
|
errmsg("invalid COPY file header (wrong length)")));
|
2001-01-03 21:04:10 +01:00
|
|
|
}
|
1997-09-07 07:04:48 +02:00
|
|
|
}
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* create workspace for CopyReadAttributes results */
|
2011-02-16 03:19:11 +01:00
|
|
|
if (!cstate->binary)
|
2010-12-06 21:31:55 +01:00
|
|
|
{
|
2011-02-16 03:19:11 +01:00
|
|
|
AttrNumber attr_count = list_length(cstate->attnumlist);
|
|
|
|
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
cstate->max_fields = attr_count;
|
|
|
|
cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *));
|
2010-12-06 21:31:55 +01:00
|
|
|
}
|
2005-08-06 22:41:58 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
MemoryContextSwitchTo(oldcontext);
|
2003-04-24 23:16:45 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
return cstate;
|
|
|
|
}
|
2008-11-06 21:51:15 +01:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/*
|
|
|
|
* Read raw fields in the next line for COPY FROM in text or csv mode.
|
|
|
|
* Return false if no more lines.
|
|
|
|
*
|
|
|
|
* An internal temporary buffer is returned via 'fields'. It is valid until
|
|
|
|
* the next call of the function. Since the function returns all raw fields
|
|
|
|
* in the input file, 'nfields' could be different from the number of columns
|
|
|
|
* in the relation.
|
|
|
|
*
|
|
|
|
* NOTE: force_not_null option are not applied to the returned fields.
|
|
|
|
*/
|
|
|
|
bool
|
|
|
|
NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
|
|
|
|
{
|
|
|
|
int fldct;
|
|
|
|
bool done;
|
|
|
|
|
|
|
|
/* only available for text or csv input */
|
|
|
|
Assert(!cstate->binary);
|
2003-04-24 23:16:45 +02:00
|
|
|
|
2005-05-07 04:22:49 +02:00
|
|
|
/* on input just throw the header line away */
|
2011-02-16 03:19:11 +01:00
|
|
|
if (cstate->cur_lineno == 0 && cstate->header_line)
|
2005-05-07 04:22:49 +02:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
cstate->cur_lineno++;
|
2011-02-16 03:19:11 +01:00
|
|
|
if (CopyReadLine(cstate))
|
2011-04-10 17:42:00 +02:00
|
|
|
return false; /* done */
|
2005-05-07 04:22:49 +02:00
|
|
|
}
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
cstate->cur_lineno++;
|
2002-07-30 18:55:06 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/* Actually read the line into memory here */
|
|
|
|
done = CopyReadLine(cstate);
|
1999-09-27 22:00:44 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/*
|
2011-04-10 17:42:00 +02:00
|
|
|
* EOF at start of line means we're done. If we see EOF after some
|
|
|
|
* characters, we act as though it was newline followed by EOF, ie,
|
|
|
|
* process the line and then exit loop on next iteration.
|
2011-02-16 03:19:11 +01:00
|
|
|
*/
|
|
|
|
if (done && cstate->line_buf.len == 0)
|
|
|
|
return false;
|
2002-09-04 22:31:48 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/* Parse the line into de-escaped field values */
|
|
|
|
if (cstate->csv_mode)
|
|
|
|
fldct = CopyReadAttributesCSV(cstate);
|
|
|
|
else
|
|
|
|
fldct = CopyReadAttributesText(cstate);
|
2001-01-22 01:50:07 +01:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
*fields = cstate->raw_fields;
|
|
|
|
*nfields = fldct;
|
|
|
|
return true;
|
|
|
|
}
|
2002-08-02 20:15:10 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/*
|
|
|
|
* Read next tuple from file for COPY FROM. Return false if no more tuples.
|
|
|
|
*
|
|
|
|
* 'econtext' is used to evaluate default expression for each columns not
|
|
|
|
* read from the file. It can be NULL when no default values are used, i.e.
|
|
|
|
* when all columns are read from the file.
|
|
|
|
*
|
|
|
|
* 'values' and 'nulls' arrays must be the same length as columns of the
|
|
|
|
* relation passed to BeginCopyFrom. This function fills the arrays.
|
|
|
|
* Oid of the tuple is returned with 'tupleOid' separately.
|
|
|
|
*/
|
|
|
|
bool
|
|
|
|
NextCopyFrom(CopyState cstate, ExprContext *econtext,
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
Datum *values, bool *nulls)
|
2011-02-16 03:19:11 +01:00
|
|
|
{
|
|
|
|
TupleDesc tupDesc;
|
|
|
|
AttrNumber num_phys_attrs,
|
|
|
|
attr_count,
|
|
|
|
num_defaults = cstate->num_defaults;
|
|
|
|
FmgrInfo *in_functions = cstate->in_functions;
|
|
|
|
Oid *typioparams = cstate->typioparams;
|
|
|
|
int i;
|
|
|
|
int *defmap = cstate->defmap;
|
|
|
|
ExprState **defexprs = cstate->defexprs;
|
2001-01-03 21:04:10 +01:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
tupDesc = RelationGetDescr(cstate->rel);
|
|
|
|
num_phys_attrs = tupDesc->natts;
|
|
|
|
attr_count = list_length(cstate->attnumlist);
|
1997-09-07 07:04:48 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/* Initialize all values for row to NULL */
|
|
|
|
MemSet(values, 0, num_phys_attrs * sizeof(Datum));
|
|
|
|
MemSet(nulls, true, num_phys_attrs * sizeof(bool));
|
2003-09-30 00:06:40 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
if (!cstate->binary)
|
|
|
|
{
|
|
|
|
char **field_strings;
|
|
|
|
ListCell *cur;
|
|
|
|
int fldct;
|
|
|
|
int fieldno;
|
|
|
|
char *string;
|
2003-09-30 00:06:40 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/* read raw fields in the next line */
|
|
|
|
if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* check for overflowing fields */
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
if (attr_count > 0 && fldct > attr_count)
|
2011-02-16 03:19:11 +01:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("extra data after last expected column")));
|
2010-12-06 21:31:55 +01:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
fieldno = 0;
|
|
|
|
|
|
|
|
/* Loop to read the user attributes on the line. */
|
|
|
|
foreach(cur, cstate->attnumlist)
|
|
|
|
{
|
|
|
|
int attnum = lfirst_int(cur);
|
|
|
|
int m = attnum - 1;
|
2017-08-20 20:19:07 +02:00
|
|
|
Form_pg_attribute att = TupleDescAttr(tupDesc, m);
|
2002-08-02 20:15:10 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
if (fieldno >= fldct)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
2011-02-16 03:19:11 +01:00
|
|
|
errmsg("missing data for column \"%s\"",
|
2017-08-20 20:19:07 +02:00
|
|
|
NameStr(att->attname))));
|
2011-02-16 03:19:11 +01:00
|
|
|
string = field_strings[fieldno++];
|
2001-01-03 21:04:10 +01:00
|
|
|
|
2012-07-12 22:26:59 +02:00
|
|
|
if (cstate->convert_select_flags &&
|
|
|
|
!cstate->convert_select_flags[m])
|
|
|
|
{
|
|
|
|
/* ignore input field, leaving column as NULL */
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2014-03-04 23:31:59 +01:00
|
|
|
if (cstate->csv_mode)
|
2002-08-02 20:15:10 +02:00
|
|
|
{
|
2014-05-06 18:12:18 +02:00
|
|
|
if (string == NULL &&
|
|
|
|
cstate->force_notnull_flags[m])
|
2014-03-04 23:31:59 +01:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* FORCE_NOT_NULL option is set and column is NULL -
|
|
|
|
* convert it to the NULL string.
|
|
|
|
*/
|
|
|
|
string = cstate->null_print;
|
|
|
|
}
|
2014-05-06 18:12:18 +02:00
|
|
|
else if (string != NULL && cstate->force_null_flags[m]
|
|
|
|
&& strcmp(string, cstate->null_print) == 0)
|
2014-03-04 23:31:59 +01:00
|
|
|
{
|
|
|
|
/*
|
2014-05-06 18:12:18 +02:00
|
|
|
* FORCE_NULL option is set and column matches the NULL
|
|
|
|
* string. It must have been quoted, or otherwise the
|
|
|
|
* string would already have been set to NULL. Convert it
|
|
|
|
* to NULL as specified.
|
2014-03-04 23:31:59 +01:00
|
|
|
*/
|
|
|
|
string = NULL;
|
|
|
|
}
|
2002-09-20 05:52:50 +02:00
|
|
|
}
|
|
|
|
|
2017-08-20 20:19:07 +02:00
|
|
|
cstate->cur_attname = NameStr(att->attname);
|
2011-02-16 03:19:11 +01:00
|
|
|
cstate->cur_attval = string;
|
|
|
|
values[m] = InputFunctionCall(&in_functions[m],
|
|
|
|
string,
|
|
|
|
typioparams[m],
|
2017-08-20 20:19:07 +02:00
|
|
|
att->atttypmod);
|
2011-02-16 03:19:11 +01:00
|
|
|
if (string != NULL)
|
|
|
|
nulls[m] = false;
|
|
|
|
cstate->cur_attname = NULL;
|
|
|
|
cstate->cur_attval = NULL;
|
2002-08-02 20:15:10 +02:00
|
|
|
}
|
|
|
|
|
Remove WITH OIDS support, change oid catalog column visibility.
Previously tables declared WITH OIDS, including a significant fraction
of the catalog tables, stored the oid column not as a normal column,
but as part of the tuple header.
This special column was not shown by default, which was somewhat odd,
as it's often (consider e.g. pg_class.oid) one of the more important
parts of a row. Neither pg_dump nor COPY included the contents of the
oid column by default.
The fact that the oid column was not an ordinary column necessitated a
significant amount of special case code to support oid columns. That
already was painful for the existing, but upcoming work aiming to make
table storage pluggable, would have required expanding and duplicating
that "specialness" significantly.
WITH OIDS has been deprecated since 2005 (commit ff02d0a05280e0).
Remove it.
Removing includes:
- CREATE TABLE and ALTER TABLE syntax for declaring the table to be
WITH OIDS has been removed (WITH (oids[ = true]) will error out)
- pg_dump does not support dumping tables declared WITH OIDS and will
issue a warning when dumping one (and ignore the oid column).
- restoring an pg_dump archive with pg_restore will warn when
restoring a table with oid contents (and ignore the oid column)
- COPY will refuse to load binary dump that includes oids.
- pg_upgrade will error out when encountering tables declared WITH
OIDS, they have to be altered to remove the oid column first.
- Functionality to access the oid of the last inserted row (like
plpgsql's RESULT_OID, spi's SPI_lastoid, ...) has been removed.
The syntax for declaring a table WITHOUT OIDS (or WITH (oids = false)
for CREATE TABLE) is still supported. While that requires a bit of
support code, it seems unnecessary to break applications / dumps that
do not use oids, and are explicit about not using them.
The biggest user of WITH OID columns was postgres' catalog. This
commit changes all 'magic' oid columns to be columns that are normally
declared and stored. To reduce unnecessary query breakage all the
newly added columns are still named 'oid', even if a table's column
naming scheme would indicate 'reloid' or such. This obviously
requires adapting a lot code, mostly replacing oid access via
HeapTupleGetOid() with access to the underlying Form_pg_*->oid column.
The bootstrap process now assigns oids for all oid columns in
genbki.pl that do not have an explicit value (starting at the largest
oid previously used), only oids assigned later by oids will be above
FirstBootstrapObjectId. As the oid column now is a normal column the
special bootstrap syntax for oids has been removed.
Oids are not automatically assigned during insertion anymore, all
backend code explicitly assigns oids with GetNewOidWithIndex(). For
the rare case that insertions into the catalog via SQL are called for
the new pg_nextoid() function can be used (which only works on catalog
tables).
The fact that oid columns on system tables are now normal columns
means that they will be included in the set of columns expanded
by * (i.e. SELECT * FROM pg_class will now include the table's oid,
previously it did not). It'd not technically be hard to hide oid
column by default, but that'd mean confusing behavior would either
have to be carried forward forever, or it'd cause breakage down the
line.
While it's not unlikely that further adjustments are needed, the
scope/invasiveness of the patch makes it worthwhile to get merge this
now. It's painful to maintain externally, too complicated to commit
after the code code freeze, and a dependency of a number of other
patches.
Catversion bump, for obvious reasons.
Author: Andres Freund, with contributions by John Naylor
Discussion: https://postgr.es/m/20180930034810.ywp2c7awz7opzcfr@alap3.anarazel.de
2018-11-21 00:36:57 +01:00
|
|
|
Assert(fieldno == attr_count);
|
2011-02-16 03:19:11 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* binary */
|
|
|
|
int16 fld_count;
|
|
|
|
ListCell *cur;
|
2002-08-02 20:15:10 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
cstate->cur_lineno++;
|
2000-07-15 00:18:02 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
if (!CopyGetInt16(cstate, &fld_count))
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
2011-02-16 03:19:11 +01:00
|
|
|
/* EOF detected (end of file, or protocol-level EOF) */
|
|
|
|
return false;
|
1997-09-07 07:04:48 +02:00
|
|
|
}
|
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
if (fld_count == -1)
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
2006-03-03 20:54:10 +01:00
|
|
|
/*
|
2011-04-10 17:42:00 +02:00
|
|
|
* Received EOF marker. In a V3-protocol copy, wait for the
|
|
|
|
* protocol-level EOF, and complain if it doesn't come
|
|
|
|
* immediately. This ensures that we correctly handle CopyFail,
|
|
|
|
* if client chooses to send that now.
|
2011-02-16 03:19:11 +01:00
|
|
|
*
|
2011-04-10 17:42:00 +02:00
|
|
|
* Note that we MUST NOT try to read more data in an old-protocol
|
2014-05-06 18:12:18 +02:00
|
|
|
* copy, since there is no protocol-level EOF marker then. We
|
2011-04-10 17:42:00 +02:00
|
|
|
* could go either way for copy from file, but choose to throw
|
|
|
|
* error if there's data after the EOF marker, for consistency
|
|
|
|
* with the new-protocol case.
|
2006-03-03 20:54:10 +01:00
|
|
|
*/
|
2011-04-10 17:42:00 +02:00
|
|
|
char dummy;
|
1999-05-25 18:15:34 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
if (cstate->copy_dest != COPY_OLD_FE &&
|
|
|
|
CopyGetData(cstate, &dummy, 1, 1) > 0)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("received copy data after EOF marker")));
|
|
|
|
return false;
|
|
|
|
}
|
1996-07-09 08:22:35 +02:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
if (fld_count != attr_count)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("row field count is %d, expected %d",
|
|
|
|
(int) fld_count, attr_count)));
|
2002-12-15 17:17:59 +01:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
i = 0;
|
|
|
|
foreach(cur, cstate->attnumlist)
|
|
|
|
{
|
|
|
|
int attnum = lfirst_int(cur);
|
|
|
|
int m = attnum - 1;
|
2017-08-20 20:19:07 +02:00
|
|
|
Form_pg_attribute att = TupleDescAttr(tupDesc, m);
|
2011-02-16 03:19:11 +01:00
|
|
|
|
2017-08-20 20:19:07 +02:00
|
|
|
cstate->cur_attname = NameStr(att->attname);
|
2011-02-16 03:19:11 +01:00
|
|
|
i++;
|
|
|
|
values[m] = CopyReadBinaryAttribute(cstate,
|
|
|
|
i,
|
|
|
|
&in_functions[m],
|
|
|
|
typioparams[m],
|
2017-08-20 20:19:07 +02:00
|
|
|
att->atttypmod,
|
2011-02-16 03:19:11 +01:00
|
|
|
&nulls[m]);
|
|
|
|
cstate->cur_attname = NULL;
|
|
|
|
}
|
2006-08-31 01:34:22 +02:00
|
|
|
}
|
2007-03-29 02:15:39 +02:00
|
|
|
|
2007-11-15 22:14:46 +01:00
|
|
|
/*
|
2011-02-16 03:19:11 +01:00
|
|
|
* Now compute and insert any defaults available for the columns not
|
2014-05-06 18:12:18 +02:00
|
|
|
* provided by the input data. Anything not processed here or above will
|
2011-04-10 17:42:00 +02:00
|
|
|
* remain NULL.
|
2007-03-29 02:15:39 +02:00
|
|
|
*/
|
2011-02-16 03:19:11 +01:00
|
|
|
for (i = 0; i < num_defaults; i++)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The caller must supply econtext and have switched into the
|
|
|
|
* per-tuple memory context in it.
|
|
|
|
*/
|
|
|
|
Assert(econtext != NULL);
|
|
|
|
Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
|
|
|
|
|
|
|
|
values[defmap[i]] = ExecEvalExpr(defexprs[i], econtext,
|
2017-01-19 23:12:38 +01:00
|
|
|
&nulls[defmap[i]]);
|
2011-02-16 03:19:11 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
2000-07-15 00:18:02 +02:00
|
|
|
}
|
1996-11-02 03:01:48 +01:00
|
|
|
|
2011-02-16 03:19:11 +01:00
|
|
|
/*
|
|
|
|
* Clean up storage and release resources for COPY FROM.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
EndCopyFrom(CopyState cstate)
|
|
|
|
{
|
|
|
|
/* No COPY FROM related resources except memory. */
|
|
|
|
|
|
|
|
EndCopy(cstate);
|
|
|
|
}
|
1996-11-02 03:01:48 +01:00
|
|
|
|
1996-07-09 08:22:35 +02:00
|
|
|
/*
|
2003-09-30 00:06:40 +02:00
|
|
|
* Read the next input line and stash it in line_buf, with conversion to
|
|
|
|
* server encoding.
|
2002-07-30 18:55:06 +02:00
|
|
|
*
|
2003-09-30 00:06:40 +02:00
|
|
|
* Result is true if read was terminated by EOF, false if terminated
|
2014-05-06 18:12:18 +02:00
|
|
|
* by newline. The terminating newline or EOF marker is not included
|
2005-08-06 22:41:58 +02:00
|
|
|
* in the final value of line_buf.
|
1996-07-09 08:22:35 +02:00
|
|
|
*/
|
2003-09-30 00:06:40 +02:00
|
|
|
static bool
|
2005-08-06 22:41:58 +02:00
|
|
|
CopyReadLine(CopyState cstate)
|
1996-07-09 08:22:35 +02:00
|
|
|
{
|
2003-09-30 00:06:40 +02:00
|
|
|
bool result;
|
2005-03-12 06:41:34 +01:00
|
|
|
|
2007-03-03 20:32:55 +01:00
|
|
|
resetStringInfo(&cstate->line_buf);
|
2013-05-23 13:49:59 +02:00
|
|
|
cstate->line_buf_valid = true;
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
/* Mark that encoding conversion hasn't occurred yet */
|
|
|
|
cstate->line_buf_converted = false;
|
|
|
|
|
|
|
|
/* Parse data and transfer into line_buf */
|
2005-12-27 19:10:48 +01:00
|
|
|
result = CopyReadLineText(cstate);
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
if (result)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Reached EOF. In protocol version 3, we should ignore anything
|
2005-10-15 04:49:52 +02:00
|
|
|
* after \. up to the protocol end of copy data. (XXX maybe better
|
|
|
|
* not to treat \. as special?)
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
|
|
|
if (cstate->copy_dest == COPY_NEW_FE)
|
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
do
|
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
cstate->raw_buf_index = cstate->raw_buf_len;
|
|
|
|
} while (CopyLoadRawBuf(cstate));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If we didn't hit EOF, then we must have transferred the EOL marker
|
|
|
|
* to line_buf along with the data. Get rid of it.
|
|
|
|
*/
|
|
|
|
switch (cstate->eol_type)
|
|
|
|
{
|
|
|
|
case EOL_NL:
|
|
|
|
Assert(cstate->line_buf.len >= 1);
|
|
|
|
Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
|
|
|
|
cstate->line_buf.len--;
|
|
|
|
cstate->line_buf.data[cstate->line_buf.len] = '\0';
|
|
|
|
break;
|
|
|
|
case EOL_CR:
|
|
|
|
Assert(cstate->line_buf.len >= 1);
|
|
|
|
Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
|
|
|
|
cstate->line_buf.len--;
|
|
|
|
cstate->line_buf.data[cstate->line_buf.len] = '\0';
|
|
|
|
break;
|
|
|
|
case EOL_CRNL:
|
|
|
|
Assert(cstate->line_buf.len >= 2);
|
|
|
|
Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
|
|
|
|
Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
|
|
|
|
cstate->line_buf.len -= 2;
|
|
|
|
cstate->line_buf.data[cstate->line_buf.len] = '\0';
|
|
|
|
break;
|
|
|
|
case EOL_UNKNOWN:
|
|
|
|
/* shouldn't get here */
|
|
|
|
Assert(false);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Done reading the line. Convert it to server encoding. */
|
|
|
|
if (cstate->need_transcoding)
|
2005-03-12 06:41:34 +01:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
char *cvt;
|
|
|
|
|
2011-02-21 06:08:04 +01:00
|
|
|
cvt = pg_any_to_server(cstate->line_buf.data,
|
|
|
|
cstate->line_buf.len,
|
|
|
|
cstate->file_encoding);
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cvt != cstate->line_buf.data)
|
|
|
|
{
|
|
|
|
/* transfer converted data back to line_buf */
|
2007-03-03 20:32:55 +01:00
|
|
|
resetStringInfo(&cstate->line_buf);
|
2005-08-06 22:41:58 +02:00
|
|
|
appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
|
|
|
|
pfree(cvt);
|
|
|
|
}
|
2005-03-12 06:41:34 +01:00
|
|
|
}
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* Now it's safe to use the buffer in error messages */
|
|
|
|
cstate->line_buf_converted = true;
|
1998-09-01 06:40:42 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
return result;
|
|
|
|
}
|
1997-09-07 07:04:48 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
2005-12-27 19:10:48 +01:00
|
|
|
* CopyReadLineText - inner loop of CopyReadLine for text mode
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
|
|
|
static bool
|
|
|
|
CopyReadLineText(CopyState cstate)
|
|
|
|
{
|
|
|
|
char *copy_raw_buf;
|
|
|
|
int raw_buf_ptr;
|
|
|
|
int copy_buf_len;
|
2005-12-27 19:10:48 +01:00
|
|
|
bool need_data = false;
|
|
|
|
bool hit_eof = false;
|
|
|
|
bool result = false;
|
|
|
|
char mblen_str[2];
|
2006-10-04 02:30:14 +02:00
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
/* CSV variables */
|
|
|
|
bool first_char_in_line = true;
|
|
|
|
bool in_quote = false,
|
|
|
|
last_was_esc = false;
|
|
|
|
char quotec = '\0';
|
|
|
|
char escapec = '\0';
|
2003-09-30 00:06:40 +02:00
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
if (cstate->csv_mode)
|
|
|
|
{
|
|
|
|
quotec = cstate->quote[0];
|
|
|
|
escapec = cstate->escape[0];
|
|
|
|
/* ignore special escape processing if it's the same as quotec */
|
|
|
|
if (quotec == escapec)
|
|
|
|
escapec = '\0';
|
|
|
|
}
|
2000-01-16 22:37:50 +01:00
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
mblen_str[1] = '\0';
|
1999-09-12 00:28:11 +02:00
|
|
|
|
2003-09-30 00:06:40 +02:00
|
|
|
/*
|
2005-10-15 04:49:52 +02:00
|
|
|
* The objective of this loop is to transfer the entire next input line
|
|
|
|
* into line_buf. Hence, we only care for detecting newlines (\r and/or
|
|
|
|
* \n) and the end-of-copy marker (\.).
|
2005-03-12 06:41:34 +01:00
|
|
|
*
|
2005-12-27 19:10:48 +01:00
|
|
|
* In CSV mode, \r and \n inside a quoted field are just part of the data
|
|
|
|
* value and are put in line_buf. We keep just enough state to know if we
|
|
|
|
* are currently in a quoted field or not.
|
2005-03-12 06:41:34 +01:00
|
|
|
*
|
2005-12-27 19:10:48 +01:00
|
|
|
* These four characters, and the CSV escape and quote characters, are
|
|
|
|
* assumed the same in frontend and backend encodings.
|
2005-03-12 06:41:34 +01:00
|
|
|
*
|
2005-12-27 19:10:48 +01:00
|
|
|
* For speed, we try to move data from raw_buf to line_buf in chunks
|
2006-10-04 02:30:14 +02:00
|
|
|
* rather than one character at a time. raw_buf_ptr points to the next
|
2005-12-27 19:10:48 +01:00
|
|
|
* character to examine; any characters from raw_buf_index to raw_buf_ptr
|
2006-10-04 02:30:14 +02:00
|
|
|
* have been determined to be part of the line, but not yet transferred to
|
|
|
|
* line_buf.
|
2005-08-06 22:41:58 +02:00
|
|
|
*
|
2005-11-22 19:17:34 +01:00
|
|
|
* For a little extra speed within the loop, we copy raw_buf and
|
|
|
|
* raw_buf_len into local variables.
|
2003-09-30 00:06:40 +02:00
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
copy_raw_buf = cstate->raw_buf;
|
|
|
|
raw_buf_ptr = cstate->raw_buf_index;
|
|
|
|
copy_buf_len = cstate->raw_buf_len;
|
|
|
|
|
1999-09-12 00:28:11 +02:00
|
|
|
for (;;)
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
int prev_raw_ptr;
|
|
|
|
char c;
|
2005-03-12 06:41:34 +01:00
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
/*
|
2006-10-04 02:30:14 +02:00
|
|
|
* Load more data if needed. Ideally we would just force four bytes
|
|
|
|
* of read-ahead and avoid the many calls to
|
|
|
|
* IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
|
|
|
|
* does not allow us to read too far ahead or we might read into the
|
2014-05-06 18:12:18 +02:00
|
|
|
* next data, so we read-ahead only as far we know we can. One
|
2006-10-04 02:30:14 +02:00
|
|
|
* optimization would be to read-ahead four byte here if
|
|
|
|
* cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
|
|
|
|
* considering the size of the buffer.
|
2005-12-27 19:10:48 +01:00
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
if (raw_buf_ptr >= copy_buf_len || need_data)
|
2005-03-12 06:41:34 +01:00
|
|
|
{
|
2005-12-27 19:10:48 +01:00
|
|
|
REFILL_LINEBUF;
|
2005-10-15 04:49:52 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
2014-05-06 18:12:18 +02:00
|
|
|
* Try to read some more data. This will certainly reset
|
2005-08-06 22:41:58 +02:00
|
|
|
* raw_buf_index to zero, and raw_buf_ptr must go with it.
|
|
|
|
*/
|
|
|
|
if (!CopyLoadRawBuf(cstate))
|
|
|
|
hit_eof = true;
|
|
|
|
raw_buf_ptr = 0;
|
|
|
|
copy_buf_len = cstate->raw_buf_len;
|
2005-10-15 04:49:52 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
|
|
|
* If we are completely out of data, break out of the loop,
|
|
|
|
* reporting EOF.
|
|
|
|
*/
|
|
|
|
if (copy_buf_len <= 0)
|
|
|
|
{
|
|
|
|
result = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
need_data = false;
|
2005-03-12 06:41:34 +01:00
|
|
|
}
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* OK to fetch a character */
|
|
|
|
prev_raw_ptr = raw_buf_ptr;
|
|
|
|
c = copy_raw_buf[raw_buf_ptr++];
|
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
if (cstate->csv_mode)
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
2005-12-27 19:10:48 +01:00
|
|
|
* If character is '\\' or '\r', we may need to look ahead below.
|
2006-10-04 02:30:14 +02:00
|
|
|
* Force fetch of the next character if we don't already have it.
|
|
|
|
* We need to do this before changing CSV state, in case one of
|
|
|
|
* these characters is also the quote or escape character.
|
2005-12-27 19:10:48 +01:00
|
|
|
*
|
2006-10-04 02:30:14 +02:00
|
|
|
* Note: old-protocol does not like forced prefetch, but it's OK
|
|
|
|
* here since we cannot validly be at EOF.
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
2005-12-27 19:10:48 +01:00
|
|
|
if (c == '\\' || c == '\r')
|
2002-07-30 18:55:06 +02:00
|
|
|
{
|
2005-12-27 19:10:48 +01:00
|
|
|
IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
|
2003-09-30 00:06:40 +02:00
|
|
|
}
|
2005-03-12 06:41:34 +01:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
2006-10-04 02:30:14 +02:00
|
|
|
* Dealing with quotes and escapes here is mildly tricky. If the
|
|
|
|
* quote char is also the escape char, there's no problem - we
|
|
|
|
* just use the char as a toggle. If they are different, we need
|
|
|
|
* to ensure that we only take account of an escape inside a
|
|
|
|
* quoted field and immediately preceding a quote char, and not
|
2015-05-20 15:18:11 +02:00
|
|
|
* the second in an escape-escape sequence.
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
2005-12-27 19:10:48 +01:00
|
|
|
if (in_quote && c == escapec)
|
|
|
|
last_was_esc = !last_was_esc;
|
|
|
|
if (c == quotec && !last_was_esc)
|
|
|
|
in_quote = !in_quote;
|
|
|
|
if (c != escapec)
|
2005-10-15 04:49:52 +02:00
|
|
|
last_was_esc = false;
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
2005-12-27 19:10:48 +01:00
|
|
|
* Updating the line count for embedded CR and/or LF chars is
|
2006-10-04 02:30:14 +02:00
|
|
|
* necessarily a little fragile - this test is probably about the
|
2014-05-06 18:12:18 +02:00
|
|
|
* best we can do. (XXX it's arguable whether we should do this
|
2006-10-04 02:30:14 +02:00
|
|
|
* at all --- is cur_lineno a physical or logical count?)
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
2005-12-27 19:10:48 +01:00
|
|
|
if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
|
|
|
|
cstate->cur_lineno++;
|
2003-09-30 00:06:40 +02:00
|
|
|
}
|
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
/* Process \r */
|
|
|
|
if (c == '\r' && (!cstate->csv_mode || !in_quote))
|
2005-08-06 22:41:58 +02:00
|
|
|
{
|
|
|
|
/* Check for \r\n on first line, _and_ handle \r\n. */
|
|
|
|
if (cstate->eol_type == EOL_UNKNOWN ||
|
|
|
|
cstate->eol_type == EOL_CRNL)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If need more data, go back to loop top to load it.
|
|
|
|
*
|
2005-11-22 19:17:34 +01:00
|
|
|
* Note that if we are at EOF, c will wind up as '\0' because
|
|
|
|
* of the guaranteed pad of raw_buf.
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
2005-12-27 19:10:48 +01:00
|
|
|
IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
|
|
|
|
|
|
|
|
/* get next char */
|
2005-08-06 22:41:58 +02:00
|
|
|
c = copy_raw_buf[raw_buf_ptr];
|
|
|
|
|
|
|
|
if (c == '\n')
|
|
|
|
{
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
raw_buf_ptr++; /* eat newline */
|
|
|
|
cstate->eol_type = EOL_CRNL; /* in case not set yet */
|
2005-08-06 22:41:58 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* found \r, but no \n */
|
|
|
|
if (cstate->eol_type == EOL_CRNL)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
2007-12-27 18:00:56 +01:00
|
|
|
!cstate->csv_mode ?
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("literal carriage return found in data") :
|
|
|
|
errmsg("unquoted carriage return found in data"),
|
2007-12-27 18:00:56 +01:00
|
|
|
!cstate->csv_mode ?
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errhint("Use \"\\r\" to represent carriage return.") :
|
2007-12-27 18:00:56 +01:00
|
|
|
errhint("Use quoted CSV field to represent carriage return.")));
|
2006-10-04 02:30:14 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
2005-10-15 04:49:52 +02:00
|
|
|
* if we got here, it is the first line and we didn't find
|
|
|
|
* \n, so don't consume the peeked character
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
|
|
|
cstate->eol_type = EOL_CR;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else if (cstate->eol_type == EOL_NL)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
2007-12-27 18:00:56 +01:00
|
|
|
!cstate->csv_mode ?
|
|
|
|
errmsg("literal carriage return found in data") :
|
|
|
|
errmsg("unquoted carriage return found in data"),
|
|
|
|
!cstate->csv_mode ?
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errhint("Use \"\\r\" to represent carriage return.") :
|
2007-12-27 18:00:56 +01:00
|
|
|
errhint("Use quoted CSV field to represent carriage return.")));
|
2005-08-06 22:41:58 +02:00
|
|
|
/* If reach here, we have found the line terminator */
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
/* Process \n */
|
|
|
|
if (c == '\n' && (!cstate->csv_mode || !in_quote))
|
2005-08-06 22:41:58 +02:00
|
|
|
{
|
|
|
|
if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
2007-12-27 18:00:56 +01:00
|
|
|
!cstate->csv_mode ?
|
|
|
|
errmsg("literal newline found in data") :
|
|
|
|
errmsg("unquoted newline found in data"),
|
|
|
|
!cstate->csv_mode ?
|
|
|
|
errhint("Use \"\\n\" to represent newline.") :
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errhint("Use quoted CSV field to represent newline.")));
|
2005-10-15 04:49:52 +02:00
|
|
|
cstate->eol_type = EOL_NL; /* in case not set yet */
|
2005-08-06 22:41:58 +02:00
|
|
|
/* If reach here, we have found the line terminator */
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2006-10-04 02:30:14 +02:00
|
|
|
* In CSV mode, we only recognize \. alone on a line. This is because
|
|
|
|
* \. is a valid CSV data value.
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
2005-12-27 19:10:48 +01:00
|
|
|
if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
|
2005-08-06 22:41:58 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
char c2;
|
2005-08-06 22:41:58 +02:00
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
|
|
|
|
IF_NEED_REFILL_AND_EOF_BREAK(0);
|
2005-08-06 22:41:58 +02:00
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
/* -----
|
|
|
|
* get next character
|
|
|
|
* Note: we do not change c so if it isn't \., we can fall
|
2011-02-21 06:08:04 +01:00
|
|
|
* through and continue processing for file encoding.
|
2005-12-27 19:10:48 +01:00
|
|
|
* -----
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
|
|
|
c2 = copy_raw_buf[raw_buf_ptr];
|
|
|
|
|
|
|
|
if (c2 == '.')
|
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
raw_buf_ptr++; /* consume the '.' */
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Note: if we loop back for more data here, it does not
|
2005-10-15 04:49:52 +02:00
|
|
|
* matter that the CSV state change checks are re-executed; we
|
|
|
|
* will come back here with no important state changed.
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
|
|
|
if (cstate->eol_type == EOL_CRNL)
|
|
|
|
{
|
2005-12-27 19:10:48 +01:00
|
|
|
/* Get the next character */
|
|
|
|
IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
|
2005-08-06 22:41:58 +02:00
|
|
|
/* if hit_eof, c2 will become '\0' */
|
|
|
|
c2 = copy_raw_buf[raw_buf_ptr++];
|
2005-12-27 19:10:48 +01:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (c2 == '\n')
|
2005-12-27 19:10:48 +01:00
|
|
|
{
|
|
|
|
if (!cstate->csv_mode)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("end-of-copy marker does not match previous newline style")));
|
|
|
|
else
|
|
|
|
NO_END_OF_COPY_GOTO;
|
|
|
|
}
|
|
|
|
else if (c2 != '\r')
|
|
|
|
{
|
|
|
|
if (!cstate->csv_mode)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("end-of-copy marker corrupt")));
|
|
|
|
else
|
|
|
|
NO_END_OF_COPY_GOTO;
|
|
|
|
}
|
2005-08-06 22:41:58 +02:00
|
|
|
}
|
2005-12-27 19:10:48 +01:00
|
|
|
|
|
|
|
/* Get the next character */
|
|
|
|
IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
|
2005-08-06 22:41:58 +02:00
|
|
|
/* if hit_eof, c2 will become '\0' */
|
|
|
|
c2 = copy_raw_buf[raw_buf_ptr++];
|
2005-12-27 19:10:48 +01:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (c2 != '\r' && c2 != '\n')
|
2005-12-27 19:10:48 +01:00
|
|
|
{
|
|
|
|
if (!cstate->csv_mode)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("end-of-copy marker corrupt")));
|
|
|
|
else
|
|
|
|
NO_END_OF_COPY_GOTO;
|
|
|
|
}
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
|
|
|
|
(cstate->eol_type == EOL_CRNL && c2 != '\n') ||
|
|
|
|
(cstate->eol_type == EOL_CR && c2 != '\r'))
|
2005-12-27 19:10:48 +01:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("end-of-copy marker does not match previous newline style")));
|
2005-12-27 19:10:48 +01:00
|
|
|
}
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
/*
|
2005-10-15 04:49:52 +02:00
|
|
|
* Transfer only the data before the \. into line_buf, then
|
|
|
|
* discard the data and the \. sequence.
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
|
|
|
if (prev_raw_ptr > cstate->raw_buf_index)
|
2005-12-27 19:10:48 +01:00
|
|
|
appendBinaryStringInfo(&cstate->line_buf,
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
cstate->raw_buf + cstate->raw_buf_index,
|
|
|
|
prev_raw_ptr - cstate->raw_buf_index);
|
2005-08-06 22:41:58 +02:00
|
|
|
cstate->raw_buf_index = raw_buf_ptr;
|
|
|
|
result = true; /* report EOF */
|
|
|
|
break;
|
|
|
|
}
|
2005-12-27 19:10:48 +01:00
|
|
|
else if (!cstate->csv_mode)
|
2006-10-04 02:30:14 +02:00
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
/*
|
2006-10-04 02:30:14 +02:00
|
|
|
* If we are here, it means we found a backslash followed by
|
|
|
|
* something other than a period. In non-CSV mode, anything
|
|
|
|
* after a backslash is special, so we skip over that second
|
|
|
|
* character too. If we didn't do that \\. would be
|
2010-09-22 12:37:46 +02:00
|
|
|
* considered an eof-of copy, while in non-CSV mode it is a
|
2014-05-06 18:12:18 +02:00
|
|
|
* literal backslash followed by a period. In CSV mode,
|
2006-10-04 02:30:14 +02:00
|
|
|
* backslashes are not special, so we want to process the
|
|
|
|
* character after the backslash just like a normal character,
|
|
|
|
* so we don't increment in those cases.
|
2005-12-27 19:10:48 +01:00
|
|
|
*/
|
|
|
|
raw_buf_ptr++;
|
2005-08-06 22:41:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2006-10-04 02:30:14 +02:00
|
|
|
* This label is for CSV cases where \. appears at the start of a
|
|
|
|
* line, but there is more text after it, meaning it was a data value.
|
2005-12-27 19:10:48 +01:00
|
|
|
* We are more strict for \. in CSV mode because \. could be a data
|
|
|
|
* value, while in non-CSV mode, \. cannot be a data value.
|
|
|
|
*/
|
|
|
|
not_end_of_copy:
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Process all bytes of a multi-byte character as a group.
|
2005-08-06 22:41:58 +02:00
|
|
|
*
|
2006-10-04 02:30:14 +02:00
|
|
|
* We only support multi-byte sequences where the first byte has the
|
|
|
|
* high-bit set, so as an optimization we can avoid this block
|
|
|
|
* entirely if it is not set.
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
2005-12-27 19:10:48 +01:00
|
|
|
if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
|
2005-08-06 22:41:58 +02:00
|
|
|
{
|
|
|
|
int mblen;
|
|
|
|
|
2019-01-25 13:54:38 +01:00
|
|
|
/*
|
|
|
|
* It is enough to look at the first byte in all our encodings, to
|
|
|
|
* get the length. (GB18030 is a bit special, but still works for
|
|
|
|
* our purposes; see comment in pg_gb18030_mblen())
|
|
|
|
*/
|
2005-12-27 19:10:48 +01:00
|
|
|
mblen_str[0] = c;
|
2011-02-21 06:08:04 +01:00
|
|
|
mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str);
|
2019-01-25 13:54:38 +01:00
|
|
|
|
2005-12-27 19:10:48 +01:00
|
|
|
IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
|
|
|
|
IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
|
2005-10-15 04:49:52 +02:00
|
|
|
raw_buf_ptr += mblen - 1;
|
2005-08-06 22:41:58 +02:00
|
|
|
}
|
2005-12-27 19:10:48 +01:00
|
|
|
first_char_in_line = false;
|
2005-08-06 22:41:58 +02:00
|
|
|
} /* end of outer loop */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Transfer any still-uncopied data to line_buf.
|
|
|
|
*/
|
2005-12-27 19:10:48 +01:00
|
|
|
REFILL_LINEBUF;
|
2004-10-29 21:18:22 +02:00
|
|
|
|
2003-09-30 00:06:40 +02:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2005-06-02 03:21:22 +02:00
|
|
|
/*
|
|
|
|
* Return decimal value for a hexadecimal digit
|
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
static int
|
|
|
|
GetDecimalFromHex(char hex)
|
2005-06-02 03:21:22 +02:00
|
|
|
{
|
2005-09-01 17:34:31 +02:00
|
|
|
if (isdigit((unsigned char) hex))
|
2005-06-02 03:21:22 +02:00
|
|
|
return hex - '0';
|
|
|
|
else
|
2005-09-01 17:34:31 +02:00
|
|
|
return tolower((unsigned char) hex) - 'a' + 10;
|
2005-06-02 03:21:22 +02:00
|
|
|
}
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
|
|
|
* Parse the current line into separate attributes (fields),
|
|
|
|
* performing de-escaping as needed.
|
|
|
|
*
|
|
|
|
* The input is in line_buf. We use attribute_buf to hold the result
|
2011-04-10 17:42:00 +02:00
|
|
|
* strings. cstate->raw_fields[k] is set to point to the k'th attribute
|
|
|
|
* string, or NULL when the input matches the null marker string.
|
2010-12-06 21:31:55 +01:00
|
|
|
* This array is expanded as necessary.
|
|
|
|
*
|
2011-04-10 17:42:00 +02:00
|
|
|
* (Note that the caller cannot check for nulls since the returned
|
|
|
|
* string would be the post-de-escaping equivalent, which may look
|
2010-12-06 21:31:55 +01:00
|
|
|
* the same as some valid data string.)
|
2003-09-30 00:06:40 +02:00
|
|
|
*
|
2003-10-06 04:38:53 +02:00
|
|
|
* delim is the column delimiter string (must be just one byte for now).
|
|
|
|
* null_print is the null marker string. Note that this is compared to
|
|
|
|
* the pre-de-escaped input string.
|
|
|
|
*
|
2010-12-06 21:31:55 +01:00
|
|
|
* The return value is the number of fields actually read.
|
2003-09-30 00:06:40 +02:00
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
static int
|
2010-12-06 21:31:55 +01:00
|
|
|
CopyReadAttributesText(CopyState cstate)
|
2003-09-30 00:06:40 +02:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
char delimc = cstate->delim[0];
|
|
|
|
int fieldno;
|
|
|
|
char *output_ptr;
|
|
|
|
char *cur_ptr;
|
|
|
|
char *line_end_ptr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We need a special case for zero-column tables: check that the input
|
|
|
|
* line is empty, and return.
|
|
|
|
*/
|
2010-12-06 21:31:55 +01:00
|
|
|
if (cstate->max_fields <= 0)
|
2005-08-06 22:41:58 +02:00
|
|
|
{
|
|
|
|
if (cstate->line_buf.len != 0)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("extra data after last expected column")));
|
|
|
|
return 0;
|
|
|
|
}
|
2003-09-30 00:06:40 +02:00
|
|
|
|
2007-03-03 20:32:55 +01:00
|
|
|
resetStringInfo(&cstate->attribute_buf);
|
2003-09-30 00:06:40 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
|
|
|
* The de-escaped attributes will certainly not be longer than the input
|
|
|
|
* data line, so we can just force attribute_buf to be large enough and
|
2014-05-06 18:12:18 +02:00
|
|
|
* then transfer data without any checks for enough space. We need to do
|
2005-10-15 04:49:52 +02:00
|
|
|
* it this way because enlarging attribute_buf mid-stream would invalidate
|
2010-12-06 21:31:55 +01:00
|
|
|
* pointers already stored into cstate->raw_fields[].
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
|
|
|
if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
|
|
|
|
enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
|
|
|
|
output_ptr = cstate->attribute_buf.data;
|
|
|
|
|
|
|
|
/* set pointer variables for loop */
|
|
|
|
cur_ptr = cstate->line_buf.data;
|
|
|
|
line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
|
2003-09-30 00:06:40 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* Outer loop iterates over fields */
|
|
|
|
fieldno = 0;
|
2003-09-30 00:06:40 +02:00
|
|
|
for (;;)
|
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
bool found_delim = false;
|
|
|
|
char *start_ptr;
|
|
|
|
char *end_ptr;
|
|
|
|
int input_len;
|
2009-04-19 23:08:54 +02:00
|
|
|
bool saw_non_ascii = false;
|
2005-08-06 22:41:58 +02:00
|
|
|
|
2010-12-06 21:31:55 +01:00
|
|
|
/* Make sure there is enough space for the next value */
|
|
|
|
if (fieldno >= cstate->max_fields)
|
|
|
|
{
|
|
|
|
cstate->max_fields *= 2;
|
2011-04-10 17:42:00 +02:00
|
|
|
cstate->raw_fields =
|
|
|
|
repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
|
2010-12-06 21:31:55 +01:00
|
|
|
}
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
/* Remember start of field on both input and output sides */
|
|
|
|
start_ptr = cur_ptr;
|
2010-12-06 21:31:55 +01:00
|
|
|
cstate->raw_fields[fieldno] = output_ptr;
|
2005-08-06 22:41:58 +02:00
|
|
|
|
2012-03-26 05:17:22 +02:00
|
|
|
/*
|
|
|
|
* Scan data for field.
|
|
|
|
*
|
|
|
|
* Note that in this loop, we are scanning to locate the end of field
|
|
|
|
* and also speculatively performing de-escaping. Once we find the
|
|
|
|
* end-of-field, we can match the raw field contents against the null
|
|
|
|
* marker string. Only after that comparison fails do we know that
|
|
|
|
* de-escaping is actually the right thing to do; therefore we *must
|
|
|
|
* not* throw any syntax errors before we've done the null-marker
|
|
|
|
* check.
|
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
for (;;)
|
2003-09-30 00:06:40 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
char c;
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
end_ptr = cur_ptr;
|
|
|
|
if (cur_ptr >= line_end_ptr)
|
2003-09-30 00:06:40 +02:00
|
|
|
break;
|
2005-08-06 22:41:58 +02:00
|
|
|
c = *cur_ptr++;
|
|
|
|
if (c == delimc)
|
|
|
|
{
|
|
|
|
found_delim = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (c == '\\')
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cur_ptr >= line_end_ptr)
|
|
|
|
break;
|
|
|
|
c = *cur_ptr++;
|
|
|
|
switch (c)
|
|
|
|
{
|
|
|
|
case '0':
|
|
|
|
case '1':
|
|
|
|
case '2':
|
|
|
|
case '3':
|
|
|
|
case '4':
|
|
|
|
case '5':
|
|
|
|
case '6':
|
|
|
|
case '7':
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
/* handle \013 */
|
|
|
|
int val;
|
|
|
|
|
|
|
|
val = OCTVALUE(c);
|
|
|
|
if (cur_ptr < line_end_ptr)
|
1999-05-25 18:15:34 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
c = *cur_ptr;
|
|
|
|
if (ISOCTAL(c))
|
2002-07-30 18:55:06 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
cur_ptr++;
|
|
|
|
val = (val << 3) + OCTVALUE(c);
|
|
|
|
if (cur_ptr < line_end_ptr)
|
2003-09-30 00:06:40 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
c = *cur_ptr;
|
|
|
|
if (ISOCTAL(c))
|
|
|
|
{
|
|
|
|
cur_ptr++;
|
|
|
|
val = (val << 3) + OCTVALUE(c);
|
|
|
|
}
|
2003-09-30 00:06:40 +02:00
|
|
|
}
|
2002-07-30 18:55:06 +02:00
|
|
|
}
|
1997-09-08 04:41:22 +02:00
|
|
|
}
|
2005-10-15 04:49:52 +02:00
|
|
|
c = val & 0377;
|
2009-04-19 23:08:54 +02:00
|
|
|
if (c == '\0' || IS_HIGHBIT_SET(c))
|
|
|
|
saw_non_ascii = true;
|
1997-09-07 07:04:48 +02:00
|
|
|
}
|
2005-10-15 04:49:52 +02:00
|
|
|
break;
|
2005-08-06 22:41:58 +02:00
|
|
|
case 'x':
|
|
|
|
/* Handle \x3F */
|
|
|
|
if (cur_ptr < line_end_ptr)
|
2005-06-02 03:21:22 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
char hexchar = *cur_ptr;
|
2005-06-02 03:21:22 +02:00
|
|
|
|
2005-09-01 17:34:31 +02:00
|
|
|
if (isxdigit((unsigned char) hexchar))
|
2005-06-02 03:21:22 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
int val = GetDecimalFromHex(hexchar);
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
cur_ptr++;
|
|
|
|
if (cur_ptr < line_end_ptr)
|
2005-06-02 03:21:22 +02:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
hexchar = *cur_ptr;
|
2005-09-01 17:34:31 +02:00
|
|
|
if (isxdigit((unsigned char) hexchar))
|
2005-08-06 22:41:58 +02:00
|
|
|
{
|
|
|
|
cur_ptr++;
|
|
|
|
val = (val << 4) + GetDecimalFromHex(hexchar);
|
|
|
|
}
|
2005-06-02 03:21:22 +02:00
|
|
|
}
|
2005-08-06 22:41:58 +02:00
|
|
|
c = val & 0xff;
|
2009-04-19 23:08:54 +02:00
|
|
|
if (c == '\0' || IS_HIGHBIT_SET(c))
|
|
|
|
saw_non_ascii = true;
|
2005-06-02 03:21:22 +02:00
|
|
|
}
|
|
|
|
}
|
2005-08-06 22:41:58 +02:00
|
|
|
break;
|
|
|
|
case 'b':
|
|
|
|
c = '\b';
|
|
|
|
break;
|
|
|
|
case 'f':
|
|
|
|
c = '\f';
|
|
|
|
break;
|
|
|
|
case 'n':
|
|
|
|
c = '\n';
|
|
|
|
break;
|
|
|
|
case 'r':
|
|
|
|
c = '\r';
|
|
|
|
break;
|
|
|
|
case 't':
|
|
|
|
c = '\t';
|
|
|
|
break;
|
|
|
|
case 'v':
|
|
|
|
c = '\v';
|
|
|
|
break;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* in all other cases, take the char after '\'
|
|
|
|
* literally
|
|
|
|
*/
|
|
|
|
}
|
2007-11-15 22:14:46 +01:00
|
|
|
}
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
/* Add c to output string */
|
|
|
|
*output_ptr++ = c;
|
1997-09-07 07:04:48 +02:00
|
|
|
}
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
/* Check whether raw input matched null marker */
|
|
|
|
input_len = end_ptr - start_ptr;
|
|
|
|
if (input_len == cstate->null_print_len &&
|
|
|
|
strncmp(start_ptr, cstate->null_print, input_len) == 0)
|
2010-12-06 21:31:55 +01:00
|
|
|
cstate->raw_fields[fieldno] = NULL;
|
2012-03-26 05:17:22 +02:00
|
|
|
else
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* At this point we know the field is supposed to contain data.
|
|
|
|
*
|
|
|
|
* If we de-escaped any non-7-bit-ASCII chars, make sure the
|
|
|
|
* resulting string is valid data for the db encoding.
|
|
|
|
*/
|
|
|
|
if (saw_non_ascii)
|
|
|
|
{
|
|
|
|
char *fld = cstate->raw_fields[fieldno];
|
|
|
|
|
|
|
|
pg_verifymbstr(fld, output_ptr - fld, false);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Terminate attribute value in output area */
|
|
|
|
*output_ptr++ = '\0';
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
fieldno++;
|
|
|
|
/* Done if we hit EOL instead of a delim */
|
|
|
|
if (!found_delim)
|
|
|
|
break;
|
1999-09-12 00:28:11 +02:00
|
|
|
}
|
2000-01-16 22:37:50 +01:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* Clean up state of attribute_buf */
|
|
|
|
output_ptr--;
|
|
|
|
Assert(*output_ptr == '\0');
|
|
|
|
cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
|
2003-10-06 04:38:53 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
return fieldno;
|
1996-07-09 08:22:35 +02:00
|
|
|
}
|
|
|
|
|
2004-04-19 19:22:31 +02:00
|
|
|
/*
|
2005-08-06 22:41:58 +02:00
|
|
|
* Parse the current line into separate attributes (fields),
|
|
|
|
* performing de-escaping as needed. This has exactly the same API as
|
|
|
|
* CopyReadAttributesText, except we parse the fields according to
|
|
|
|
* "standard" (i.e. common) CSV usage.
|
2004-04-19 19:22:31 +02:00
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
static int
|
2010-12-06 21:31:55 +01:00
|
|
|
CopyReadAttributesCSV(CopyState cstate)
|
2004-04-19 19:22:31 +02:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
char delimc = cstate->delim[0];
|
|
|
|
char quotec = cstate->quote[0];
|
|
|
|
char escapec = cstate->escape[0];
|
|
|
|
int fieldno;
|
|
|
|
char *output_ptr;
|
|
|
|
char *cur_ptr;
|
|
|
|
char *line_end_ptr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We need a special case for zero-column tables: check that the input
|
|
|
|
* line is empty, and return.
|
|
|
|
*/
|
2010-12-06 21:31:55 +01:00
|
|
|
if (cstate->max_fields <= 0)
|
2005-08-06 22:41:58 +02:00
|
|
|
{
|
|
|
|
if (cstate->line_buf.len != 0)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("extra data after last expected column")));
|
|
|
|
return 0;
|
|
|
|
}
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2007-03-03 20:32:55 +01:00
|
|
|
resetStringInfo(&cstate->attribute_buf);
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/*
|
|
|
|
* The de-escaped attributes will certainly not be longer than the input
|
|
|
|
* data line, so we can just force attribute_buf to be large enough and
|
2014-05-06 18:12:18 +02:00
|
|
|
* then transfer data without any checks for enough space. We need to do
|
2005-10-15 04:49:52 +02:00
|
|
|
* it this way because enlarging attribute_buf mid-stream would invalidate
|
2010-12-06 21:31:55 +01:00
|
|
|
* pointers already stored into cstate->raw_fields[].
|
2005-08-06 22:41:58 +02:00
|
|
|
*/
|
|
|
|
if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
|
|
|
|
enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
|
|
|
|
output_ptr = cstate->attribute_buf.data;
|
|
|
|
|
|
|
|
/* set pointer variables for loop */
|
|
|
|
cur_ptr = cstate->line_buf.data;
|
|
|
|
line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* Outer loop iterates over fields */
|
|
|
|
fieldno = 0;
|
2004-04-19 19:22:31 +02:00
|
|
|
for (;;)
|
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
bool found_delim = false;
|
|
|
|
bool saw_quote = false;
|
|
|
|
char *start_ptr;
|
|
|
|
char *end_ptr;
|
|
|
|
int input_len;
|
|
|
|
|
2010-12-06 21:31:55 +01:00
|
|
|
/* Make sure there is enough space for the next value */
|
|
|
|
if (fieldno >= cstate->max_fields)
|
|
|
|
{
|
|
|
|
cstate->max_fields *= 2;
|
2011-04-10 17:42:00 +02:00
|
|
|
cstate->raw_fields =
|
|
|
|
repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
|
2010-12-06 21:31:55 +01:00
|
|
|
}
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* Remember start of field on both input and output sides */
|
|
|
|
start_ptr = cur_ptr;
|
2010-12-06 21:31:55 +01:00
|
|
|
cstate->raw_fields[fieldno] = output_ptr;
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2009-06-11 16:49:15 +02:00
|
|
|
/*
|
|
|
|
* Scan data for field,
|
2008-03-08 02:16:26 +01:00
|
|
|
*
|
2009-06-11 16:49:15 +02:00
|
|
|
* The loop starts in "not quote" mode and then toggles between that
|
|
|
|
* and "in quote" mode. The loop exits normally if it is in "not
|
|
|
|
* quote" mode and a delimiter or line end is seen.
|
2008-03-08 02:16:26 +01:00
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
for (;;)
|
2004-04-19 19:22:31 +02:00
|
|
|
{
|
2005-10-15 04:49:52 +02:00
|
|
|
char c;
|
2004-08-29 07:07:03 +02:00
|
|
|
|
2008-03-08 02:16:26 +01:00
|
|
|
/* Not in quote */
|
|
|
|
for (;;)
|
2005-08-06 22:41:58 +02:00
|
|
|
{
|
2008-03-08 02:16:26 +01:00
|
|
|
end_ptr = cur_ptr;
|
|
|
|
if (cur_ptr >= line_end_ptr)
|
|
|
|
goto endfield;
|
|
|
|
c = *cur_ptr++;
|
|
|
|
/* unquoted field delimiter */
|
|
|
|
if (c == delimc)
|
|
|
|
{
|
|
|
|
found_delim = true;
|
|
|
|
goto endfield;
|
|
|
|
}
|
|
|
|
/* start of quoted field (or part of field) */
|
|
|
|
if (c == quotec)
|
|
|
|
{
|
|
|
|
saw_quote = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
/* Add c to output string */
|
|
|
|
*output_ptr++ = c;
|
2005-08-06 22:41:58 +02:00
|
|
|
}
|
2008-03-08 02:16:26 +01:00
|
|
|
|
|
|
|
/* In quote */
|
|
|
|
for (;;)
|
2005-08-06 22:41:58 +02:00
|
|
|
{
|
2008-03-08 02:16:26 +01:00
|
|
|
end_ptr = cur_ptr;
|
|
|
|
if (cur_ptr >= line_end_ptr)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("unterminated CSV quoted field")));
|
|
|
|
|
|
|
|
c = *cur_ptr++;
|
2005-08-06 22:41:58 +02:00
|
|
|
|
2008-03-08 02:16:26 +01:00
|
|
|
/* escape within a quoted field */
|
|
|
|
if (c == escapec)
|
|
|
|
{
|
|
|
|
/*
|
2009-06-11 16:49:15 +02:00
|
|
|
* peek at the next char if available, and escape it if it
|
|
|
|
* is an escape char or a quote char
|
2008-03-08 02:16:26 +01:00
|
|
|
*/
|
|
|
|
if (cur_ptr < line_end_ptr)
|
2005-08-06 22:41:58 +02:00
|
|
|
{
|
2008-03-08 02:16:26 +01:00
|
|
|
char nextc = *cur_ptr;
|
|
|
|
|
|
|
|
if (nextc == escapec || nextc == quotec)
|
|
|
|
{
|
|
|
|
*output_ptr++ = nextc;
|
|
|
|
cur_ptr++;
|
|
|
|
continue;
|
|
|
|
}
|
2005-08-06 22:41:58 +02:00
|
|
|
}
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
2009-06-11 16:49:15 +02:00
|
|
|
|
2008-03-08 02:16:26 +01:00
|
|
|
/*
|
2009-06-11 16:49:15 +02:00
|
|
|
* end of quoted field. Must do this test after testing for
|
|
|
|
* escape in case quote char and escape char are the same
|
|
|
|
* (which is the common case).
|
2008-03-08 02:16:26 +01:00
|
|
|
*/
|
|
|
|
if (c == quotec)
|
|
|
|
break;
|
2005-10-15 04:49:52 +02:00
|
|
|
|
2008-03-08 02:16:26 +01:00
|
|
|
/* Add c to output string */
|
|
|
|
*output_ptr++ = c;
|
2005-08-06 22:41:58 +02:00
|
|
|
}
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
2009-06-11 16:49:15 +02:00
|
|
|
endfield:
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* Terminate attribute value in output area */
|
|
|
|
*output_ptr++ = '\0';
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* Check whether raw input matched null marker */
|
|
|
|
input_len = end_ptr - start_ptr;
|
|
|
|
if (!saw_quote && input_len == cstate->null_print_len &&
|
|
|
|
strncmp(start_ptr, cstate->null_print, input_len) == 0)
|
2010-12-06 21:31:55 +01:00
|
|
|
cstate->raw_fields[fieldno] = NULL;
|
2005-08-06 22:41:58 +02:00
|
|
|
|
|
|
|
fieldno++;
|
|
|
|
/* Done if we hit EOL instead of a delim */
|
|
|
|
if (!found_delim)
|
|
|
|
break;
|
|
|
|
}
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
/* Clean up state of attribute_buf */
|
|
|
|
output_ptr--;
|
|
|
|
Assert(*output_ptr == '\0');
|
|
|
|
cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
|
|
|
|
|
|
|
|
return fieldno;
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
|
2003-05-09 23:19:50 +02:00
|
|
|
/*
|
|
|
|
* Read a binary attribute
|
|
|
|
*/
|
|
|
|
static Datum
|
2005-08-06 22:41:58 +02:00
|
|
|
CopyReadBinaryAttribute(CopyState cstate,
|
|
|
|
int column_no, FmgrInfo *flinfo,
|
2005-07-10 23:14:00 +02:00
|
|
|
Oid typioparam, int32 typmod,
|
2003-05-09 23:19:50 +02:00
|
|
|
bool *isnull)
|
|
|
|
{
|
|
|
|
int32 fld_size;
|
|
|
|
Datum result;
|
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!CopyGetInt32(cstate, &fld_size))
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("unexpected EOF in COPY data")));
|
2003-05-09 23:19:50 +02:00
|
|
|
if (fld_size == -1)
|
|
|
|
{
|
|
|
|
*isnull = true;
|
2006-04-04 21:35:37 +02:00
|
|
|
return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
|
2003-05-09 23:19:50 +02:00
|
|
|
}
|
|
|
|
if (fld_size < 0)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
2003-09-30 00:06:40 +02:00
|
|
|
errmsg("invalid field size")));
|
2003-05-09 23:19:50 +02:00
|
|
|
|
|
|
|
/* reset attribute_buf to empty, and load raw data in it */
|
2007-03-03 20:32:55 +01:00
|
|
|
resetStringInfo(&cstate->attribute_buf);
|
2003-05-09 23:19:50 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
enlargeStringInfo(&cstate->attribute_buf, fld_size);
|
|
|
|
if (CopyGetData(cstate, cstate->attribute_buf.data,
|
|
|
|
fld_size, fld_size) != fld_size)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
|
|
|
|
errmsg("unexpected EOF in COPY data")));
|
2003-05-09 23:19:50 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
cstate->attribute_buf.len = fld_size;
|
|
|
|
cstate->attribute_buf.data[fld_size] = '\0';
|
2003-05-09 23:19:50 +02:00
|
|
|
|
|
|
|
/* Call the column type's binary input converter */
|
2006-04-04 21:35:37 +02:00
|
|
|
result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
|
|
|
|
typioparam, typmod);
|
2003-05-09 23:19:50 +02:00
|
|
|
|
|
|
|
/* Trouble if it didn't eat the whole buffer */
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
2003-07-22 21:00:12 +02:00
|
|
|
(errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
|
2003-09-30 00:06:40 +02:00
|
|
|
errmsg("incorrect binary data format")));
|
2003-05-09 23:19:50 +02:00
|
|
|
|
|
|
|
*isnull = false;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Send text representation of one attribute, with conversion and escaping
|
|
|
|
*/
|
2006-05-25 20:42:17 +02:00
|
|
|
#define DUMPSOFAR() \
|
|
|
|
do { \
|
|
|
|
if (ptr > start) \
|
|
|
|
CopySendData(cstate, start, ptr - start); \
|
|
|
|
} while (0)
|
|
|
|
|
1996-07-23 04:23:54 +02:00
|
|
|
static void
|
2006-05-25 20:42:17 +02:00
|
|
|
CopyAttributeOutText(CopyState cstate, char *string)
|
1996-07-23 04:23:54 +02:00
|
|
|
{
|
2006-05-25 20:42:17 +02:00
|
|
|
char *ptr;
|
|
|
|
char *start;
|
1998-07-27 21:38:40 +02:00
|
|
|
char c;
|
2005-08-06 22:41:58 +02:00
|
|
|
char delimc = cstate->delim[0];
|
1998-08-24 03:14:24 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
if (cstate->need_transcoding)
|
2011-02-21 06:08:04 +01:00
|
|
|
ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
|
2001-01-06 04:33:17 +01:00
|
|
|
else
|
2006-05-25 20:42:17 +02:00
|
|
|
ptr = string;
|
1998-08-24 03:14:24 +02:00
|
|
|
|
2006-05-25 20:42:17 +02:00
|
|
|
/*
|
|
|
|
* We have to grovel through the string searching for control characters
|
|
|
|
* and instances of the delimiter character. In most cases, though, these
|
2014-05-06 18:12:18 +02:00
|
|
|
* are infrequent. To avoid overhead from calling CopySendData once per
|
2007-11-15 22:14:46 +01:00
|
|
|
* character, we dump out all characters between escaped characters in a
|
|
|
|
* single call. The loop invariant is that the data from "start" to "ptr"
|
|
|
|
* can be sent literally, but hasn't yet been.
|
2007-06-18 01:39:28 +02:00
|
|
|
*
|
|
|
|
* We can skip pg_encoding_mblen() overhead when encoding is safe, because
|
|
|
|
* in valid backend encodings, extra bytes of a multibyte character never
|
|
|
|
* look like ASCII. This loop is sufficiently performance-critical that
|
2007-11-15 22:14:46 +01:00
|
|
|
* it's worth making two copies of it to get the IS_HIGHBIT_SET() test out
|
|
|
|
* of the normal safe-encoding path.
|
2006-05-25 20:42:17 +02:00
|
|
|
*/
|
2007-06-18 01:39:28 +02:00
|
|
|
if (cstate->encoding_embeds_ascii)
|
1997-09-07 07:04:48 +02:00
|
|
|
{
|
2007-06-18 01:39:28 +02:00
|
|
|
start = ptr;
|
|
|
|
while ((c = *ptr) != '\0')
|
2002-02-12 22:25:41 +01:00
|
|
|
{
|
2007-12-27 17:45:22 +01:00
|
|
|
if ((unsigned char) c < (unsigned char) 0x20)
|
2007-06-18 01:39:28 +02:00
|
|
|
{
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
/*
|
2009-06-11 16:49:15 +02:00
|
|
|
* \r and \n must be escaped, the others are traditional. We
|
|
|
|
* prefer to dump these using the C-like notation, rather than
|
|
|
|
* a backslash and the literal character, because it makes the
|
|
|
|
* dump file a bit more proof against Microsoftish data
|
|
|
|
* mangling.
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
*/
|
2007-06-18 01:39:28 +02:00
|
|
|
switch (c)
|
|
|
|
{
|
|
|
|
case '\b':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 'b';
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
case '\f':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 'f';
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
case '\n':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 'n';
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
case '\r':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 'r';
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
case '\t':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 't';
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
case '\v':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 'v';
|
2007-06-18 01:39:28 +02:00
|
|
|
break;
|
|
|
|
default:
|
2007-12-27 17:45:22 +01:00
|
|
|
/* If it's the delimiter, must backslash it */
|
|
|
|
if (c == delimc)
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
/* All ASCII control chars are length 1 */
|
|
|
|
ptr++;
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
continue; /* fall to end of loop */
|
2007-06-18 01:39:28 +02:00
|
|
|
}
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
/* if we get here, we need to convert the control char */
|
|
|
|
DUMPSOFAR();
|
|
|
|
CopySendChar(cstate, '\\');
|
|
|
|
CopySendChar(cstate, c);
|
2009-06-11 16:49:15 +02:00
|
|
|
start = ++ptr; /* do not include char in next run */
|
2007-06-18 01:39:28 +02:00
|
|
|
}
|
2007-12-27 17:45:22 +01:00
|
|
|
else if (c == '\\' || c == delimc)
|
|
|
|
{
|
|
|
|
DUMPSOFAR();
|
|
|
|
CopySendChar(cstate, '\\');
|
|
|
|
start = ptr++; /* we include char in next run */
|
|
|
|
}
|
2007-06-18 01:39:28 +02:00
|
|
|
else if (IS_HIGHBIT_SET(c))
|
2011-02-21 06:08:04 +01:00
|
|
|
ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
|
2007-06-18 01:39:28 +02:00
|
|
|
else
|
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
start = ptr;
|
|
|
|
while ((c = *ptr) != '\0')
|
|
|
|
{
|
2007-12-27 17:45:22 +01:00
|
|
|
if ((unsigned char) c < (unsigned char) 0x20)
|
2007-06-18 01:39:28 +02:00
|
|
|
{
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
/*
|
2009-06-11 16:49:15 +02:00
|
|
|
* \r and \n must be escaped, the others are traditional. We
|
|
|
|
* prefer to dump these using the C-like notation, rather than
|
|
|
|
* a backslash and the literal character, because it makes the
|
|
|
|
* dump file a bit more proof against Microsoftish data
|
|
|
|
* mangling.
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
*/
|
2007-06-18 01:39:28 +02:00
|
|
|
switch (c)
|
2006-05-25 20:42:17 +02:00
|
|
|
{
|
2007-06-18 01:39:28 +02:00
|
|
|
case '\b':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 'b';
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
case '\f':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 'f';
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
case '\n':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 'n';
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
case '\r':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 'r';
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
case '\t':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 't';
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
case '\v':
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
c = 'v';
|
2007-06-18 01:39:28 +02:00
|
|
|
break;
|
|
|
|
default:
|
2007-12-27 17:45:22 +01:00
|
|
|
/* If it's the delimiter, must backslash it */
|
|
|
|
if (c == delimc)
|
|
|
|
break;
|
2007-06-18 01:39:28 +02:00
|
|
|
/* All ASCII control chars are length 1 */
|
|
|
|
ptr++;
|
Phase 2 of pgindent updates.
Change pg_bsd_indent to follow upstream rules for placement of comments
to the right of code, and remove pgindent hack that caused comments
following #endif to not obey the general rule.
Commit e3860ffa4dd0dad0dd9eea4be9cc1412373a8c89 wasn't actually using
the published version of pg_bsd_indent, but a hacked-up version that
tried to minimize the amount of movement of comments to the right of
code. The situation of interest is where such a comment has to be
moved to the right of its default placement at column 33 because there's
code there. BSD indent has always moved right in units of tab stops
in such cases --- but in the previous incarnation, indent was working
in 8-space tab stops, while now it knows we use 4-space tabs. So the
net result is that in about half the cases, such comments are placed
one tab stop left of before. This is better all around: it leaves
more room on the line for comment text, and it means that in such
cases the comment uniformly starts at the next 4-space tab stop after
the code, rather than sometimes one and sometimes two tabs after.
Also, ensure that comments following #endif are indented the same
as comments following other preprocessor commands such as #else.
That inconsistency turns out to have been self-inflicted damage
from a poorly-thought-through post-indent "fixup" in pgindent.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:18:54 +02:00
|
|
|
continue; /* fall to end of loop */
|
2006-05-25 20:42:17 +02:00
|
|
|
}
|
Revert COPY OUT to follow the pre-8.3 handling of ASCII control characters,
namely that \r, \n, \t, \b, \f, \v are dumped as those two-character
representations rather than a backslash and the literal control character.
I had made it do the other to save some code, but this was ill-advised,
because dump files in which these characters appear literally are prone to
newline mangling. Fortunately, doing it the old way should only cost a few
more lines of code, and not slow down the copy loop materially.
Per bug #3795 from Lou Duchez.
2007-12-03 01:03:05 +01:00
|
|
|
/* if we get here, we need to convert the control char */
|
|
|
|
DUMPSOFAR();
|
|
|
|
CopySendChar(cstate, '\\');
|
|
|
|
CopySendChar(cstate, c);
|
2009-06-11 16:49:15 +02:00
|
|
|
start = ++ptr; /* do not include char in next run */
|
2007-06-18 01:39:28 +02:00
|
|
|
}
|
2007-12-27 17:45:22 +01:00
|
|
|
else if (c == '\\' || c == delimc)
|
|
|
|
{
|
|
|
|
DUMPSOFAR();
|
|
|
|
CopySendChar(cstate, '\\');
|
|
|
|
start = ptr++; /* we include char in next run */
|
|
|
|
}
|
2007-06-18 01:39:28 +02:00
|
|
|
else
|
|
|
|
ptr++;
|
2002-02-12 22:25:41 +01:00
|
|
|
}
|
1997-09-07 07:04:48 +02:00
|
|
|
}
|
2006-05-25 20:42:17 +02:00
|
|
|
|
|
|
|
DUMPSOFAR();
|
1996-07-23 04:23:54 +02:00
|
|
|
}
|
2002-07-18 06:43:51 +02:00
|
|
|
|
2004-04-19 19:22:31 +02:00
|
|
|
/*
|
2006-05-25 20:42:17 +02:00
|
|
|
* Send text representation of one attribute, with conversion and
|
|
|
|
* CSV-style escaping
|
2004-04-19 19:22:31 +02:00
|
|
|
*/
|
|
|
|
static void
|
2006-05-25 20:42:17 +02:00
|
|
|
CopyAttributeOutCSV(CopyState cstate, char *string,
|
2005-12-28 04:25:32 +01:00
|
|
|
bool use_quote, bool single_attr)
|
2004-04-19 19:22:31 +02:00
|
|
|
{
|
2006-05-25 20:42:17 +02:00
|
|
|
char *ptr;
|
|
|
|
char *start;
|
2004-04-19 19:22:31 +02:00
|
|
|
char c;
|
2005-08-06 22:41:58 +02:00
|
|
|
char delimc = cstate->delim[0];
|
|
|
|
char quotec = cstate->quote[0];
|
|
|
|
char escapec = cstate->escape[0];
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2006-05-25 20:42:17 +02:00
|
|
|
/* force quoting if it matches null_print (before conversion!) */
|
|
|
|
if (!use_quote && strcmp(string, cstate->null_print) == 0)
|
2005-08-06 22:41:58 +02:00
|
|
|
use_quote = true;
|
|
|
|
|
|
|
|
if (cstate->need_transcoding)
|
2011-02-21 06:08:04 +01:00
|
|
|
ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding);
|
2004-04-19 19:22:31 +02:00
|
|
|
else
|
2006-05-25 20:42:17 +02:00
|
|
|
ptr = string;
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2004-08-29 07:07:03 +02:00
|
|
|
/*
|
2006-05-25 20:42:17 +02:00
|
|
|
* Make a preliminary pass to discover if it needs quoting
|
2004-04-19 19:22:31 +02:00
|
|
|
*/
|
2005-08-06 22:41:58 +02:00
|
|
|
if (!use_quote)
|
2004-04-19 19:22:31 +02:00
|
|
|
{
|
2005-12-28 04:25:32 +01:00
|
|
|
/*
|
2006-10-04 02:30:14 +02:00
|
|
|
* Because '\.' can be a data value, quote it if it appears alone on a
|
|
|
|
* line so it is not interpreted as the end-of-data marker.
|
2005-12-28 04:25:32 +01:00
|
|
|
*/
|
2006-05-25 20:42:17 +02:00
|
|
|
if (single_attr && strcmp(ptr, "\\.") == 0)
|
2006-10-04 02:30:14 +02:00
|
|
|
use_quote = true;
|
|
|
|
else
|
|
|
|
{
|
2006-05-25 20:42:17 +02:00
|
|
|
char *tptr = ptr;
|
|
|
|
|
|
|
|
while ((c = *tptr) != '\0')
|
2005-08-06 22:41:58 +02:00
|
|
|
{
|
2005-12-28 04:25:32 +01:00
|
|
|
if (c == delimc || c == quotec || c == '\n' || c == '\r')
|
|
|
|
{
|
|
|
|
use_quote = true;
|
|
|
|
break;
|
|
|
|
}
|
2006-05-25 20:42:17 +02:00
|
|
|
if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
|
2011-02-21 06:08:04 +01:00
|
|
|
tptr += pg_encoding_mblen(cstate->file_encoding, tptr);
|
2005-12-28 04:25:32 +01:00
|
|
|
else
|
2006-05-25 20:42:17 +02:00
|
|
|
tptr++;
|
2005-08-06 22:41:58 +02:00
|
|
|
}
|
|
|
|
}
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
|
|
|
|
2004-04-21 02:34:18 +02:00
|
|
|
if (use_quote)
|
2006-05-25 20:42:17 +02:00
|
|
|
{
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendChar(cstate, quotec);
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2006-05-25 20:42:17 +02:00
|
|
|
/*
|
|
|
|
* We adopt the same optimization strategy as in CopyAttributeOutText
|
|
|
|
*/
|
|
|
|
start = ptr;
|
|
|
|
while ((c = *ptr) != '\0')
|
|
|
|
{
|
|
|
|
if (c == quotec || c == escapec)
|
|
|
|
{
|
|
|
|
DUMPSOFAR();
|
|
|
|
CopySendChar(cstate, escapec);
|
|
|
|
start = ptr; /* we include char in next run */
|
|
|
|
}
|
|
|
|
if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii)
|
2011-02-21 06:08:04 +01:00
|
|
|
ptr += pg_encoding_mblen(cstate->file_encoding, ptr);
|
2006-05-25 20:42:17 +02:00
|
|
|
else
|
|
|
|
ptr++;
|
|
|
|
}
|
|
|
|
DUMPSOFAR();
|
2004-04-19 19:22:31 +02:00
|
|
|
|
2005-08-06 22:41:58 +02:00
|
|
|
CopySendChar(cstate, quotec);
|
2006-05-25 20:42:17 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* If it doesn't need quoting, we can just dump it as-is */
|
|
|
|
CopySendString(cstate, ptr);
|
|
|
|
}
|
2004-04-19 19:22:31 +02:00
|
|
|
}
|
|
|
|
|
2002-07-18 06:43:51 +02:00
|
|
|
/*
|
2002-08-02 20:15:10 +02:00
|
|
|
* CopyGetAttnums - build an integer list of attnums to be copied
|
|
|
|
*
|
|
|
|
* The input attnamelist is either the user-specified column list,
|
|
|
|
* or NIL if there was none (in which case we want all the non-dropped
|
|
|
|
* columns).
|
2006-08-31 01:34:22 +02:00
|
|
|
*
|
2019-03-30 08:13:09 +01:00
|
|
|
* We don't include generated columns in the generated full list and we don't
|
|
|
|
* allow them to be specified explicitly. They don't make sense for COPY
|
|
|
|
* FROM, but we could possibly allow them for COPY TO. But this way it's at
|
|
|
|
* least ensured that whatever we copy out can be copied back in.
|
|
|
|
*
|
2006-08-31 01:34:22 +02:00
|
|
|
* rel can be NULL ... it's only used for error reports.
|
2002-07-18 06:43:51 +02:00
|
|
|
*/
|
2002-08-02 20:15:10 +02:00
|
|
|
static List *
|
2006-08-31 01:34:22 +02:00
|
|
|
CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
|
2002-07-18 06:43:51 +02:00
|
|
|
{
|
2002-09-04 22:31:48 +02:00
|
|
|
List *attnums = NIL;
|
2002-07-30 18:55:06 +02:00
|
|
|
|
2002-08-02 20:15:10 +02:00
|
|
|
if (attnamelist == NIL)
|
2002-07-30 18:55:06 +02:00
|
|
|
{
|
2002-08-02 20:15:10 +02:00
|
|
|
/* Generate default column list */
|
2002-09-04 22:31:48 +02:00
|
|
|
int attr_count = tupDesc->natts;
|
2002-08-02 20:15:10 +02:00
|
|
|
int i;
|
2002-07-30 18:55:06 +02:00
|
|
|
|
|
|
|
for (i = 0; i < attr_count; i++)
|
|
|
|
{
|
2017-08-20 20:19:07 +02:00
|
|
|
if (TupleDescAttr(tupDesc, i)->attisdropped)
|
2002-08-02 20:15:10 +02:00
|
|
|
continue;
|
2019-03-30 08:13:09 +01:00
|
|
|
if (TupleDescAttr(tupDesc, i)->attgenerated)
|
|
|
|
continue;
|
2004-05-26 06:41:50 +02:00
|
|
|
attnums = lappend_int(attnums, i + 1);
|
2002-08-02 20:15:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* Validate the user-supplied list and extract attnums */
|
2004-05-26 06:41:50 +02:00
|
|
|
ListCell *l;
|
2002-08-02 20:15:10 +02:00
|
|
|
|
|
|
|
foreach(l, attnamelist)
|
|
|
|
{
|
2002-08-19 17:08:47 +02:00
|
|
|
char *name = strVal(lfirst(l));
|
2002-08-02 20:15:10 +02:00
|
|
|
int attnum;
|
2006-08-31 01:34:22 +02:00
|
|
|
int i;
|
2002-08-02 20:15:10 +02:00
|
|
|
|
2006-03-23 01:19:30 +01:00
|
|
|
/* Lookup column name */
|
2006-08-31 01:34:22 +02:00
|
|
|
attnum = InvalidAttrNumber;
|
|
|
|
for (i = 0; i < tupDesc->natts; i++)
|
|
|
|
{
|
2017-08-20 20:19:07 +02:00
|
|
|
Form_pg_attribute att = TupleDescAttr(tupDesc, i);
|
|
|
|
|
|
|
|
if (att->attisdropped)
|
2006-08-31 01:34:22 +02:00
|
|
|
continue;
|
2017-08-20 20:19:07 +02:00
|
|
|
if (namestrcmp(&(att->attname), name) == 0)
|
2006-08-31 01:34:22 +02:00
|
|
|
{
|
2019-03-30 08:13:09 +01:00
|
|
|
if (att->attgenerated)
|
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
|
|
|
|
errmsg("column \"%s\" is a generated column",
|
|
|
|
name),
|
|
|
|
errdetail("Generated columns cannot be used in COPY.")));
|
2017-08-20 20:19:07 +02:00
|
|
|
attnum = att->attnum;
|
2006-08-31 01:34:22 +02:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2006-03-23 01:19:30 +01:00
|
|
|
if (attnum == InvalidAttrNumber)
|
2006-08-31 01:34:22 +02:00
|
|
|
{
|
|
|
|
if (rel != NULL)
|
|
|
|
ereport(ERROR,
|
2006-10-04 02:30:14 +02:00
|
|
|
(errcode(ERRCODE_UNDEFINED_COLUMN),
|
Phase 3 of pgindent updates.
Don't move parenthesized lines to the left, even if that means they
flow past the right margin.
By default, BSD indent lines up statement continuation lines that are
within parentheses so that they start just to the right of the preceding
left parenthesis. However, traditionally, if that resulted in the
continuation line extending to the right of the desired right margin,
then indent would push it left just far enough to not overrun the margin,
if it could do so without making the continuation line start to the left of
the current statement indent. That makes for a weird mix of indentations
unless one has been completely rigid about never violating the 80-column
limit.
This behavior has been pretty universally panned by Postgres developers.
Hence, disable it with indent's new -lpl switch, so that parenthesized
lines are always lined up with the preceding left paren.
This patch is much less interesting than the first round of indent
changes, but also bulkier, so I thought it best to separate the effects.
Discussion: https://postgr.es/m/E1dAmxK-0006EE-1r@gemulon.postgresql.org
Discussion: https://postgr.es/m/30527.1495162840@sss.pgh.pa.us
2017-06-21 21:35:54 +02:00
|
|
|
errmsg("column \"%s\" of relation \"%s\" does not exist",
|
|
|
|
name, RelationGetRelationName(rel))));
|
2006-08-31 01:34:22 +02:00
|
|
|
else
|
|
|
|
ereport(ERROR,
|
2006-10-04 02:30:14 +02:00
|
|
|
(errcode(ERRCODE_UNDEFINED_COLUMN),
|
|
|
|
errmsg("column \"%s\" does not exist",
|
|
|
|
name)));
|
2006-08-31 01:34:22 +02:00
|
|
|
}
|
2002-08-02 20:15:10 +02:00
|
|
|
/* Check for duplicates */
|
2004-05-26 06:41:50 +02:00
|
|
|
if (list_member_int(attnums, attnum))
|
2003-07-20 23:56:35 +02:00
|
|
|
ereport(ERROR,
|
|
|
|
(errcode(ERRCODE_DUPLICATE_COLUMN),
|
2004-08-29 07:07:03 +02:00
|
|
|
errmsg("column \"%s\" specified more than once",
|
|
|
|
name)));
|
2004-05-26 06:41:50 +02:00
|
|
|
attnums = lappend_int(attnums, attnum);
|
2002-07-30 18:55:06 +02:00
|
|
|
}
|
|
|
|
}
|
2002-07-18 06:43:51 +02:00
|
|
|
|
2002-08-02 20:15:10 +02:00
|
|
|
return attnums;
|
|
|
|
}
|
2006-08-31 01:34:22 +02:00
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* copy_dest_startup --- executor startup
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
copy_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo)
|
|
|
|
{
|
|
|
|
/* no-op */
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* copy_dest_receive --- receive one tuple
|
|
|
|
*/
|
2016-06-06 20:52:58 +02:00
|
|
|
static bool
|
2006-08-31 01:34:22 +02:00
|
|
|
copy_dest_receive(TupleTableSlot *slot, DestReceiver *self)
|
|
|
|
{
|
2006-10-04 02:30:14 +02:00
|
|
|
DR_copy *myState = (DR_copy *) self;
|
2006-08-31 01:34:22 +02:00
|
|
|
CopyState cstate = myState->cstate;
|
|
|
|
|
tableam: Add table_multi_insert() and revamp/speed-up COPY FROM buffering.
This adds table_multi_insert(), and converts COPY FROM, the only user
of heap_multi_insert, to it.
A simple conversion of COPY FROM use slots would have yielded a
slowdown when inserting into a partitioned table for some
workloads. Different partitions might need different slots (both slot
types and their descriptors), and dropping / creating slots when
there's constant partition changes is measurable.
Thus instead revamp the COPY FROM buffering for partitioned tables to
allow to buffer inserts into multiple tables, flushing only when
limits are reached across all partition buffers. By only dropping
slots when there've been inserts into too many different partitions,
the aforementioned overhead is gone. By allowing larger batches, even
when there are frequent partition changes, we actuall speed such cases
up significantly.
By using slots COPY of very narrow rows into unlogged / temporary
might slow down very slightly (due to the indirect function calls).
Author: David Rowley, Andres Freund, Haribabu Kommi
Discussion:
https://postgr.es/m/20180703070645.wchpu5muyto5n647@alap3.anarazel.de
https://postgr.es/m/20190327054923.t3epfuewxfqdt22e@alap3.anarazel.de
2019-04-05 00:47:19 +02:00
|
|
|
/* Send the data */
|
|
|
|
CopyOneRowTo(cstate, slot);
|
2011-02-16 03:19:11 +01:00
|
|
|
myState->processed++;
|
2016-06-06 20:52:58 +02:00
|
|
|
|
|
|
|
return true;
|
2006-08-31 01:34:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* copy_dest_shutdown --- executor end
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
copy_dest_shutdown(DestReceiver *self)
|
|
|
|
{
|
|
|
|
/* no-op */
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* copy_dest_destroy --- release DestReceiver object
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
copy_dest_destroy(DestReceiver *self)
|
|
|
|
{
|
|
|
|
pfree(self);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* CreateCopyDestReceiver -- create a suitable DestReceiver object
|
|
|
|
*/
|
|
|
|
DestReceiver *
|
|
|
|
CreateCopyDestReceiver(void)
|
|
|
|
{
|
2006-10-04 02:30:14 +02:00
|
|
|
DR_copy *self = (DR_copy *) palloc(sizeof(DR_copy));
|
2006-08-31 01:34:22 +02:00
|
|
|
|
|
|
|
self->pub.receiveSlot = copy_dest_receive;
|
|
|
|
self->pub.rStartup = copy_dest_startup;
|
|
|
|
self->pub.rShutdown = copy_dest_shutdown;
|
|
|
|
self->pub.rDestroy = copy_dest_destroy;
|
|
|
|
self->pub.mydest = DestCopyOut;
|
|
|
|
|
|
|
|
self->cstate = NULL; /* will be set later */
|
2011-02-18 06:29:40 +01:00
|
|
|
self->processed = 0;
|
2006-08-31 01:34:22 +02:00
|
|
|
|
|
|
|
return (DestReceiver *) self;
|
|
|
|
}
|