postgresql/contrib/vacuumlo/vacuumlo.c
Tom Lane 67a472d71c Remove arbitrary restrictions on password length.
This patch started out with the goal of harmonizing various arbitrary
limits on password length, but after awhile a better idea emerged:
let's just get rid of those fixed limits.

recv_password_packet() has an arbitrary limit on the packet size,
which we don't really need, so just drop it.  (Note that this doesn't
really affect anything for MD5 or SCRAM password verification, since
those will hash the user's password to something shorter anyway.
It does matter for auth methods that require a cleartext password.)

Likewise remove the arbitrary error condition in pg_saslprep().

The remaining limits are mostly in client-side code that prompts
for passwords.  To improve those, refactor simple_prompt() so that
it allocates its own result buffer that can be made as big as
necessary.  Actually, it proves best to make a separate routine
pg_get_line() that has essentially the semantics of fgets(), except
that it allocates a suitable result buffer and hence will never
return a truncated line.  (pg_get_line has a lot of potential
applications to replace randomly-sized fgets buffers elsewhere,
but I'll leave that for another patch.)

I built pg_get_line() atop stringinfo.c, which requires moving
that code to src/common/; but that seems fine since it was a poor
fit for src/port/ anyway.

This patch is mostly mine, but it owes a good deal to Nathan Bossart
who pressed for a solution to the password length problem and
created a predecessor patch.  Also thanks to Peter Eisentraut and
Stephen Frost for ideas and discussion.

Discussion: https://postgr.es/m/09512C4F-8CB9-4021-B455-EF4C4F0D55A0@amazon.com
2020-09-03 20:09:18 -04:00

557 lines
14 KiB
C

/*-------------------------------------------------------------------------
*
* vacuumlo.c
* This removes orphaned large objects from a database.
*
* Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
*
* IDENTIFICATION
* contrib/vacuumlo/vacuumlo.c
*
*-------------------------------------------------------------------------
*/
#include "postgres_fe.h"
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#ifdef HAVE_TERMIOS_H
#include <termios.h>
#endif
#include "catalog/pg_class_d.h"
#include "common/connect.h"
#include "common/logging.h"
#include "common/string.h"
#include "getopt_long.h"
#include "libpq-fe.h"
#include "pg_getopt.h"
#define BUFSIZE 1024
enum trivalue
{
TRI_DEFAULT,
TRI_NO,
TRI_YES
};
struct _param
{
char *pg_user;
enum trivalue pg_prompt;
char *pg_port;
char *pg_host;
const char *progname;
int verbose;
int dry_run;
long transaction_limit;
};
static int vacuumlo(const char *database, const struct _param *param);
static void usage(const char *progname);
/*
* This vacuums LOs of one database. It returns 0 on success, -1 on failure.
*/
static int
vacuumlo(const char *database, const struct _param *param)
{
PGconn *conn;
PGresult *res,
*res2;
char buf[BUFSIZE];
long matched;
long deleted;
int i;
bool new_pass;
bool success = true;
static char *password = NULL;
/* Note: password can be carried over from a previous call */
if (param->pg_prompt == TRI_YES && !password)
password = simple_prompt("Password: ", false);
/*
* Start the connection. Loop until we have a password if requested by
* backend.
*/
do
{
#define PARAMS_ARRAY_SIZE 7
const char *keywords[PARAMS_ARRAY_SIZE];
const char *values[PARAMS_ARRAY_SIZE];
keywords[0] = "host";
values[0] = param->pg_host;
keywords[1] = "port";
values[1] = param->pg_port;
keywords[2] = "user";
values[2] = param->pg_user;
keywords[3] = "password";
values[3] = password;
keywords[4] = "dbname";
values[4] = database;
keywords[5] = "fallback_application_name";
values[5] = param->progname;
keywords[6] = NULL;
values[6] = NULL;
new_pass = false;
conn = PQconnectdbParams(keywords, values, true);
if (!conn)
{
pg_log_error("connection to database \"%s\" failed", database);
return -1;
}
if (PQstatus(conn) == CONNECTION_BAD &&
PQconnectionNeedsPassword(conn) &&
!password &&
param->pg_prompt != TRI_NO)
{
PQfinish(conn);
password = simple_prompt("Password: ", false);
new_pass = true;
}
} while (new_pass);
/* check to see that the backend connection was successfully made */
if (PQstatus(conn) == CONNECTION_BAD)
{
pg_log_error("connection to database \"%s\" failed: %s",
database, PQerrorMessage(conn));
PQfinish(conn);
return -1;
}
if (param->verbose)
{
fprintf(stdout, "Connected to database \"%s\"\n", database);
if (param->dry_run)
fprintf(stdout, "Test run: no large objects will be removed!\n");
}
res = PQexec(conn, ALWAYS_SECURE_SEARCH_PATH_SQL);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
pg_log_error("failed to set search_path: %s", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return -1;
}
PQclear(res);
/*
* First we create and populate the LO temp table
*/
buf[0] = '\0';
strcat(buf, "CREATE TEMP TABLE vacuum_l AS ");
if (PQserverVersion(conn) >= 90000)
strcat(buf, "SELECT oid AS lo FROM pg_largeobject_metadata");
else
strcat(buf, "SELECT DISTINCT loid AS lo FROM pg_largeobject");
res = PQexec(conn, buf);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
pg_log_error("failed to create temp table: %s", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return -1;
}
PQclear(res);
/*
* Analyze the temp table so that planner will generate decent plans for
* the DELETEs below.
*/
buf[0] = '\0';
strcat(buf, "ANALYZE vacuum_l");
res = PQexec(conn, buf);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
pg_log_error("failed to vacuum temp table: %s", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return -1;
}
PQclear(res);
/*
* Now find any candidate tables that have columns of type oid.
*
* NOTE: we ignore system tables and temp tables by the expedient of
* rejecting tables in schemas named 'pg_*'. In particular, the temp
* table formed above is ignored, and pg_largeobject will be too. If
* either of these were scanned, obviously we'd end up with nothing to
* delete...
*/
buf[0] = '\0';
strcat(buf, "SELECT s.nspname, c.relname, a.attname ");
strcat(buf, "FROM pg_class c, pg_attribute a, pg_namespace s, pg_type t ");
strcat(buf, "WHERE a.attnum > 0 AND NOT a.attisdropped ");
strcat(buf, " AND a.attrelid = c.oid ");
strcat(buf, " AND a.atttypid = t.oid ");
strcat(buf, " AND c.relnamespace = s.oid ");
strcat(buf, " AND t.typname in ('oid', 'lo') ");
strcat(buf, " AND c.relkind in (" CppAsString2(RELKIND_RELATION) ", " CppAsString2(RELKIND_MATVIEW) ")");
strcat(buf, " AND s.nspname !~ '^pg_'");
res = PQexec(conn, buf);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
pg_log_error("failed to find OID columns: %s", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return -1;
}
for (i = 0; i < PQntuples(res); i++)
{
char *schema,
*table,
*field;
schema = PQgetvalue(res, i, 0);
table = PQgetvalue(res, i, 1);
field = PQgetvalue(res, i, 2);
if (param->verbose)
fprintf(stdout, "Checking %s in %s.%s\n", field, schema, table);
schema = PQescapeIdentifier(conn, schema, strlen(schema));
table = PQescapeIdentifier(conn, table, strlen(table));
field = PQescapeIdentifier(conn, field, strlen(field));
if (!schema || !table || !field)
{
pg_log_error("%s", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
if (schema != NULL)
PQfreemem(schema);
if (table != NULL)
PQfreemem(table);
if (field != NULL)
PQfreemem(field);
return -1;
}
snprintf(buf, BUFSIZE,
"DELETE FROM vacuum_l "
"WHERE lo IN (SELECT %s FROM %s.%s)",
field, schema, table);
res2 = PQexec(conn, buf);
if (PQresultStatus(res2) != PGRES_COMMAND_OK)
{
pg_log_error("failed to check %s in table %s.%s: %s",
field, schema, table, PQerrorMessage(conn));
PQclear(res2);
PQclear(res);
PQfinish(conn);
PQfreemem(schema);
PQfreemem(table);
PQfreemem(field);
return -1;
}
PQclear(res2);
PQfreemem(schema);
PQfreemem(table);
PQfreemem(field);
}
PQclear(res);
/*
* Now, those entries remaining in vacuum_l are orphans. Delete 'em.
*
* We don't want to run each delete as an individual transaction, because
* the commit overhead would be high. However, since 9.0 the backend will
* acquire a lock per deleted LO, so deleting too many LOs per transaction
* risks running out of room in the shared-memory lock table. Accordingly,
* we delete up to transaction_limit LOs per transaction.
*/
res = PQexec(conn, "begin");
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
pg_log_error("failed to start transaction: %s", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return -1;
}
PQclear(res);
buf[0] = '\0';
strcat(buf,
"DECLARE myportal CURSOR WITH HOLD FOR SELECT lo FROM vacuum_l");
res = PQexec(conn, buf);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
pg_log_error("DECLARE CURSOR failed: %s", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return -1;
}
PQclear(res);
snprintf(buf, BUFSIZE, "FETCH FORWARD %ld IN myportal",
param->transaction_limit > 0 ? param->transaction_limit : 1000L);
deleted = 0;
do
{
res = PQexec(conn, buf);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
pg_log_error("FETCH FORWARD failed: %s", PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return -1;
}
matched = PQntuples(res);
if (matched <= 0)
{
/* at end of resultset */
PQclear(res);
break;
}
for (i = 0; i < matched; i++)
{
Oid lo = atooid(PQgetvalue(res, i, 0));
if (param->verbose)
{
fprintf(stdout, "\rRemoving lo %6u ", lo);
fflush(stdout);
}
if (param->dry_run == 0)
{
if (lo_unlink(conn, lo) < 0)
{
pg_log_error("failed to remove lo %u: %s", lo,
PQerrorMessage(conn));
if (PQtransactionStatus(conn) == PQTRANS_INERROR)
{
success = false;
break; /* out of inner for-loop */
}
}
else
deleted++;
}
else
deleted++;
if (param->transaction_limit > 0 &&
(deleted % param->transaction_limit) == 0)
{
res2 = PQexec(conn, "commit");
if (PQresultStatus(res2) != PGRES_COMMAND_OK)
{
pg_log_error("failed to commit transaction: %s",
PQerrorMessage(conn));
PQclear(res2);
PQclear(res);
PQfinish(conn);
return -1;
}
PQclear(res2);
res2 = PQexec(conn, "begin");
if (PQresultStatus(res2) != PGRES_COMMAND_OK)
{
pg_log_error("failed to start transaction: %s",
PQerrorMessage(conn));
PQclear(res2);
PQclear(res);
PQfinish(conn);
return -1;
}
PQclear(res2);
}
}
PQclear(res);
} while (success);
/*
* That's all folks!
*/
res = PQexec(conn, "commit");
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
pg_log_error("failed to commit transaction: %s",
PQerrorMessage(conn));
PQclear(res);
PQfinish(conn);
return -1;
}
PQclear(res);
PQfinish(conn);
if (param->verbose)
{
if (param->dry_run)
fprintf(stdout, "\rWould remove %ld large objects from database \"%s\".\n",
deleted, database);
else if (success)
fprintf(stdout,
"\rSuccessfully removed %ld large objects from database \"%s\".\n",
deleted, database);
else
fprintf(stdout, "\rRemoval from database \"%s\" failed at object %ld of %ld.\n",
database, deleted, matched);
}
return ((param->dry_run || success) ? 0 : -1);
}
static void
usage(const char *progname)
{
printf("%s removes unreferenced large objects from databases.\n\n", progname);
printf("Usage:\n %s [OPTION]... DBNAME...\n\n", progname);
printf("Options:\n");
printf(" -l, --limit=LIMIT commit after removing each LIMIT large objects\n");
printf(" -n, --dry-run don't remove large objects, just show what would be done\n");
printf(" -v, --verbose write a lot of progress messages\n");
printf(" -V, --version output version information, then exit\n");
printf(" -?, --help show this help, then exit\n");
printf("\nConnection options:\n");
printf(" -h, --host=HOSTNAME database server host or socket directory\n");
printf(" -p, --port=PORT database server port\n");
printf(" -U, --username=USERNAME user name to connect as\n");
printf(" -w, --no-password never prompt for password\n");
printf(" -W, --password force password prompt\n");
printf("\n");
printf("Report bugs to <%s>.\n", PACKAGE_BUGREPORT);
printf("%s home page: <%s>\n", PACKAGE_NAME, PACKAGE_URL);
}
int
main(int argc, char **argv)
{
static struct option long_options[] = {
{"host", required_argument, NULL, 'h'},
{"limit", required_argument, NULL, 'l'},
{"dry-run", no_argument, NULL, 'n'},
{"port", required_argument, NULL, 'p'},
{"username", required_argument, NULL, 'U'},
{"verbose", no_argument, NULL, 'v'},
{"version", no_argument, NULL, 'V'},
{"no-password", no_argument, NULL, 'w'},
{"password", no_argument, NULL, 'W'},
{"help", no_argument, NULL, '?'},
{NULL, 0, NULL, 0}
};
int rc = 0;
struct _param param;
int c;
int port;
const char *progname;
int optindex;
pg_logging_init(argv[0]);
progname = get_progname(argv[0]);
/* Set default parameter values */
param.pg_user = NULL;
param.pg_prompt = TRI_DEFAULT;
param.pg_host = NULL;
param.pg_port = NULL;
param.progname = progname;
param.verbose = 0;
param.dry_run = 0;
param.transaction_limit = 1000;
/* Process command-line arguments */
if (argc > 1)
{
if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
{
usage(progname);
exit(0);
}
if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
{
puts("vacuumlo (PostgreSQL) " PG_VERSION);
exit(0);
}
}
while ((c = getopt_long(argc, argv, "h:l:np:U:vwW", long_options, &optindex)) != -1)
{
switch (c)
{
case '?':
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
case 'h':
param.pg_host = pg_strdup(optarg);
break;
case 'l':
param.transaction_limit = strtol(optarg, NULL, 10);
if (param.transaction_limit < 0)
{
pg_log_error("transaction limit must not be negative (0 disables)");
exit(1);
}
break;
case 'n':
param.dry_run = 1;
param.verbose = 1;
break;
case 'p':
port = strtol(optarg, NULL, 10);
if ((port < 1) || (port > 65535))
{
pg_log_error("invalid port number: %s", optarg);
exit(1);
}
param.pg_port = pg_strdup(optarg);
break;
case 'U':
param.pg_user = pg_strdup(optarg);
break;
case 'v':
param.verbose = 1;
break;
case 'w':
param.pg_prompt = TRI_NO;
break;
case 'W':
param.pg_prompt = TRI_YES;
break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
}
}
/* No database given? Show usage */
if (optind >= argc)
{
pg_log_error("missing required argument: database name");
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
}
for (c = optind; c < argc; c++)
{
/* Work on selected database */
rc += (vacuumlo(argv[c], &param) != 0);
}
return rc;
}