2022-03-21 20:02:25 +01:00
|
|
|
/* -------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* pgstat_wal.c
|
|
|
|
* Implementation of WAL statistics.
|
|
|
|
*
|
|
|
|
* This file contains the implementation of WAL statistics. It is kept
|
|
|
|
* separate from pgstat.c to enforce the line between the statistics access /
|
|
|
|
* storage implementation and the details about individual types of
|
|
|
|
* statistics.
|
|
|
|
*
|
2023-01-02 21:00:37 +01:00
|
|
|
* Copyright (c) 2001-2023, PostgreSQL Global Development Group
|
2022-03-21 20:02:25 +01:00
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
|
|
|
* src/backend/utils/activity/pgstat_wal.c
|
|
|
|
* -------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "postgres.h"
|
|
|
|
|
|
|
|
#include "utils/pgstat_internal.h"
|
|
|
|
#include "executor/instrument.h"
|
|
|
|
|
|
|
|
|
2023-03-30 23:23:14 +02:00
|
|
|
PgStat_PendingWalStats PendingWalStats = {0};
|
2022-03-21 20:02:25 +01:00
|
|
|
|
|
|
|
/*
|
2023-05-02 05:23:08 +02:00
|
|
|
* WAL usage counters saved from pgWalUsage at the previous call to
|
2022-04-06 23:08:57 +02:00
|
|
|
* pgstat_report_wal(). This is used to calculate how much WAL usage
|
|
|
|
* happens between pgstat_report_wal() calls, by subtracting
|
2022-03-21 20:02:25 +01:00
|
|
|
* the previous counters from the current ones.
|
|
|
|
*/
|
|
|
|
static WalUsage prevWalUsage;
|
|
|
|
|
|
|
|
|
2022-04-04 21:14:34 +02:00
|
|
|
/*
|
2022-04-07 06:29:46 +02:00
|
|
|
* Calculate how much WAL usage counters have increased and update
|
pgstat: Infrastructure for more detailed IO statistics
This commit adds the infrastructure for more detailed IO statistics. The calls
to actually count IOs, a system view to access the new statistics,
documentation and tests will be added in subsequent commits, to make review
easier.
While we already had some IO statistics, e.g. in pg_stat_bgwriter and
pg_stat_database, they did not provide sufficient detail to understand what
the main sources of IO are, or whether configuration changes could avoid
IO. E.g., pg_stat_bgwriter.buffers_backend does contain the number of buffers
written out by a backend, but as that includes extending relations (always
done by backends) and writes triggered by the use of buffer access strategies,
it cannot easily be used to tune background writer or checkpointer. Similarly,
pg_stat_database.blks_read cannot easily be used to tune shared_buffers /
compute a cache hit ratio, as the use of buffer access strategies will often
prevent a large fraction of the read blocks to end up in shared_buffers.
The new IO statistics count IO operations (evict, extend, fsync, read, reuse,
and write), and are aggregated for each combination of backend type (backend,
autovacuum worker, bgwriter, etc), target object of the IO (relations, temp
relations) and context of the IO (normal, vacuum, bulkread, bulkwrite).
What is tracked in this series of patches, is sufficient to perform the
aforementioned analyses. Further details, e.g. tracking the number of buffer
hits, would make that even easier, but was left out for now, to keep the scope
of the already large patchset manageable.
Bumps PGSTAT_FILE_FORMAT_ID.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Andres Freund <andres@anarazel.de>
Reviewed-by: Justin Pryzby <pryzby@telsasoft.com>
Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Discussion: https://postgr.es/m/20200124195226.lth52iydq2n2uilq@alap3.anarazel.de
2023-02-09 05:53:42 +01:00
|
|
|
* shared WAL and IO statistics.
|
2022-03-21 20:02:25 +01:00
|
|
|
*
|
2022-04-07 06:29:46 +02:00
|
|
|
* Must be called by processes that generate WAL, that do not call
|
|
|
|
* pgstat_report_stat(), like walwriter.
|
2022-03-21 20:02:25 +01:00
|
|
|
*/
|
|
|
|
void
|
2022-04-06 23:08:57 +02:00
|
|
|
pgstat_report_wal(bool force)
|
2022-03-21 20:02:25 +01:00
|
|
|
{
|
2022-04-07 06:29:46 +02:00
|
|
|
pgstat_flush_wal(force);
|
pgstat: Infrastructure for more detailed IO statistics
This commit adds the infrastructure for more detailed IO statistics. The calls
to actually count IOs, a system view to access the new statistics,
documentation and tests will be added in subsequent commits, to make review
easier.
While we already had some IO statistics, e.g. in pg_stat_bgwriter and
pg_stat_database, they did not provide sufficient detail to understand what
the main sources of IO are, or whether configuration changes could avoid
IO. E.g., pg_stat_bgwriter.buffers_backend does contain the number of buffers
written out by a backend, but as that includes extending relations (always
done by backends) and writes triggered by the use of buffer access strategies,
it cannot easily be used to tune background writer or checkpointer. Similarly,
pg_stat_database.blks_read cannot easily be used to tune shared_buffers /
compute a cache hit ratio, as the use of buffer access strategies will often
prevent a large fraction of the read blocks to end up in shared_buffers.
The new IO statistics count IO operations (evict, extend, fsync, read, reuse,
and write), and are aggregated for each combination of backend type (backend,
autovacuum worker, bgwriter, etc), target object of the IO (relations, temp
relations) and context of the IO (normal, vacuum, bulkread, bulkwrite).
What is tracked in this series of patches, is sufficient to perform the
aforementioned analyses. Further details, e.g. tracking the number of buffer
hits, would make that even easier, but was left out for now, to keep the scope
of the already large patchset manageable.
Bumps PGSTAT_FILE_FORMAT_ID.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Andres Freund <andres@anarazel.de>
Reviewed-by: Justin Pryzby <pryzby@telsasoft.com>
Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Discussion: https://postgr.es/m/20200124195226.lth52iydq2n2uilq@alap3.anarazel.de
2023-02-09 05:53:42 +01:00
|
|
|
|
|
|
|
pgstat_flush_io(force);
|
2022-04-07 06:29:46 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Support function for the SQL-callable pgstat* functions. Returns
|
|
|
|
* a pointer to the WAL statistics struct.
|
|
|
|
*/
|
|
|
|
PgStat_WalStats *
|
|
|
|
pgstat_fetch_stat_wal(void)
|
|
|
|
{
|
|
|
|
pgstat_snapshot_fixed(PGSTAT_KIND_WAL);
|
|
|
|
|
|
|
|
return &pgStatLocal.snapshot.wal;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Calculate how much WAL usage counters have increased by subtracting the
|
|
|
|
* previous counters from the current ones.
|
|
|
|
*
|
|
|
|
* If nowait is true, this function returns true if the lock could not be
|
|
|
|
* acquired. Otherwise return false.
|
|
|
|
*/
|
|
|
|
bool
|
|
|
|
pgstat_flush_wal(bool nowait)
|
|
|
|
{
|
|
|
|
PgStatShared_Wal *stats_shmem = &pgStatLocal.shmem->wal;
|
2023-03-30 23:23:14 +02:00
|
|
|
WalUsage wal_usage_diff = {0};
|
2022-04-07 06:29:46 +02:00
|
|
|
|
|
|
|
Assert(IsUnderPostmaster || !IsPostmasterEnvironment);
|
|
|
|
Assert(pgStatLocal.shmem != NULL &&
|
|
|
|
!pgStatLocal.shmem->is_shutdown);
|
2022-03-21 20:02:25 +01:00
|
|
|
|
|
|
|
/*
|
2022-04-07 06:29:46 +02:00
|
|
|
* This function can be called even if nothing at all has happened. Avoid
|
|
|
|
* taking lock for nothing in that case.
|
2022-03-21 20:02:25 +01:00
|
|
|
*/
|
2022-04-07 06:29:46 +02:00
|
|
|
if (!pgstat_have_pending_wal())
|
|
|
|
return false;
|
2022-03-21 20:02:25 +01:00
|
|
|
|
|
|
|
/*
|
2022-04-07 06:29:46 +02:00
|
|
|
* We don't update the WAL usage portion of the local WalStats elsewhere.
|
|
|
|
* Calculate how much WAL usage counters were increased by subtracting the
|
|
|
|
* previous counters from the current ones.
|
2022-03-21 20:02:25 +01:00
|
|
|
*/
|
2023-03-30 23:23:14 +02:00
|
|
|
WalUsageAccumDiff(&wal_usage_diff, &pgWalUsage, &prevWalUsage);
|
2022-04-07 06:29:46 +02:00
|
|
|
|
|
|
|
if (!nowait)
|
|
|
|
LWLockAcquire(&stats_shmem->lock, LW_EXCLUSIVE);
|
|
|
|
else if (!LWLockConditionalAcquire(&stats_shmem->lock, LW_EXCLUSIVE))
|
|
|
|
return true;
|
|
|
|
|
2023-03-30 23:23:14 +02:00
|
|
|
#define WALSTAT_ACC(fld, var_to_add) \
|
|
|
|
(stats_shmem->stats.fld += var_to_add.fld)
|
|
|
|
#define WALSTAT_ACC_INSTR_TIME(fld) \
|
|
|
|
(stats_shmem->stats.fld += INSTR_TIME_GET_MICROSEC(PendingWalStats.fld))
|
|
|
|
WALSTAT_ACC(wal_records, wal_usage_diff);
|
|
|
|
WALSTAT_ACC(wal_fpi, wal_usage_diff);
|
|
|
|
WALSTAT_ACC(wal_bytes, wal_usage_diff);
|
|
|
|
WALSTAT_ACC(wal_buffers_full, PendingWalStats);
|
|
|
|
WALSTAT_ACC(wal_write, PendingWalStats);
|
|
|
|
WALSTAT_ACC(wal_sync, PendingWalStats);
|
|
|
|
WALSTAT_ACC_INSTR_TIME(wal_write_time);
|
|
|
|
WALSTAT_ACC_INSTR_TIME(wal_sync_time);
|
|
|
|
#undef WALSTAT_ACC_INSTR_TIME
|
2022-04-07 06:29:46 +02:00
|
|
|
#undef WALSTAT_ACC
|
|
|
|
|
|
|
|
LWLockRelease(&stats_shmem->lock);
|
2022-03-21 20:02:25 +01:00
|
|
|
|
|
|
|
/*
|
2022-04-07 06:29:46 +02:00
|
|
|
* Save the current counters for the subsequent calculation of WAL usage.
|
2022-03-21 20:02:25 +01:00
|
|
|
*/
|
2022-04-07 06:29:46 +02:00
|
|
|
prevWalUsage = pgWalUsage;
|
2022-03-21 20:02:25 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Clear out the statistics buffer, so it can be re-used.
|
|
|
|
*/
|
2022-04-07 06:29:46 +02:00
|
|
|
MemSet(&PendingWalStats, 0, sizeof(PendingWalStats));
|
|
|
|
|
|
|
|
return false;
|
2022-03-21 20:02:25 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2022-04-07 06:29:46 +02:00
|
|
|
pgstat_init_wal(void)
|
2022-03-21 20:02:25 +01:00
|
|
|
{
|
|
|
|
/*
|
2022-04-07 06:29:46 +02:00
|
|
|
* Initialize prevWalUsage with pgWalUsage so that pgstat_flush_wal() can
|
2022-03-21 20:02:25 +01:00
|
|
|
* calculate how much pgWalUsage counters are increased by subtracting
|
|
|
|
* prevWalUsage from pgWalUsage.
|
|
|
|
*/
|
|
|
|
prevWalUsage = pgWalUsage;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* To determine whether any WAL activity has occurred since last time, not
|
|
|
|
* only the number of generated WAL records but also the numbers of WAL
|
|
|
|
* writes and syncs need to be checked. Because even transaction that
|
|
|
|
* generates no WAL records can write or sync WAL data when flushing the
|
|
|
|
* data pages.
|
|
|
|
*/
|
|
|
|
bool
|
2022-04-07 06:29:46 +02:00
|
|
|
pgstat_have_pending_wal(void)
|
2022-03-21 20:02:25 +01:00
|
|
|
{
|
|
|
|
return pgWalUsage.wal_records != prevWalUsage.wal_records ||
|
2022-04-07 06:29:46 +02:00
|
|
|
PendingWalStats.wal_write != 0 ||
|
|
|
|
PendingWalStats.wal_sync != 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
pgstat_wal_reset_all_cb(TimestampTz ts)
|
|
|
|
{
|
|
|
|
PgStatShared_Wal *stats_shmem = &pgStatLocal.shmem->wal;
|
|
|
|
|
|
|
|
LWLockAcquire(&stats_shmem->lock, LW_EXCLUSIVE);
|
|
|
|
memset(&stats_shmem->stats, 0, sizeof(stats_shmem->stats));
|
|
|
|
stats_shmem->stats.stat_reset_timestamp = ts;
|
|
|
|
LWLockRelease(&stats_shmem->lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
pgstat_wal_snapshot_cb(void)
|
|
|
|
{
|
|
|
|
PgStatShared_Wal *stats_shmem = &pgStatLocal.shmem->wal;
|
|
|
|
|
|
|
|
LWLockAcquire(&stats_shmem->lock, LW_SHARED);
|
|
|
|
memcpy(&pgStatLocal.snapshot.wal, &stats_shmem->stats,
|
|
|
|
sizeof(pgStatLocal.snapshot.wal));
|
|
|
|
LWLockRelease(&stats_shmem->lock);
|
2022-03-21 20:02:25 +01:00
|
|
|
}
|