2022-03-21 20:02:25 +01:00
|
|
|
/* -------------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* pgstat_wal.c
|
|
|
|
* Implementation of WAL statistics.
|
|
|
|
*
|
|
|
|
* This file contains the implementation of WAL statistics. It is kept
|
|
|
|
* separate from pgstat.c to enforce the line between the statistics access /
|
|
|
|
* storage implementation and the details about individual types of
|
|
|
|
* statistics.
|
|
|
|
*
|
2024-01-04 02:49:05 +01:00
|
|
|
* Copyright (c) 2001-2024, PostgreSQL Global Development Group
|
2022-03-21 20:02:25 +01:00
|
|
|
*
|
|
|
|
* IDENTIFICATION
|
|
|
|
* src/backend/utils/activity/pgstat_wal.c
|
|
|
|
* -------------------------------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "postgres.h"
|
|
|
|
|
|
|
|
#include "utils/pgstat_internal.h"
|
|
|
|
#include "executor/instrument.h"
|
|
|
|
|
|
|
|
|
2023-03-30 23:23:14 +02:00
|
|
|
PgStat_PendingWalStats PendingWalStats = {0};
|
2022-03-21 20:02:25 +01:00
|
|
|
|
|
|
|
/*
|
2023-05-02 05:23:08 +02:00
|
|
|
* WAL usage counters saved from pgWalUsage at the previous call to
|
2022-04-06 23:08:57 +02:00
|
|
|
* pgstat_report_wal(). This is used to calculate how much WAL usage
|
|
|
|
* happens between pgstat_report_wal() calls, by subtracting
|
2022-03-21 20:02:25 +01:00
|
|
|
* the previous counters from the current ones.
|
|
|
|
*/
|
|
|
|
static WalUsage prevWalUsage;
|
|
|
|
|
|
|
|
|
2022-04-04 21:14:34 +02:00
|
|
|
/*
|
2022-04-07 06:29:46 +02:00
|
|
|
* Calculate how much WAL usage counters have increased and update
|
pgstat: Infrastructure for more detailed IO statistics
This commit adds the infrastructure for more detailed IO statistics. The calls
to actually count IOs, a system view to access the new statistics,
documentation and tests will be added in subsequent commits, to make review
easier.
While we already had some IO statistics, e.g. in pg_stat_bgwriter and
pg_stat_database, they did not provide sufficient detail to understand what
the main sources of IO are, or whether configuration changes could avoid
IO. E.g., pg_stat_bgwriter.buffers_backend does contain the number of buffers
written out by a backend, but as that includes extending relations (always
done by backends) and writes triggered by the use of buffer access strategies,
it cannot easily be used to tune background writer or checkpointer. Similarly,
pg_stat_database.blks_read cannot easily be used to tune shared_buffers /
compute a cache hit ratio, as the use of buffer access strategies will often
prevent a large fraction of the read blocks to end up in shared_buffers.
The new IO statistics count IO operations (evict, extend, fsync, read, reuse,
and write), and are aggregated for each combination of backend type (backend,
autovacuum worker, bgwriter, etc), target object of the IO (relations, temp
relations) and context of the IO (normal, vacuum, bulkread, bulkwrite).
What is tracked in this series of patches, is sufficient to perform the
aforementioned analyses. Further details, e.g. tracking the number of buffer
hits, would make that even easier, but was left out for now, to keep the scope
of the already large patchset manageable.
Bumps PGSTAT_FILE_FORMAT_ID.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Andres Freund <andres@anarazel.de>
Reviewed-by: Justin Pryzby <pryzby@telsasoft.com>
Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Discussion: https://postgr.es/m/20200124195226.lth52iydq2n2uilq@alap3.anarazel.de
2023-02-09 05:53:42 +01:00
|
|
|
* shared WAL and IO statistics.
|
2022-03-21 20:02:25 +01:00
|
|
|
*
|
2022-04-07 06:29:46 +02:00
|
|
|
* Must be called by processes that generate WAL, that do not call
|
|
|
|
* pgstat_report_stat(), like walwriter.
|
Fix behavior of "force" in pgstat_report_wal()
As implemented in 5891c7a8ed8f, setting "force" to true in
pgstat_report_wal() causes the routine to not wait for the pgstat
shmem lock if it cannot be acquired, in which case the WAL and I/O
statistics finish by not being flushed. The origin of the confusion
comes from pgstat_flush_wal() and pgstat_flush_io(), that use "nowait"
as sole argument. The I/O stats are new in v16.
This is the opposite behavior of what has been used in
pgstat_report_stat(), where "force" is the opposite of "nowait". In
this case, when "force" is true, the routine sets "nowait" to false,
which would cause the routine to wait for the pgstat shmem lock,
ensuring that the stats are always flushed. When "force" is false,
"nowait" is set to true, and the stats would only not be flushed if the
pgstat shmem lock can be acquired, returning immediately without
flushing the stats if the lock cannot be acquired.
This commit changes pgstat_report_wal() so as "force" has the same
behavior as in pgstat_report_stat(). There are currently three callers
of pgstat_report_wal():
- Two in the checkpointer where force=true during a shutdown and the
main checkpointer loop. Now the code behaves so as the stats are always
flushed.
- One in the main loop of the bgwriter, where force=false. Now the code
behaves so as the stats would not be flushed if the pgstat shmem lock
could not be acquired.
Before this commit, some stats on WAL and I/O could have been lost after
a shutdown, for example.
Reported-by: Ryoga Yoshida
Author: Ryoga Yoshida, Michael Paquier
Discussion: https://postgr.es/m/f87a4d7be70530606b864fd1df91718c@oss.nttdata.com
Backpatch-through: 15
2023-09-26 02:29:47 +02:00
|
|
|
*
|
|
|
|
* "force" set to true ensures that the statistics are flushed; note that
|
|
|
|
* this needs to acquire the pgstat shmem LWLock, waiting on it. When
|
|
|
|
* set to false, the statistics may not be flushed if the lock could not
|
|
|
|
* be acquired.
|
2022-03-21 20:02:25 +01:00
|
|
|
*/
|
|
|
|
void
|
2022-04-06 23:08:57 +02:00
|
|
|
pgstat_report_wal(bool force)
|
2022-03-21 20:02:25 +01:00
|
|
|
{
|
Fix behavior of "force" in pgstat_report_wal()
As implemented in 5891c7a8ed8f, setting "force" to true in
pgstat_report_wal() causes the routine to not wait for the pgstat
shmem lock if it cannot be acquired, in which case the WAL and I/O
statistics finish by not being flushed. The origin of the confusion
comes from pgstat_flush_wal() and pgstat_flush_io(), that use "nowait"
as sole argument. The I/O stats are new in v16.
This is the opposite behavior of what has been used in
pgstat_report_stat(), where "force" is the opposite of "nowait". In
this case, when "force" is true, the routine sets "nowait" to false,
which would cause the routine to wait for the pgstat shmem lock,
ensuring that the stats are always flushed. When "force" is false,
"nowait" is set to true, and the stats would only not be flushed if the
pgstat shmem lock can be acquired, returning immediately without
flushing the stats if the lock cannot be acquired.
This commit changes pgstat_report_wal() so as "force" has the same
behavior as in pgstat_report_stat(). There are currently three callers
of pgstat_report_wal():
- Two in the checkpointer where force=true during a shutdown and the
main checkpointer loop. Now the code behaves so as the stats are always
flushed.
- One in the main loop of the bgwriter, where force=false. Now the code
behaves so as the stats would not be flushed if the pgstat shmem lock
could not be acquired.
Before this commit, some stats on WAL and I/O could have been lost after
a shutdown, for example.
Reported-by: Ryoga Yoshida
Author: Ryoga Yoshida, Michael Paquier
Discussion: https://postgr.es/m/f87a4d7be70530606b864fd1df91718c@oss.nttdata.com
Backpatch-through: 15
2023-09-26 02:29:47 +02:00
|
|
|
bool nowait;
|
|
|
|
|
|
|
|
/* like in pgstat.c, don't wait for lock acquisition when !force */
|
|
|
|
nowait = !force;
|
|
|
|
|
|
|
|
/* flush wal stats */
|
|
|
|
pgstat_flush_wal(nowait);
|
pgstat: Infrastructure for more detailed IO statistics
This commit adds the infrastructure for more detailed IO statistics. The calls
to actually count IOs, a system view to access the new statistics,
documentation and tests will be added in subsequent commits, to make review
easier.
While we already had some IO statistics, e.g. in pg_stat_bgwriter and
pg_stat_database, they did not provide sufficient detail to understand what
the main sources of IO are, or whether configuration changes could avoid
IO. E.g., pg_stat_bgwriter.buffers_backend does contain the number of buffers
written out by a backend, but as that includes extending relations (always
done by backends) and writes triggered by the use of buffer access strategies,
it cannot easily be used to tune background writer or checkpointer. Similarly,
pg_stat_database.blks_read cannot easily be used to tune shared_buffers /
compute a cache hit ratio, as the use of buffer access strategies will often
prevent a large fraction of the read blocks to end up in shared_buffers.
The new IO statistics count IO operations (evict, extend, fsync, read, reuse,
and write), and are aggregated for each combination of backend type (backend,
autovacuum worker, bgwriter, etc), target object of the IO (relations, temp
relations) and context of the IO (normal, vacuum, bulkread, bulkwrite).
What is tracked in this series of patches, is sufficient to perform the
aforementioned analyses. Further details, e.g. tracking the number of buffer
hits, would make that even easier, but was left out for now, to keep the scope
of the already large patchset manageable.
Bumps PGSTAT_FILE_FORMAT_ID.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Andres Freund <andres@anarazel.de>
Reviewed-by: Justin Pryzby <pryzby@telsasoft.com>
Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Discussion: https://postgr.es/m/20200124195226.lth52iydq2n2uilq@alap3.anarazel.de
2023-02-09 05:53:42 +01:00
|
|
|
|
Fix behavior of "force" in pgstat_report_wal()
As implemented in 5891c7a8ed8f, setting "force" to true in
pgstat_report_wal() causes the routine to not wait for the pgstat
shmem lock if it cannot be acquired, in which case the WAL and I/O
statistics finish by not being flushed. The origin of the confusion
comes from pgstat_flush_wal() and pgstat_flush_io(), that use "nowait"
as sole argument. The I/O stats are new in v16.
This is the opposite behavior of what has been used in
pgstat_report_stat(), where "force" is the opposite of "nowait". In
this case, when "force" is true, the routine sets "nowait" to false,
which would cause the routine to wait for the pgstat shmem lock,
ensuring that the stats are always flushed. When "force" is false,
"nowait" is set to true, and the stats would only not be flushed if the
pgstat shmem lock can be acquired, returning immediately without
flushing the stats if the lock cannot be acquired.
This commit changes pgstat_report_wal() so as "force" has the same
behavior as in pgstat_report_stat(). There are currently three callers
of pgstat_report_wal():
- Two in the checkpointer where force=true during a shutdown and the
main checkpointer loop. Now the code behaves so as the stats are always
flushed.
- One in the main loop of the bgwriter, where force=false. Now the code
behaves so as the stats would not be flushed if the pgstat shmem lock
could not be acquired.
Before this commit, some stats on WAL and I/O could have been lost after
a shutdown, for example.
Reported-by: Ryoga Yoshida
Author: Ryoga Yoshida, Michael Paquier
Discussion: https://postgr.es/m/f87a4d7be70530606b864fd1df91718c@oss.nttdata.com
Backpatch-through: 15
2023-09-26 02:29:47 +02:00
|
|
|
/* flush IO stats */
|
|
|
|
pgstat_flush_io(nowait);
|
2022-04-07 06:29:46 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Support function for the SQL-callable pgstat* functions. Returns
|
|
|
|
* a pointer to the WAL statistics struct.
|
|
|
|
*/
|
|
|
|
PgStat_WalStats *
|
|
|
|
pgstat_fetch_stat_wal(void)
|
|
|
|
{
|
|
|
|
pgstat_snapshot_fixed(PGSTAT_KIND_WAL);
|
|
|
|
|
|
|
|
return &pgStatLocal.snapshot.wal;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Calculate how much WAL usage counters have increased by subtracting the
|
|
|
|
* previous counters from the current ones.
|
|
|
|
*
|
|
|
|
* If nowait is true, this function returns true if the lock could not be
|
|
|
|
* acquired. Otherwise return false.
|
|
|
|
*/
|
|
|
|
bool
|
|
|
|
pgstat_flush_wal(bool nowait)
|
|
|
|
{
|
|
|
|
PgStatShared_Wal *stats_shmem = &pgStatLocal.shmem->wal;
|
2023-03-30 23:23:14 +02:00
|
|
|
WalUsage wal_usage_diff = {0};
|
2022-04-07 06:29:46 +02:00
|
|
|
|
|
|
|
Assert(IsUnderPostmaster || !IsPostmasterEnvironment);
|
|
|
|
Assert(pgStatLocal.shmem != NULL &&
|
|
|
|
!pgStatLocal.shmem->is_shutdown);
|
2022-03-21 20:02:25 +01:00
|
|
|
|
|
|
|
/*
|
2022-04-07 06:29:46 +02:00
|
|
|
* This function can be called even if nothing at all has happened. Avoid
|
|
|
|
* taking lock for nothing in that case.
|
2022-03-21 20:02:25 +01:00
|
|
|
*/
|
2022-04-07 06:29:46 +02:00
|
|
|
if (!pgstat_have_pending_wal())
|
|
|
|
return false;
|
2022-03-21 20:02:25 +01:00
|
|
|
|
|
|
|
/*
|
2022-04-07 06:29:46 +02:00
|
|
|
* We don't update the WAL usage portion of the local WalStats elsewhere.
|
|
|
|
* Calculate how much WAL usage counters were increased by subtracting the
|
|
|
|
* previous counters from the current ones.
|
2022-03-21 20:02:25 +01:00
|
|
|
*/
|
2023-03-30 23:23:14 +02:00
|
|
|
WalUsageAccumDiff(&wal_usage_diff, &pgWalUsage, &prevWalUsage);
|
2022-04-07 06:29:46 +02:00
|
|
|
|
|
|
|
if (!nowait)
|
|
|
|
LWLockAcquire(&stats_shmem->lock, LW_EXCLUSIVE);
|
|
|
|
else if (!LWLockConditionalAcquire(&stats_shmem->lock, LW_EXCLUSIVE))
|
|
|
|
return true;
|
|
|
|
|
2023-03-30 23:23:14 +02:00
|
|
|
#define WALSTAT_ACC(fld, var_to_add) \
|
|
|
|
(stats_shmem->stats.fld += var_to_add.fld)
|
|
|
|
#define WALSTAT_ACC_INSTR_TIME(fld) \
|
|
|
|
(stats_shmem->stats.fld += INSTR_TIME_GET_MICROSEC(PendingWalStats.fld))
|
|
|
|
WALSTAT_ACC(wal_records, wal_usage_diff);
|
|
|
|
WALSTAT_ACC(wal_fpi, wal_usage_diff);
|
|
|
|
WALSTAT_ACC(wal_bytes, wal_usage_diff);
|
|
|
|
WALSTAT_ACC(wal_buffers_full, PendingWalStats);
|
|
|
|
WALSTAT_ACC(wal_write, PendingWalStats);
|
|
|
|
WALSTAT_ACC(wal_sync, PendingWalStats);
|
|
|
|
WALSTAT_ACC_INSTR_TIME(wal_write_time);
|
|
|
|
WALSTAT_ACC_INSTR_TIME(wal_sync_time);
|
|
|
|
#undef WALSTAT_ACC_INSTR_TIME
|
2022-04-07 06:29:46 +02:00
|
|
|
#undef WALSTAT_ACC
|
|
|
|
|
|
|
|
LWLockRelease(&stats_shmem->lock);
|
2022-03-21 20:02:25 +01:00
|
|
|
|
|
|
|
/*
|
2022-04-07 06:29:46 +02:00
|
|
|
* Save the current counters for the subsequent calculation of WAL usage.
|
2022-03-21 20:02:25 +01:00
|
|
|
*/
|
2022-04-07 06:29:46 +02:00
|
|
|
prevWalUsage = pgWalUsage;
|
2022-03-21 20:02:25 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Clear out the statistics buffer, so it can be re-used.
|
|
|
|
*/
|
2022-04-07 06:29:46 +02:00
|
|
|
MemSet(&PendingWalStats, 0, sizeof(PendingWalStats));
|
|
|
|
|
|
|
|
return false;
|
2022-03-21 20:02:25 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2022-04-07 06:29:46 +02:00
|
|
|
pgstat_init_wal(void)
|
2022-03-21 20:02:25 +01:00
|
|
|
{
|
|
|
|
/*
|
2022-04-07 06:29:46 +02:00
|
|
|
* Initialize prevWalUsage with pgWalUsage so that pgstat_flush_wal() can
|
2022-03-21 20:02:25 +01:00
|
|
|
* calculate how much pgWalUsage counters are increased by subtracting
|
|
|
|
* prevWalUsage from pgWalUsage.
|
|
|
|
*/
|
|
|
|
prevWalUsage = pgWalUsage;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* To determine whether any WAL activity has occurred since last time, not
|
|
|
|
* only the number of generated WAL records but also the numbers of WAL
|
|
|
|
* writes and syncs need to be checked. Because even transaction that
|
|
|
|
* generates no WAL records can write or sync WAL data when flushing the
|
|
|
|
* data pages.
|
|
|
|
*/
|
|
|
|
bool
|
2022-04-07 06:29:46 +02:00
|
|
|
pgstat_have_pending_wal(void)
|
2022-03-21 20:02:25 +01:00
|
|
|
{
|
|
|
|
return pgWalUsage.wal_records != prevWalUsage.wal_records ||
|
2022-04-07 06:29:46 +02:00
|
|
|
PendingWalStats.wal_write != 0 ||
|
|
|
|
PendingWalStats.wal_sync != 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
pgstat_wal_reset_all_cb(TimestampTz ts)
|
|
|
|
{
|
|
|
|
PgStatShared_Wal *stats_shmem = &pgStatLocal.shmem->wal;
|
|
|
|
|
|
|
|
LWLockAcquire(&stats_shmem->lock, LW_EXCLUSIVE);
|
|
|
|
memset(&stats_shmem->stats, 0, sizeof(stats_shmem->stats));
|
|
|
|
stats_shmem->stats.stat_reset_timestamp = ts;
|
|
|
|
LWLockRelease(&stats_shmem->lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
pgstat_wal_snapshot_cb(void)
|
|
|
|
{
|
|
|
|
PgStatShared_Wal *stats_shmem = &pgStatLocal.shmem->wal;
|
|
|
|
|
|
|
|
LWLockAcquire(&stats_shmem->lock, LW_SHARED);
|
|
|
|
memcpy(&pgStatLocal.snapshot.wal, &stats_shmem->stats,
|
|
|
|
sizeof(pgStatLocal.snapshot.wal));
|
|
|
|
LWLockRelease(&stats_shmem->lock);
|
2022-03-21 20:02:25 +01:00
|
|
|
}
|