rdelaage
/
postgresql
mirror of https://git.postgresql.org/git/postgresql.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add writeback to pg_stat_io 

28e626bde0 added the concept of IOOps but neglected to include writeback
operations. ac8d53dae5 added time spent doing these I/O operations. Without
counting writeback, checkpointer write time in the log often differed
substantially from that in pg_stat_io. To fix this, add IOOp IOOP_WRITEBACK
and track writeback in pg_stat_io.

Bumps catversion.

Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Reported-by: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/20230419172326.dhgyo4wrrhulovt6%40awork3.anarazel.de
This commit is contained in:
Andres Freund 2023-05-17 11:18:35 -07:00
parent 52676dc2e0
commit 093e5c57d5
12 changed files with 82 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
doc/src/sgml/monitoring.sgml
View File

@ -3867,6 +3867,32 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
</entry>
</row>
<row>
<entry role="catalog_table_entry">
<para role="column_definition">
<structfield>writebacks</structfield> <type>bigint</type>
</para>
<para>
Number of units of size <varname>op_bytes</varname> which the process
requested the kernel write out to permanent storage.
</para>
</entry>
</row>
<row>
<entry role="catalog_table_entry">
<para role="column_definition">
<structfield>writeback_time</structfield> <type>double precision</type>
</para>
<para>
Time spent in writeback operations in milliseconds (if
<xref linkend="guc-track-io-timing"/> is enabled, otherwise zero). This
includes the time spent queueing write-out requests and, potentially,
the time spent to write out the dirty data.
</para>
</entry>
</row>
<row>
<entry role="catalog_table_entry">
<para role="column_definition">

2
src/backend/catalog/system_views.sql
View File

@ -1131,6 +1131,8 @@ SELECT
b.read_time,
b.writes,
b.write_time,
b.writebacks,
b.writeback_time,
b.extends,
b.extend_time,
b.op_bytes,

32
src/backend/storage/buffer/bufmgr.c
View File

@ -1685,7 +1685,7 @@ again:
FlushBuffer(buf_hdr, NULL, IOOBJECT_RELATION, io_context);
LWLockRelease(content_lock);
ScheduleBufferTagForWriteback(&BackendWritebackContext,
ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context,
&buf_hdr->tag);
}
@ -2725,8 +2725,11 @@ BufferSync(int flags)
CheckpointWriteDelay(flags, (double) num_processed / num_to_scan);
}
/* issue all pending flushes */
IssuePendingWritebacks(&wb_context);
/*
* Issue all pending flushes. Only checkpointer calls BufferSync(), so
* IOContext will always be IOCONTEXT_NORMAL.
*/
IssuePendingWritebacks(&wb_context, IOCONTEXT_NORMAL);
pfree(per_ts_stat);
per_ts_stat = NULL;
@ -3110,7 +3113,11 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context)
UnpinBuffer(bufHdr);
ScheduleBufferTagForWriteback(wb_context, &tag);
/*
* SyncOneBuffer() is only called by checkpointer and bgwriter, so
* IOContext will always be IOCONTEXT_NORMAL.
*/
ScheduleBufferTagForWriteback(wb_context, IOCONTEXT_NORMAL, &tag);
return result | BUF_WRITTEN;
}
@ -5445,7 +5452,8 @@ WritebackContextInit(WritebackContext *context, int *max_pending)
* Add buffer to list of pending writeback requests.
*/
void
ScheduleBufferTagForWriteback(WritebackContext *wb_context, BufferTag *tag)
ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context,
BufferTag *tag)
{
PendingWriteback *pending;
@ -5471,7 +5479,7 @@ ScheduleBufferTagForWriteback(WritebackContext *wb_context, BufferTag *tag)
* is now disabled.
*/
if (wb_context->nr_pending >= *wb_context->max_pending)
IssuePendingWritebacks(wb_context);
IssuePendingWritebacks(wb_context, io_context);
}
#define ST_SORT sort_pending_writebacks
@ -5489,8 +5497,9 @@ ScheduleBufferTagForWriteback(WritebackContext *wb_context, BufferTag *tag)
* error out - it's just a hint.
*/
void
IssuePendingWritebacks(WritebackContext *wb_context)
IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context)
{
instr_time io_start;
int i;
if (wb_context->nr_pending == 0)
@ -5503,6 +5512,8 @@ IssuePendingWritebacks(WritebackContext *wb_context)
sort_pending_writebacks(wb_context->pending_writebacks,
wb_context->nr_pending);
io_start = pgstat_prepare_io_time();
/*
* Coalesce neighbouring writes, but nothing else. For that we iterate
* through the, now sorted, array of pending flushes, and look forward to
@ -5556,6 +5567,13 @@ IssuePendingWritebacks(WritebackContext *wb_context)
smgrwriteback(reln, BufTagGetForkNum(&tag), tag.blockNum, nblocks);
}
/*
* Assume that writeback requests are only issued for buffers containing
* blocks of permanent relations.
*/
pgstat_count_io_op_time(IOOBJECT_RELATION, io_context,
IOOP_WRITEBACK, io_start, wb_context->nr_pending);
wb_context->nr_pending = 0;
}

14
src/backend/utils/activity/pgstat_io.c
View File

@ -424,6 +424,14 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object,
bktype == B_CHECKPOINTER) && io_op == IOOP_EXTEND)
return false;
/*
* Temporary tables are not logged and thus do not require fsync'ing.
* Writeback is not requested for temporary tables.
*/
if (io_object == IOOBJECT_TEMP_RELATION &&
(io_op == IOOP_FSYNC || io_op == IOOP_WRITEBACK))
return false;
/*
* Some IOOps are not valid in certain IOContexts and some IOOps are only
* valid in certain contexts.
@ -448,12 +456,6 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object,
if (strategy_io_context && io_op == IOOP_FSYNC)
return false;
/*
* Temporary tables are not logged and thus do not require fsync'ing.
*/
if (io_context == IOCONTEXT_NORMAL &&
io_object == IOOBJECT_TEMP_RELATION && io_op == IOOP_FSYNC)
return false;
return true;
}

5
src/backend/utils/adt/pgstatfuncs.c
View File

@ -1268,6 +1268,8 @@ typedef enum io_stat_col
IO_COL_READ_TIME,
IO_COL_WRITES,
IO_COL_WRITE_TIME,
IO_COL_WRITEBACKS,
IO_COL_WRITEBACK_TIME,
IO_COL_EXTENDS,
IO_COL_EXTEND_TIME,
IO_COL_CONVERSION,
@ -1303,6 +1305,8 @@ pgstat_get_io_op_index(IOOp io_op)
return IO_COL_REUSES;
case IOOP_WRITE:
return IO_COL_WRITES;
case IOOP_WRITEBACK:
return IO_COL_WRITEBACKS;
}
elog(ERROR, "unrecognized IOOp value: %d", io_op);
@ -1322,6 +1326,7 @@ pgstat_get_io_time_index(IOOp io_op)
{
case IOOP_READ:
case IOOP_WRITE:
case IOOP_WRITEBACK:
case IOOP_EXTEND:
case IOOP_FSYNC:
return pgstat_get_io_op_index(io_op) + 1;

2
src/include/catalog/catversion.h
View File

@ -57,6 +57,6 @@
*/
/* yyyymmddN */
#define CATALOG_VERSION_NO 202305172
#define CATALOG_VERSION_NO 202305173
#endif

6
src/include/catalog/pg_proc.dat
View File

@ -5734,9 +5734,9 @@
proname => 'pg_stat_get_io', provolatile => 'v',
prorows => '30', proretset => 't',
proparallel => 'r', prorettype => 'record', proargtypes => '',
proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}',
proallargtypes => '{text,text,text,int8,float8,int8,float8,int8,float8,int8,float8,int8,int8,int8,int8,int8,float8,timestamptz}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
proargnames => '{backend_type,object,context,reads,read_time,writes,write_time,writebacks,writeback_time,extends,extend_time,op_bytes,hits,evictions,reuses,fsyncs,fsync_time,stats_reset}',
prosrc => 'pg_stat_get_io' },
{ oid => '1136', descr => 'statistics: information about WAL activity',

3
src/include/pgstat.h
View File

@ -300,9 +300,10 @@ typedef enum IOOp
IOOP_READ,
IOOP_REUSE,
IOOP_WRITE,
IOOP_WRITEBACK,
} IOOp;
#define IOOP_NUM_TYPES (IOOP_WRITE + 1)
#define IOOP_NUM_TYPES (IOOP_WRITEBACK + 1)
typedef struct PgStat_BktypeIO
{

5
src/include/storage/buf_internals.h
View File

@ -388,8 +388,9 @@ extern PGDLLIMPORT CkptSortItem *CkptBufferIds;
*/
/* bufmgr.c */
extern void WritebackContextInit(WritebackContext *context, int *max_pending);
extern void IssuePendingWritebacks(WritebackContext *wb_context);
extern void ScheduleBufferTagForWriteback(WritebackContext *wb_context, BufferTag *tag);
extern void IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context);
extern void ScheduleBufferTagForWriteback(WritebackContext *wb_context,
IOContext io_context, BufferTag *tag);
/* freelist.c */
extern IOContext IOContextForStrategy(BufferAccessStrategy strategy);

4
src/test/regress/expected/rules.out
View File

@ -1887,6 +1887,8 @@ pg_stat_io| SELECT backend_type,
read_time,
writes,
write_time,
writebacks,
writeback_time,
extends,
extend_time,
op_bytes,
@ -1896,7 +1898,7 @@ pg_stat_io| SELECT backend_type,
fsyncs,
fsync_time,
stats_reset
FROM pg_stat_get_io() b(backend_type, object, context, reads, read_time, writes, write_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset);
FROM pg_stat_get_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset);
pg_stat_progress_analyze| SELECT s.pid,
s.datid,
d.datname,

4
src/test/regress/expected/stats.out
View File

@ -1445,7 +1445,7 @@ SELECT pg_stat_have_stats('io', 0, 0);
t
(1 row)
SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(hits) AS io_stats_pre_reset
SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset
FROM pg_stat_io \gset
SELECT pg_stat_reset_shared('io');
pg_stat_reset_shared
@ -1453,7 +1453,7 @@ SELECT pg_stat_reset_shared('io');
(1 row)
SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(hits) AS io_stats_post_reset
SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset
FROM pg_stat_io \gset
SELECT :io_stats_post_reset < :io_stats_pre_reset;
?column?

4
src/test/regress/sql/stats.sql
View File

@ -716,10 +716,10 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext
-- Test IO stats reset
SELECT pg_stat_have_stats('io', 0, 0);
SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(hits) AS io_stats_pre_reset
SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset
FROM pg_stat_io \gset
SELECT pg_stat_reset_shared('io');
SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(hits) AS io_stats_post_reset
SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset
FROM pg_stat_io \gset
SELECT :io_stats_post_reset < :io_stats_pre_reset;