mirror of
https://github.com/postgres/postgres.git
synced 2026-04-08 10:38:53 -04:00
f19c0eccae
This allows data checksums to be enabled, or disabled, in a running cluster without restricting access to the cluster during processing. Data checksums could prior to this only be enabled during initdb or when the cluster is offline using the pg_checksums app. This commit introduce functionality to enable, or disable, data checksums while the cluster is running regardless of how it was initialized. A background worker launcher process is responsible for launching a dynamic per-database background worker which will mark all buffers dirty for all relation with storage in order for them to have data checksums calculated on write. Once all relations in all databases have been processed, the data_checksums state will be set to on and the cluster will at that point be identical to one which had data checksums enabled during initialization or via offline processing. When data checksums are being enabled, concurrent I/O operations from backends other than the data checksums worker will write the checksums but not verify them on reading. Only when all backends have absorbed the procsignalbarrier for setting data_checksums to on will they also start verifying checksums on reading. The same process is repeated during disabling; all backends write checksums but do not verify them until the barrier for setting the state to off has been absorbed by all. This in-progress state is used to ensure there are no false negatives (or positives) due to reading a checksum which is not in sync with the page. A new testmodule, test_checksums, is introduced with an extensive set of tests covering both online and offline data checksum mode changes. The tests which run concurrent pgbdench during online processing are gated behind the PG_TEST_EXTRA flag due to being very expensive to run. Two levels of PG_TEST_EXTRA flags exist to turn on a subset of the expensive tests, or the full suite of multiple runs. This work is based on an earlier version of this patch which was reviewed by among others Heikki Linnakangas, Robert Haas, Andres Freund, Tomas Vondra, Michael Banck and Andrey Borodin. During the work on this new version, Tomas Vondra has given invaluable assistance with not only coding and reviewing but very in-depth testing. Author: Daniel Gustafsson <daniel@yesql.se> Author: Magnus Hagander <magnus@hagander.net> Co-authored-by: Tomas Vondra <tomas@vondra.me> Reviewed-by: Tomas Vondra <tomas@vondra.me> Reviewed-by: Andres Freund <andres@anarazel.de> Reviewed-by: Heikki Linnakangas <hlinnaka@iki.fi> Discussion: https://postgr.es/m/CABUevExz9hUUOLnJVr2kpw9Cx=o4MCr1SVKwbupzuxP7ckNutA@mail.gmail.com Discussion: https://postgr.es/m/20181030051643.elbxjww5jjgnjaxg@alap3.anarazel.de Discussion: https://postgr.es/m/CABUevEwE3urLtwxxqdgd5O2oQz9J717ZzMbh+ziCSa5YLLU_BA@mail.gmail.com
204 lines
7.8 KiB
C
204 lines
7.8 KiB
C
/*-------------------------------------------------------------------------
|
|
*
|
|
* progress.h
|
|
* Constants used with the progress reporting facilities defined in
|
|
* backend_status.h. These are possibly interesting to extensions, so we
|
|
* expose them via this header file. Note that if you update these
|
|
* constants, you probably also need to update the views based on them
|
|
* in system_views.sql.
|
|
*
|
|
* Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
|
|
* Portions Copyright (c) 1994, Regents of the University of California
|
|
*
|
|
* src/include/commands/progress.h
|
|
*
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
#ifndef PROGRESS_H
|
|
#define PROGRESS_H
|
|
|
|
/* Progress parameters for (lazy) vacuum */
|
|
#define PROGRESS_VACUUM_PHASE 0
|
|
#define PROGRESS_VACUUM_TOTAL_HEAP_BLKS 1
|
|
#define PROGRESS_VACUUM_HEAP_BLKS_SCANNED 2
|
|
#define PROGRESS_VACUUM_HEAP_BLKS_VACUUMED 3
|
|
#define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4
|
|
#define PROGRESS_VACUUM_MAX_DEAD_TUPLE_BYTES 5
|
|
#define PROGRESS_VACUUM_DEAD_TUPLE_BYTES 6
|
|
#define PROGRESS_VACUUM_NUM_DEAD_ITEM_IDS 7
|
|
#define PROGRESS_VACUUM_INDEXES_TOTAL 8
|
|
#define PROGRESS_VACUUM_INDEXES_PROCESSED 9
|
|
#define PROGRESS_VACUUM_DELAY_TIME 10
|
|
#define PROGRESS_VACUUM_MODE 11
|
|
#define PROGRESS_VACUUM_STARTED_BY 12
|
|
|
|
/* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */
|
|
#define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1
|
|
#define PROGRESS_VACUUM_PHASE_VACUUM_INDEX 2
|
|
#define PROGRESS_VACUUM_PHASE_VACUUM_HEAP 3
|
|
#define PROGRESS_VACUUM_PHASE_INDEX_CLEANUP 4
|
|
#define PROGRESS_VACUUM_PHASE_TRUNCATE 5
|
|
#define PROGRESS_VACUUM_PHASE_FINAL_CLEANUP 6
|
|
|
|
/* Modes of vacuum (as advertised via PROGRESS_VACUUM_MODE) */
|
|
#define PROGRESS_VACUUM_MODE_NORMAL 1
|
|
#define PROGRESS_VACUUM_MODE_AGGRESSIVE 2
|
|
#define PROGRESS_VACUUM_MODE_FAILSAFE 3
|
|
|
|
/* Reasons for vacuum (as advertised via PROGRESS_VACUUM_STARTED_BY) */
|
|
#define PROGRESS_VACUUM_STARTED_BY_MANUAL 1
|
|
#define PROGRESS_VACUUM_STARTED_BY_AUTOVACUUM 2
|
|
#define PROGRESS_VACUUM_STARTED_BY_AUTOVACUUM_WRAPAROUND 3
|
|
|
|
/* Progress parameters for analyze */
|
|
#define PROGRESS_ANALYZE_PHASE 0
|
|
#define PROGRESS_ANALYZE_BLOCKS_TOTAL 1
|
|
#define PROGRESS_ANALYZE_BLOCKS_DONE 2
|
|
#define PROGRESS_ANALYZE_EXT_STATS_TOTAL 3
|
|
#define PROGRESS_ANALYZE_EXT_STATS_COMPUTED 4
|
|
#define PROGRESS_ANALYZE_CHILD_TABLES_TOTAL 5
|
|
#define PROGRESS_ANALYZE_CHILD_TABLES_DONE 6
|
|
#define PROGRESS_ANALYZE_CURRENT_CHILD_TABLE_RELID 7
|
|
#define PROGRESS_ANALYZE_DELAY_TIME 8
|
|
#define PROGRESS_ANALYZE_STARTED_BY 9
|
|
|
|
/* Phases of analyze (as advertised via PROGRESS_ANALYZE_PHASE) */
|
|
#define PROGRESS_ANALYZE_PHASE_ACQUIRE_SAMPLE_ROWS 1
|
|
#define PROGRESS_ANALYZE_PHASE_ACQUIRE_SAMPLE_ROWS_INH 2
|
|
#define PROGRESS_ANALYZE_PHASE_COMPUTE_STATS 3
|
|
#define PROGRESS_ANALYZE_PHASE_COMPUTE_EXT_STATS 4
|
|
#define PROGRESS_ANALYZE_PHASE_FINALIZE_ANALYZE 5
|
|
|
|
/* Reasons for analyze (as advertised via PROGRESS_ANALYZE_STARTED_BY) */
|
|
#define PROGRESS_ANALYZE_STARTED_BY_MANUAL 1
|
|
#define PROGRESS_ANALYZE_STARTED_BY_AUTOVACUUM 2
|
|
|
|
/*
|
|
* Progress parameters for REPACK.
|
|
*
|
|
* Values for PROGRESS_REPACK_COMMAND are as in RepackCommand.
|
|
*
|
|
* Note: Since REPACK shares code with CLUSTER, these values are also
|
|
* used by CLUSTER. (CLUSTER being now deprecated, it makes little sense to
|
|
* maintain a separate set of constants.)
|
|
*/
|
|
#define PROGRESS_REPACK_COMMAND 0
|
|
#define PROGRESS_REPACK_PHASE 1
|
|
#define PROGRESS_REPACK_INDEX_RELID 2
|
|
#define PROGRESS_REPACK_HEAP_TUPLES_SCANNED 3
|
|
#define PROGRESS_REPACK_HEAP_TUPLES_WRITTEN 4
|
|
#define PROGRESS_REPACK_TOTAL_HEAP_BLKS 5
|
|
#define PROGRESS_REPACK_HEAP_BLKS_SCANNED 6
|
|
#define PROGRESS_REPACK_INDEX_REBUILD_COUNT 7
|
|
|
|
/*
|
|
* Phases of repack (as advertised via PROGRESS_REPACK_PHASE).
|
|
*/
|
|
#define PROGRESS_REPACK_PHASE_SEQ_SCAN_HEAP 1
|
|
#define PROGRESS_REPACK_PHASE_INDEX_SCAN_HEAP 2
|
|
#define PROGRESS_REPACK_PHASE_SORT_TUPLES 3
|
|
#define PROGRESS_REPACK_PHASE_WRITE_NEW_HEAP 4
|
|
#define PROGRESS_REPACK_PHASE_SWAP_REL_FILES 5
|
|
#define PROGRESS_REPACK_PHASE_REBUILD_INDEX 6
|
|
#define PROGRESS_REPACK_PHASE_FINAL_CLEANUP 7
|
|
|
|
/* Progress parameters for CREATE INDEX */
|
|
/* 3, 4 and 5 reserved for "waitfor" metrics */
|
|
#define PROGRESS_CREATEIDX_COMMAND 0
|
|
#define PROGRESS_CREATEIDX_INDEX_OID 6
|
|
#define PROGRESS_CREATEIDX_ACCESS_METHOD_OID 8
|
|
#define PROGRESS_CREATEIDX_PHASE 9 /* AM-agnostic phase # */
|
|
#define PROGRESS_CREATEIDX_SUBPHASE 10 /* phase # filled by AM */
|
|
#define PROGRESS_CREATEIDX_TUPLES_TOTAL 11
|
|
#define PROGRESS_CREATEIDX_TUPLES_DONE 12
|
|
#define PROGRESS_CREATEIDX_PARTITIONS_TOTAL 13
|
|
#define PROGRESS_CREATEIDX_PARTITIONS_DONE 14
|
|
/* 15 and 16 reserved for "block number" metrics */
|
|
|
|
/* Phases of CREATE INDEX (as advertised via PROGRESS_CREATEIDX_PHASE) */
|
|
#define PROGRESS_CREATEIDX_PHASE_WAIT_1 1
|
|
#define PROGRESS_CREATEIDX_PHASE_BUILD 2
|
|
#define PROGRESS_CREATEIDX_PHASE_WAIT_2 3
|
|
#define PROGRESS_CREATEIDX_PHASE_VALIDATE_IDXSCAN 4
|
|
#define PROGRESS_CREATEIDX_PHASE_VALIDATE_SORT 5
|
|
#define PROGRESS_CREATEIDX_PHASE_VALIDATE_TABLESCAN 6
|
|
#define PROGRESS_CREATEIDX_PHASE_WAIT_3 7
|
|
#define PROGRESS_CREATEIDX_PHASE_WAIT_4 8
|
|
#define PROGRESS_CREATEIDX_PHASE_WAIT_5 9
|
|
|
|
/*
|
|
* Subphases of CREATE INDEX, for index_build.
|
|
*/
|
|
#define PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE 1
|
|
/* Additional phases are defined by each AM */
|
|
|
|
/* Commands of PROGRESS_CREATEIDX */
|
|
#define PROGRESS_CREATEIDX_COMMAND_CREATE 1
|
|
#define PROGRESS_CREATEIDX_COMMAND_CREATE_CONCURRENTLY 2
|
|
#define PROGRESS_CREATEIDX_COMMAND_REINDEX 3
|
|
#define PROGRESS_CREATEIDX_COMMAND_REINDEX_CONCURRENTLY 4
|
|
|
|
/* Lock holder wait counts */
|
|
#define PROGRESS_WAITFOR_TOTAL 3
|
|
#define PROGRESS_WAITFOR_DONE 4
|
|
#define PROGRESS_WAITFOR_CURRENT_PID 5
|
|
|
|
/* Block numbers in a generic relation scan */
|
|
#define PROGRESS_SCAN_BLOCKS_TOTAL 15
|
|
#define PROGRESS_SCAN_BLOCKS_DONE 16
|
|
|
|
/* Progress parameters for pg_basebackup */
|
|
#define PROGRESS_BASEBACKUP_PHASE 0
|
|
#define PROGRESS_BASEBACKUP_BACKUP_TOTAL 1
|
|
#define PROGRESS_BASEBACKUP_BACKUP_STREAMED 2
|
|
#define PROGRESS_BASEBACKUP_TBLSPC_TOTAL 3
|
|
#define PROGRESS_BASEBACKUP_TBLSPC_STREAMED 4
|
|
#define PROGRESS_BASEBACKUP_BACKUP_TYPE 5
|
|
|
|
/* Phases of pg_basebackup (as advertised via PROGRESS_BASEBACKUP_PHASE) */
|
|
#define PROGRESS_BASEBACKUP_PHASE_WAIT_CHECKPOINT 1
|
|
#define PROGRESS_BASEBACKUP_PHASE_ESTIMATE_BACKUP_SIZE 2
|
|
#define PROGRESS_BASEBACKUP_PHASE_STREAM_BACKUP 3
|
|
#define PROGRESS_BASEBACKUP_PHASE_WAIT_WAL_ARCHIVE 4
|
|
#define PROGRESS_BASEBACKUP_PHASE_TRANSFER_WAL 5
|
|
|
|
/* Types of pg_basebackup (as advertised via PROGRESS_BASEBACKUP_BACKUP_TYPE) */
|
|
#define PROGRESS_BASEBACKUP_BACKUP_TYPE_FULL 1
|
|
#define PROGRESS_BASEBACKUP_BACKUP_TYPE_INCREMENTAL 2
|
|
|
|
/* Progress parameters for PROGRESS_COPY */
|
|
#define PROGRESS_COPY_BYTES_PROCESSED 0
|
|
#define PROGRESS_COPY_BYTES_TOTAL 1
|
|
#define PROGRESS_COPY_TUPLES_PROCESSED 2
|
|
#define PROGRESS_COPY_TUPLES_EXCLUDED 3
|
|
#define PROGRESS_COPY_COMMAND 4
|
|
#define PROGRESS_COPY_TYPE 5
|
|
#define PROGRESS_COPY_TUPLES_SKIPPED 6
|
|
|
|
/* Commands of COPY (as advertised via PROGRESS_COPY_COMMAND) */
|
|
#define PROGRESS_COPY_COMMAND_FROM 1
|
|
#define PROGRESS_COPY_COMMAND_TO 2
|
|
|
|
/* Types of COPY commands (as advertised via PROGRESS_COPY_TYPE) */
|
|
#define PROGRESS_COPY_TYPE_FILE 1
|
|
#define PROGRESS_COPY_TYPE_PROGRAM 2
|
|
#define PROGRESS_COPY_TYPE_PIPE 3
|
|
#define PROGRESS_COPY_TYPE_CALLBACK 4
|
|
|
|
/* Progress parameters for PROGRESS_DATACHECKSUMS */
|
|
#define PROGRESS_DATACHECKSUMS_PHASE 0
|
|
#define PROGRESS_DATACHECKSUMS_DBS_TOTAL 1
|
|
#define PROGRESS_DATACHECKSUMS_DBS_DONE 2
|
|
#define PROGRESS_DATACHECKSUMS_RELS_TOTAL 3
|
|
#define PROGRESS_DATACHECKSUMS_RELS_DONE 4
|
|
#define PROGRESS_DATACHECKSUMS_BLOCKS_TOTAL 5
|
|
#define PROGRESS_DATACHECKSUMS_BLOCKS_DONE 6
|
|
|
|
/* Phases of datachecksumsworker operation */
|
|
#define PROGRESS_DATACHECKSUMS_PHASE_ENABLING 0
|
|
#define PROGRESS_DATACHECKSUMS_PHASE_DISABLING 1
|
|
#define PROGRESS_DATACHECKSUMS_PHASE_WAITING_TEMPREL 2
|
|
#define PROGRESS_DATACHECKSUMS_PHASE_WAITING_BARRIER 3
|
|
#define PROGRESS_DATACHECKSUMS_PHASE_DONE 4
|
|
|
|
#endif
|