postgresql/.cirrus.tasks.yml
Andres Freund e4693c68a4 ci: Make compute resources for CI configurable
See prior commit for an explanation for the goal of the change and why it had
to be split into two commits.

Reviewed-by: Daniel Gustafsson <daniel@yesql.se>
Reviewed-by: Nazir Bilal Yavuz <byavuz81@gmail.com>
Discussion: https://postgr.es/m/20230808021541.7lbzdefvma7qmn3w@awork3.anarazel.de
Backpatch: 15-, where CI support was added
2023-08-23 15:15:29 -07:00

591 lines
18 KiB
YAML

# CI configuration file for CI utilizing cirrus-ci.org
#
# For instructions on how to enable the CI integration in a repository and
# further details, see src/tools/ci/README
env:
# The lower depth accelerates git clone. Use a bit of depth so that
# concurrent tasks and retrying older jobs have a chance of working.
CIRRUS_CLONE_DEPTH: 500
# Useful to be able to analyse what in a script takes long
CIRRUS_LOG_TIMESTAMP: true
CCACHE_MAXSIZE: "250M"
# target to test, for all but windows
CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
CHECKFLAGS: -Otarget
PROVE_FLAGS: --timer
PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
PG_TEST_EXTRA: kerberos ldap ssl
# What files to preserve in case tests fail
on_failure: &on_failure
log_artifacts:
paths:
- "**/*.log"
- "**/*.diffs"
- "**/regress_log_*"
type: text/plain
task:
name: FreeBSD - 13
env:
# FreeBSD on GCP is slow when running with larger number of CPUS /
# jobs. Using one more job than cpus seems to work best.
CPUS: 2
BUILD_JOBS: 3
TEST_JOBS: 3
IMAGE_FAMILY: pg-ci-freebsd-13
DISK_SIZE: 50
CCACHE_DIR: /tmp/ccache_dir
<<: *freebsd_task_template
only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
sysinfo_script: |
id
uname -a
ulimit -a -H && ulimit -a -S
export
ccache_cache:
folder: $CCACHE_DIR
# Work around performance issues due to 32KB block size
repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
create_user_script: |
pw useradd postgres
chown -R postgres:postgres .
mkdir -p ${CCACHE_DIR}
chown -R postgres:postgres ${CCACHE_DIR}
setup_core_files_script: |
mkdir -m 770 /tmp/cores
chown root:postgres /tmp/cores
sysctl kern.corefile='/tmp/cores/%N.%P.core'
setup_additional_packages_script: |
#pkg install -y ...
# NB: Intentionally build without --with-llvm. The freebsd image size is
# already large enough to make VM startup slow, and even without llvm
# freebsd already takes longer than other platforms except for windows.
configure_script: |
su postgres <<-EOF
./configure \
--enable-cassert --enable-debug --enable-tap-tests \
--enable-nls \
\
--with-gssapi \
--with-icu \
--with-ldap \
--with-libxml \
--with-libxslt \
--with-lz4 \
--with-pam \
--with-perl \
--with-python \
--with-ssl=openssl \
--with-tcl --with-tclconfig=/usr/local/lib/tcl8.6/ \
--with-uuid=bsd \
--with-zstd \
\
--with-includes=/usr/local/include \
--with-libs=/usr/local/lib \
\
CC="ccache cc" \
CXX="ccache c++" \
CFLAGS="-Og -ggdb"
EOF
build_script: su postgres -c "gmake -s -j${BUILD_JOBS} world-bin"
upload_caches: ccache
# The use of script avoids make -Otarget complaints about fcntl() on
# platforms without support for locking pipes. See also
# https://savannah.gnu.org/bugs/?60774
# script uses pseudo-ttys, which do support locking.
test_world_script:
- su postgres -c "time script test.log gmake -s -j${TEST_JOBS} ${CHECK} ${CHECKFLAGS}"
on_failure:
<<: *on_failure
cores_script: src/tools/ci/cores_backtrace.sh freebsd /tmp/cores
# configure feature flags, shared between the task running the linux tests and
# the CompilerWarnings task
LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >-
--with-gssapi
--with-icu
--with-ldap
--with-libxml
--with-libxslt
--with-llvm
--with-lz4
--with-pam
--with-perl
--with-python
--with-selinux
--with-ssl=openssl
--with-systemd
--with-tcl --with-tclconfig=/usr/lib/tcl8.6/
--with-uuid=ossp
--with-zstd
task:
name: Linux - Debian Bullseye
env:
CPUS: 4
BUILD_JOBS: 4
TEST_JOBS: 8 # experimentally derived to be a decent choice
IMAGE_FAMILY: pg-ci-bullseye
CCACHE_DIR: /tmp/ccache_dir
DEBUGINFOD_URLS: "https://debuginfod.debian.net"
LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
<<: *linux_task_template
only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
ccache_cache:
folder: ${CCACHE_DIR}
sysinfo_script: |
id
uname -a
cat /proc/cmdline
ulimit -a -H && ulimit -a -S
export
create_user_script: |
useradd -m postgres
chown -R postgres:postgres .
mkdir -p ${CCACHE_DIR}
chown -R postgres:postgres ${CCACHE_DIR}
echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
su postgres -c "ulimit -l -H && ulimit -l -S"
setup_core_files_script: |
mkdir -m 770 /tmp/cores
chown root:postgres /tmp/cores
sysctl kernel.core_pattern='/tmp/cores/%e-%s-%p.core'
setup_additional_packages_script: |
#apt-get update
#DEBIAN_FRONTEND=noninteractive apt-get -y install ...
configure_script: |
su postgres <<-EOF
./configure \
--enable-cassert --enable-debug --enable-tap-tests \
--enable-nls \
\
${LINUX_CONFIGURE_FEATURES} \
\
CC="ccache gcc" \
CXX="ccache g++" \
CLANG="ccache clang" \
CFLAGS="-Og -ggdb" \
CXXFLAGS="-Og -ggdb"
EOF
build_script: su postgres -c "make -s -j${BUILD_JOBS} world-bin"
upload_caches: ccache
test_world_script: |
su postgres <<-EOF
ulimit -c unlimited # default is 0
make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
EOF
on_failure:
<<: *on_failure
cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores
task:
name: macOS - Ventura
env:
CPUS: 4 # always get that much for cirrusci macOS instances
BUILD_JOBS: $CPUS
# Test performance regresses noticably when using all cores. 8 seems to
# work OK. See
# https://postgr.es/m/20220927040208.l3shfcidovpzqxfh%40awork3.anarazel.de
TEST_JOBS: 8
IMAGE: ghcr.io/cirruslabs/macos-ventura-base:latest
CIRRUS_WORKING_DIR: ${HOME}/pgsql/
CCACHE_DIR: ${HOME}/ccache
MACPORTS_CACHE: ${HOME}/macports-cache
<<: *macos_task_template
only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
sysinfo_script: |
id
uname -a
ulimit -a -H && ulimit -a -S
export
setup_core_files_script:
- mkdir ${HOME}/cores
- sudo sysctl kern.corefile="${HOME}/cores/core.%P"
# Use macports, even though homebrew is installed. The installation
# of the additional packages we need would take quite a while with
# homebrew, even if we cache the downloads. We can't cache all of
# homebrew, because it's already large. So we use macports. To cache
# the installation we create a .dmg file that we mount if it already
# exists.
# XXX: The reason for the direct p5.34* references is that we'd need
# the large macport tree around to figure out that p5-io-tty is
# actually p5.34-io-tty. Using the unversioned name works, but
# updates macports every time.
macports_cache:
folder: ${MACPORTS_CACHE}
setup_additional_packages_script: |
sh src/tools/ci/ci_macports_packages.sh \
ccache \
gmake \
icu \
kerberos5 \
lz4 \
openldap \
openssl \
p5.34-io-tty \
p5.34-ipc-run \
tcl \
zstd
# Make macports install visible for subsequent steps
echo PATH=/opt/local/sbin/:/opt/local/bin/:$PATH >> $CIRRUS_ENV
upload_caches: macports
ccache_cache:
folder: $CCACHE_DIR
configure_script: |
export PKG_CONFIG_PATH="/opt/local/lib/pkgconfig/"
./configure \
--enable-cassert --enable-debug --enable-tap-tests \
--enable-nls \
\
--with-bonjour \
--with-gssapi \
--with-icu \
--with-ldap \
--with-libxml \
--with-libxslt \
--with-lz4 \
--with-perl \
--with-python \
--with-ssl=openssl \
--with-tcl --with-tclconfig=/opt/local/lib \
--with-uuid=e2fs \
--with-zstd \
\
--prefix=${HOME}/install \
--with-includes=/opt/local/include \
--with-libs=/opt/local/lib \
\
CC="ccache cc" \
CXX="ccache c++" \
CLANG="ccache ${brewpath}/llvm/bin/ccache" \
CFLAGS="-Og -ggdb" \
CXXFLAGS="-Og -ggdb" \
\
LLVM_CONFIG=${brewpath}/llvm/bin/llvm-config \
PYTHON=python3
build_script: gmake -s -j${BUILD_JOBS} world-bin
upload_caches: ccache
test_world_script: |
ulimit -c unlimited # default is 0
ulimit -n 1024 # default is 256, pretty low
# See freebsd use of script for explanation
script test.log gmake -s -j${TEST_JOBS} ${CHECK} ${CHECKFLAGS}
on_failure:
<<: *on_failure
cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"
task:
name: Windows - Server 2019, VS 2019
env:
# Half the allowed per-user CPU cores
CPUS: 4
# Our windows infrastructure doesn't have test concurrency above the level
# of a single vcregress test target. Due to that, it's useful to run prove
# with multiple jobs. For the other tasks it isn't, because two sources
# (make and prove) of concurrency can overload machines.
#
# The concrete choice of 10 is based on a small bit of experimentation and
# likely can be improved upon further.
PROVE_FLAGS: -j10 --timer
# The default cirrus working dir is in a directory msbuild complains about
CIRRUS_WORKING_DIR: "c:/cirrus"
# Avoid re-installing over and over
NO_TEMP_INSTALL: 1
# git's tar doesn't deal with drive letters, see
# https://postgr.es/m/b6782dc3-a7b0-ed56-175f-f8f54cb08d67%40dunslane.net
TAR: "c:/windows/system32/tar.exe"
# Avoids port conflicts between concurrent tap test runs
PG_TEST_USE_UNIX_SOCKETS: 1
PG_REGRESS_SOCK_DIR: "c:/cirrus/"
# -m enables parallelism
# verbosity:minimal + Summary reduce verbosity, while keeping a summary of
# errors/warnings
# ForceNoAlign prevents msbuild from introducing line-breaks for long lines
# disable file tracker, we're never going to rebuild, and it slows down the
# build
MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo
DISK_SIZE: 50
# If tests hang forever, cirrus eventually times out. In that case log
# output etc is not uploaded, making the problem hard to debug. Of course
# tests internally should have shorter timeouts, but that's proven to not
# be sufficient. 15min currently is fast enough to finish individual test
# "suites".
T_C: "\"C:/Program Files/Git/usr/bin/timeout.exe\" -v -k60s 15m"
# startcreate_script starts a postgres instance that we don't want to get
# killed at the end of that script (it's stopped in stop_script). Can't
# trivially use background_scripts because a) need pg_ctl's dropping of
# permissions b) need to wait for startup to have finished, and we don't
# currently have a tool for that...
CIRRUS_ESCAPING_PROCESSES: 1
# Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That
# prevents crash reporting from working unless binaries do SetErrorMode()
# themselves. Furthermore, it appears that either python or, more likely,
# the C runtime has a bug where SEM_NOGPFAULTERRORBOX can very
# occasionally *trigger* a crash on process exit - which is hard to debug,
# given that it explicitly prevents crash dumps from working...
# 0x8001 is SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX
CIRRUS_WINDOWS_ERROR_MODE: 0x8001
IMAGE_FAMILY: pg-ci-windows-ci-vs-2019
<<: *windows_task_template
only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
sysinfo_script: |
chcp
systeminfo
powershell -Command get-psdrive -psprovider filesystem
set
setup_additional_packages_script: |
REM choco install -y --no-progress ...
configure_script:
# copy errors out when using forward slashes
- copy src\tools\ci\windows_build_config.pl src\tools\msvc\config.pl
- vcvarsall x64
- perl src/tools/msvc/mkvcbuild.pl
build_script:
- vcvarsall x64
- msbuild %MSBFLAGS% pgsql.sln
tempinstall_script:
# Installation on windows currently only completely works from src/tools/msvc
- cd src/tools/msvc && perl install.pl %CIRRUS_WORKING_DIR%/tmp_install
test_regress_parallel_script: |
%T_C% perl src/tools/msvc/vcregress.pl check parallel
startcreate_script: |
rem paths to binaries need backslashes
tmp_install\bin\pg_ctl.exe initdb -D tmp_check/db -l tmp_check/initdb.log --options=--no-sync
echo include '%TEMP_CONFIG%' >> tmp_check/db/postgresql.conf
tmp_install\bin\pg_ctl.exe start -D tmp_check/db -l tmp_check/postmaster.log
test_pl_script: |
%T_C% perl src/tools/msvc/vcregress.pl plcheck
test_isolation_script: |
%T_C% perl src/tools/msvc/vcregress.pl isolationcheck
test_modules_script: |
%T_C% perl src/tools/msvc/vcregress.pl modulescheck
test_contrib_script: |
%T_C% perl src/tools/msvc/vcregress.pl contribcheck
stop_script: |
tmp_install\bin\pg_ctl.exe stop -D tmp_check/db -l tmp_check/postmaster.log
test_ssl_script: |
set with_ssl=openssl
%T_C% perl src/tools/msvc/vcregress.pl taptest ./src/test/ssl/
test_subscription_script: |
%T_C% perl src/tools/msvc/vcregress.pl taptest ./src/test/subscription/
test_authentication_script: |
%T_C% perl src/tools/msvc/vcregress.pl taptest ./src/test/authentication/
test_recovery_script: |
%T_C% perl src/tools/msvc/vcregress.pl recoverycheck
test_bin_script: |
%T_C% perl src/tools/msvc/vcregress.pl bincheck
test_ecpg_script: |
rem tries to build additional stuff
vcvarsall x64
rem References ecpg_regression.proj in the current dir
cd src/tools/msvc
%T_C% perl vcregress.pl ecpgcheck
on_failure:
<<: *on_failure
crashlog_artifacts:
path: "crashlog-*.txt"
type: text/plain
task:
name: CompilerWarnings
# To limit unnecessary work only run this once the normal linux test succeeds
depends_on:
- Linux - Debian Bullseye
env:
CPUS: 4
BUILD_JOBS: 4
IMAGE_FAMILY: pg-ci-bullseye
# Use larger ccache cache, as this task compiles with multiple compilers /
# flag combinations
CCACHE_MAXSIZE: "1GB"
CCACHE_DIR: "/tmp/ccache_dir"
LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
# task that did not run, count as a success, so we need to recheck Linux'
# condition here ...
only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
<<: *linux_task_template
sysinfo_script: |
id
uname -a
cat /proc/cmdline
ulimit -a -H && ulimit -a -S
gcc -v
clang -v
export
ccache_cache:
folder: $CCACHE_DIR
setup_additional_packages_script: |
#apt-get update
#DEBIAN_FRONTEND=noninteractive apt-get -y install ...
###
# Test that code can be built with gcc/clang without warnings
###
setup_script: echo "COPT=-Werror" > src/Makefile.custom
# Trace probes have a history of getting accidentally broken. Use the
# different compilers to build with different combinations of dtrace on/off
# and cassert on/off.
# gcc, cassert off, dtrace on
always:
gcc_warning_script: |
time ./configure \
--cache gcc.cache \
--enable-dtrace \
${LINUX_CONFIGURE_FEATURES} \
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} world-bin
# gcc, cassert on, dtrace off
always:
gcc_a_warning_script: |
time ./configure \
--cache gcc.cache \
--enable-cassert \
${LINUX_CONFIGURE_FEATURES} \
CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} world-bin
# clang, cassert off, dtrace off
always:
clang_warning_script: |
time ./configure \
--cache clang.cache \
${LINUX_CONFIGURE_FEATURES} \
CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} world-bin
# clang, cassert on, dtrace on
always:
clang_a_warning_script: |
time ./configure \
--cache clang.cache \
--enable-cassert \
--enable-dtrace \
${LINUX_CONFIGURE_FEATURES} \
CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} world-bin
# cross-compile to windows
always:
mingw_cross_warning_script: |
time ./configure \
--host=x86_64-w64-mingw32 \
--enable-cassert \
CC="ccache x86_64-w64-mingw32-gcc" \
CXX="ccache x86_64-w64-mingw32-g++"
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} world-bin
###
# Verify docs can be built
###
# XXX: Only do this if there have been changes in doc/ since last build
always:
docs_build_script: |
time ./configure \
--cache gcc.cache \
CC="ccache gcc" \
CXX="ccache g++" \
CLANG="ccache clang"
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} -C doc
###
# Verify headerscheck / cpluspluscheck succeed
#
# - Don't use ccache, the files are uncacheable, polluting ccache's
# cache
# - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
# - XXX have to disable ICU to avoid errors:
# https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
# - XXX: the -Wno-register avoids verbose warnings:
# https://postgr.es/m/20220308181837.aun3tdtdvao4vb7o%40alap3.anarazel.de
###
always:
headers_headerscheck_script: |
time ./configure \
${LINUX_CONFIGURE_FEATURES} \
--without-icu \
--quiet \
CC="gcc" CXX"=g++" CLANG="clang"
make -s -j${BUILD_JOBS} clean
time make -s headerscheck EXTRAFLAGS='-fmax-errors=10'
headers_cpluspluscheck_script: |
time make -s cpluspluscheck EXTRAFLAGS='-Wno-register -fmax-errors=10'
always:
upload_caches: ccache