From a28166df8c5a00ec3d9b3550a4f6598593ebf873 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 23 Aug 2023 15:15:28 -0700 Subject: [PATCH] ci: Make compute resources for CI configurable See prior commit for an explanation for the goal of the change and why it had to be split into two commits. Reviewed-by: Daniel Gustafsson Reviewed-by: Nazir Bilal Yavuz Discussion: https://postgr.es/m/20230808021541.7lbzdefvma7qmn3w@awork3.anarazel.de Backpatch: 15-, where CI support was added --- .cirrus.tasks.yml | 45 ---------------------------- .cirrus.yml | 73 +++++++++++++++++++++++++++++++++++++++++++++ src/tools/ci/README | 17 +++++++++++ 3 files changed, 90 insertions(+), 45 deletions(-) create mode 100644 .cirrus.yml diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index f4276ad869..0cf7ba7799 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -5,12 +5,6 @@ env: - # Source of images / containers - GCP_PROJECT: pg-ci-images - IMAGE_PROJECT: $GCP_PROJECT - CONTAINER_REPO: us-docker.pkg.dev/${GCP_PROJECT}/ci - DISK_SIZE: 25 - # The lower depth accelerates git clone. Use a bit of depth so that # concurrent tasks and retrying older jobs have a chance of working. CIRRUS_CLONE_DEPTH: 500 @@ -29,45 +23,6 @@ env: PG_TEST_EXTRA: kerberos ldap ssl load_balance -# Define how to run various types of tasks. - -# VMs provided by cirrus-ci. Each user has a limited number of "free" credits -# for testing. -cirrus_community_vm_template: &cirrus_community_vm_template - compute_engine_instance: - image_project: $IMAGE_PROJECT - image: family/$IMAGE_FAMILY - platform: $PLATFORM - cpu: $CPUS - disk: $DISK_SIZE - - -default_linux_task_template: &linux_task_template - env: - PLATFORM: linux - <<: *cirrus_community_vm_template - - -default_freebsd_task_template: &freebsd_task_template - env: - PLATFORM: freebsd - <<: *cirrus_community_vm_template - - -default_windows_task_template: &windows_task_template - env: - PLATFORM: windows - <<: *cirrus_community_vm_template - - -# macos workers provided by cirrus-ci -default_macos_task_template: &macos_task_template - env: - PLATFORM: macos - macos_instance: - image: $IMAGE - - # What files to preserve in case tests fail on_failure_ac: &on_failure_ac log_artifacts: diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 0000000000..a83129ae46 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,73 @@ +# CI configuration file for CI utilizing cirrus-ci.org +# +# For instructions on how to enable the CI integration in a repository and +# further details, see src/tools/ci/README +# +# +# The actual CI tasks are defined in .cirrus.tasks.yml. To make the compute +# resources for CI configurable on a repository level, the "final" CI +# configuration is the combination of: +# +# 1) the contents of this file +# +# 2) if defined, the contents of the file referenced by the, repository +# level, REPO_CI_CONFIG_GIT_URL variable (see +# https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted +# format) +# +# 3) .cirrus.tasks.yml +# +# This composition is done by .cirrus.star + + +env: + # Source of images / containers + GCP_PROJECT: pg-ci-images + IMAGE_PROJECT: $GCP_PROJECT + CONTAINER_REPO: us-docker.pkg.dev/${GCP_PROJECT}/ci + DISK_SIZE: 25 + + +# Define how to run various types of tasks. + +# VMs provided by cirrus-ci. Each user has a limited number of "free" credits +# for testing. +cirrus_community_vm_template: &cirrus_community_vm_template + compute_engine_instance: + image_project: $IMAGE_PROJECT + image: family/$IMAGE_FAMILY + platform: $PLATFORM + cpu: $CPUS + disk: $DISK_SIZE + + +default_linux_task_template: &linux_task_template + env: + PLATFORM: linux + <<: *cirrus_community_vm_template + + +default_freebsd_task_template: &freebsd_task_template + env: + PLATFORM: freebsd + <<: *cirrus_community_vm_template + + +default_windows_task_template: &windows_task_template + env: + PLATFORM: windows + <<: *cirrus_community_vm_template + + +# macos workers provided by cirrus-ci +default_macos_task_template: &macos_task_template + env: + PLATFORM: macos + macos_instance: + image: $IMAGE + + +# Contents of REPO_CI_CONFIG_GIT_URL, if defined, will be inserted here, +# followed by the contents .cirrus.tasks.yml. This allows +# REPO_CI_CONFIG_GIT_URL to override how the task types above will be +# executed, e.g. using a custom compute account or permanent workers. diff --git a/src/tools/ci/README b/src/tools/ci/README index 80d01939e8..30ddd200c9 100644 --- a/src/tools/ci/README +++ b/src/tools/ci/README @@ -65,3 +65,20 @@ messages. Currently the following controls are available: Only runs CI on operating systems specified. This can be useful when addressing portability issues affecting only a subset of platforms. + + +Using custom compute resources for CI +===================================== + +When running a lot of tests in a repository, cirrus-ci's free credits do not +suffice. In those cases a repository can be configured to use other +infrastructure for running tests. To do so, the REPO_CI_CONFIG_GIT_URL +variable can be configured for the repository in the cirrus-ci web interface, +at https://cirrus-ci.com/github/. The file referenced +(see https://cirrus-ci.org/guide/programming-tasks/#fs) by the variable can +overwrite the default execution method for different operating systems, +defined in .cirrus.yml, by redefining the relevant yaml anchors. + +Custom compute resources can be provided using +- https://cirrus-ci.org/guide/supported-computing-services/ +- https://cirrus-ci.org/guide/persistent-workers/