From 11f53b1063e796dbb766b7ad3fd46e98806bea43 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 5 Sep 2008 12:11:18 +0000 Subject: [PATCH] Code coverage testing with gcov. Documentation is in the regression test chapter. Author: Michelle Caisse --- GNUmakefile.in | 21 +++- configure | 207 ++++++++++++++++++++++++++++++++- configure.in | 35 +++++- doc/src/sgml/installation.sgml | 17 ++- doc/src/sgml/regress.sgml | 36 +++++- src/Makefile.global.in | 52 ++++++++- src/backend/common.mk | 8 +- 7 files changed, 361 insertions(+), 15 deletions(-) diff --git a/GNUmakefile.in b/GNUmakefile.in index c71f8a6088..4335c12b00 100644 --- a/GNUmakefile.in +++ b/GNUmakefile.in @@ -1,7 +1,7 @@ # # PostgreSQL top level makefile # -# $PostgreSQL: pgsql/GNUmakefile.in,v 1.47 2008/03/18 16:24:50 petere Exp $ +# $PostgreSQL: pgsql/GNUmakefile.in,v 1.48 2008/09/05 12:11:17 petere Exp $ # subdir = @@ -61,6 +61,25 @@ GNUmakefile: GNUmakefile.in $(top_builddir)/config.status ./config.status $@ +########################################################################## + +coverage: + $(MAKE) -C src/backend $@ + +.PHONY: coverage-html +coverage-html: coverage + rm -rf coverage + mkdir coverage + $(GENHTML) --show-details --legend --output-directory=coverage --title=PostgreSQL --num-spaces=4 `find src/backend -name lcov.info -print` + +ifeq ($(enable_coverage),yes) +clean distclean maintainer-clean: clean-coverage-local +.PHONY: clean-coverage-local +clean-coverage-local: + rm -rf coverage +endif + + ########################################################################## distdir = postgresql-$(VERSION) diff --git a/configure b/configure index 96a7e1c3d4..6f87fba268 100755 --- a/configure +++ b/configure @@ -672,6 +672,10 @@ enable_shared enable_rpath enable_debug enable_profiling +GCOV +LCOV +GENHTML +enable_coverage DTRACE DTRACEFLAGS enable_dtrace @@ -1356,6 +1360,7 @@ Optional Features: --disable-spinlocks do not use spinlocks --enable-debug build with debugging symbols (-g) --enable-profiling build with profiling enabled + --enable-coverage build with coverage testing instrumentation --enable-dtrace build with DTrace support --enable-depend turn on automatic dependency tracking --enable-cassert enable assertion checks (for debugging) @@ -2468,6 +2473,178 @@ fi +# +# --enable-coverage enables generation of code coverage metrics with gcov +# + +pgac_args="$pgac_args enable_coverage" + +# Check whether --enable-coverage was given. +if test "${enable_coverage+set}" = set; then + enableval=$enable_coverage; + case $enableval in + yes) + : + ;; + no) + : + ;; + *) + { { echo "$as_me:$LINENO: error: no argument expected for --enable-coverage option" >&5 +echo "$as_me: error: no argument expected for --enable-coverage option" >&2;} + { (exit 1); exit 1; }; } + ;; + esac + +else + enable_coverage=no + +fi + + +for ac_prog in gcov +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6; } +if test "${ac_cv_prog_GCOV+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$GCOV"; then + ac_cv_prog_GCOV="$GCOV" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_GCOV="$ac_prog" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +GCOV=$ac_cv_prog_GCOV +if test -n "$GCOV"; then + { echo "$as_me:$LINENO: result: $GCOV" >&5 +echo "${ECHO_T}$GCOV" >&6; } +else + { echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6; } +fi + + + test -n "$GCOV" && break +done + +if test -z "$GCOV"; then + { { echo "$as_me:$LINENO: error: gcov not found" >&5 +echo "$as_me: error: gcov not found" >&2;} + { (exit 1); exit 1; }; } +fi +for ac_prog in lcov +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6; } +if test "${ac_cv_prog_LCOV+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$LCOV"; then + ac_cv_prog_LCOV="$LCOV" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_LCOV="$ac_prog" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +LCOV=$ac_cv_prog_LCOV +if test -n "$LCOV"; then + { echo "$as_me:$LINENO: result: $LCOV" >&5 +echo "${ECHO_T}$LCOV" >&6; } +else + { echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6; } +fi + + + test -n "$LCOV" && break +done + +if test -z "$LCOV"; then + { { echo "$as_me:$LINENO: error: lcov not found" >&5 +echo "$as_me: error: lcov not found" >&2;} + { (exit 1); exit 1; }; } +fi +for ac_prog in genhtml +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ echo "$as_me:$LINENO: checking for $ac_word" >&5 +echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6; } +if test "${ac_cv_prog_GENHTML+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + if test -n "$GENHTML"; then + ac_cv_prog_GENHTML="$GENHTML" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_GENHTML="$ac_prog" + echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done +done +IFS=$as_save_IFS + +fi +fi +GENHTML=$ac_cv_prog_GENHTML +if test -n "$GENHTML"; then + { echo "$as_me:$LINENO: result: $GENHTML" >&5 +echo "${ECHO_T}$GENHTML" >&6; } +else + { echo "$as_me:$LINENO: result: no" >&5 +echo "${ECHO_T}no" >&6; } +fi + + + test -n "$GENHTML" && break +done + +if test -z "$GENHTML"; then + { { echo "$as_me:$LINENO: error: genhtml not found" >&5 +echo "$as_me: error: genhtml not found" >&2;} + { (exit 1); exit 1; }; } +fi + + # # DTrace # @@ -3580,13 +3757,16 @@ unset CFLAGS # CFLAGS are selected so: # If the user specifies something in the environment, that is used. # else: If the template file set something, that is used. +# else: If coverage was enabled, don't set anything. # else: If the compiler is GCC, then we use -O2. -# else: If the compiler is something else, then we use -0. +# else: If the compiler is something else, then we use -O. if test "$ac_env_CFLAGS_set" = set; then CFLAGS=$ac_env_CFLAGS_value elif test "${CFLAGS+set}" = set; then : # (keep what template set) +elif test "$enable_coverage" = yes; then + : # no optimization by default elif test "$GCC" = yes; then CFLAGS="-O2" else @@ -3961,6 +4141,17 @@ if test "$enable_debug" = yes && test "$ac_cv_prog_cc_g" = yes; then CFLAGS="$CFLAGS -g" fi +# enable code coverage if --enable-coverage +if test "$enable_coverage" = yes; then + if test "$GCC" = yes; then + CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage" + else + { { echo "$as_me:$LINENO: error: --enable-coverage is supported only when using GCC" >&5 +echo "$as_me: error: --enable-coverage is supported only when using GCC" >&2;} + { (exit 1); exit 1; }; } + fi +fi + # enable profiling if --enable-profiling if test "$enable_profiling" = yes && test "$ac_cv_prog_cc_g" = yes; then if test "$GCC" = yes; then @@ -26510,6 +26701,10 @@ enable_shared!$enable_shared$ac_delim enable_rpath!$enable_rpath$ac_delim enable_debug!$enable_debug$ac_delim enable_profiling!$enable_profiling$ac_delim +GCOV!$GCOV$ac_delim +LCOV!$LCOV$ac_delim +GENHTML!$GENHTML$ac_delim +enable_coverage!$enable_coverage$ac_delim DTRACE!$DTRACE$ac_delim DTRACEFLAGS!$DTRACEFLAGS$ac_delim enable_dtrace!$enable_dtrace$ac_delim @@ -26549,10 +26744,6 @@ LDFLAGS_SL!$LDFLAGS_SL$ac_delim LD!$LD$ac_delim with_gnu_ld!$with_gnu_ld$ac_delim ld_R_works!$ld_R_works$ac_delim -RANLIB!$RANLIB$ac_delim -STRIP!$STRIP$ac_delim -STRIP_STATIC_LIB!$STRIP_STATIC_LIB$ac_delim -STRIP_SHARED_LIB!$STRIP_SHARED_LIB$ac_delim _ACEOF if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 97; then @@ -26594,6 +26785,10 @@ _ACEOF ac_delim='%!_!# ' for ac_last_try in false false false false false :; do cat >conf$$subs.sed <<_ACEOF +RANLIB!$RANLIB$ac_delim +STRIP!$STRIP$ac_delim +STRIP_STATIC_LIB!$STRIP_STATIC_LIB$ac_delim +STRIP_SHARED_LIB!$STRIP_SHARED_LIB$ac_delim TAR!$TAR$ac_delim LN_S!$LN_S$ac_delim AWK!$AWK$ac_delim @@ -26644,7 +26839,7 @@ vpath_build!$vpath_build$ac_delim LTLIBOBJS!$LTLIBOBJS$ac_delim _ACEOF - if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 48; then + if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 52; then break elif $ac_last_try; then { { echo "$as_me:$LINENO: error: could not make $CONFIG_STATUS" >&5 diff --git a/configure.in b/configure.in index e351bd1c85..97c220c5c1 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ dnl Process this file with autoconf to produce a configure script. -dnl $PostgreSQL: pgsql/configure.in,v 1.565 2008/08/29 13:02:32 petere Exp $ +dnl $PostgreSQL: pgsql/configure.in,v 1.566 2008/09/05 12:11:18 petere Exp $ dnl dnl Developers, please strive to achieve this order: dnl @@ -203,6 +203,25 @@ PGAC_ARG_BOOL(enable, profiling, no, [ --enable-profiling build with profiling enabled ]) AC_SUBST(enable_profiling) +# +# --enable-coverage enables generation of code coverage metrics with gcov +# +PGAC_ARG_BOOL(enable, coverage, no, + [ --enable-coverage build with coverage testing instrumentation]) +AC_CHECK_PROGS(GCOV, gcov) +if test -z "$GCOV"; then + AC_MSG_ERROR([gcov not found]) +fi +AC_CHECK_PROGS(LCOV, lcov) +if test -z "$LCOV"; then + AC_MSG_ERROR([lcov not found]) +fi +AC_CHECK_PROGS(GENHTML, genhtml) +if test -z "$GENHTML"; then + AC_MSG_ERROR([genhtml not found]) +fi +AC_SUBST(enable_coverage) + # # DTrace # @@ -370,13 +389,16 @@ unset CFLAGS # CFLAGS are selected so: # If the user specifies something in the environment, that is used. # else: If the template file set something, that is used. +# else: If coverage was enabled, don't set anything. # else: If the compiler is GCC, then we use -O2. -# else: If the compiler is something else, then we use -0. +# else: If the compiler is something else, then we use -O. if test "$ac_env_CFLAGS_set" = set; then CFLAGS=$ac_env_CFLAGS_value elif test "${CFLAGS+set}" = set; then : # (keep what template set) +elif test "$enable_coverage" = yes; then + : # no optimization by default elif test "$GCC" = yes; then CFLAGS="-O2" else @@ -415,6 +437,15 @@ if test "$enable_debug" = yes && test "$ac_cv_prog_cc_g" = yes; then CFLAGS="$CFLAGS -g" fi +# enable code coverage if --enable-coverage +if test "$enable_coverage" = yes; then + if test "$GCC" = yes; then + CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage" + else + AC_MSG_ERROR([--enable-coverage is supported only when using GCC]) + fi +fi + # enable profiling if --enable-profiling if test "$enable_profiling" = yes && test "$ac_cv_prog_cc_g" = yes; then if test "$GCC" = yes; then diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index a5dfa8d081..cee4681ce3 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -1,4 +1,4 @@ - + <![%standalone-include[<productname>PostgreSQL</>]]> @@ -1233,6 +1233,21 @@ su - postgres </listitem> </varlistentry> + <varlistentry> + <term><option>--enable-coverage</option></term> + <listitem> + <para> + If using GCC, all programs and libraries are compiled with + code coverage testing instrumentation. When run, they + generate files in the build directory with code coverage + metrics. + <![%standalone-ignore[See <xref linkend="regress-coverage"> + for more information.]]> This option is for use only with GCC + and when doing development work. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><option>--enable-profiling</option></term> <listitem> diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgml index d09137fcc0..0e082603bf 100644 --- a/doc/src/sgml/regress.sgml +++ b/doc/src/sgml/regress.sgml @@ -1,4 +1,4 @@ -<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.59 2008/05/30 00:04:32 tgl Exp $ --> +<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.60 2008/09/05 12:11:18 petere Exp $ --> <chapter id="regress"> <title id="regress-title">Regression Tests @@ -419,5 +419,37 @@ float8:out:i.86-.*-openbsd=float8-small-is-zero.out - + + + Test Coverage Examination + + + The PostgreSQL source code can be compiled with coverage testing + instrumentation, so that it becomes possible to examine which + parts of the code are covered by the regression tests or any other + test suite that is run with the code. This is currently supported + when compiling with GCC and requires the gcov + and lcov programs. + + + + A typical workflow would look like this: + +./configure --enable-coverage ... OTHER OPTIONS ... +gmake +gmake check # or other test suite +gmake coverage-html + + Then point your HTML browser + to coverage/index.html. + + + + To reset the execution counts between test runs, run + +gmake coverage-clean + + + + diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 6212a1467f..def667a103 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -1,5 +1,5 @@ # -*-makefile-*- -# $PostgreSQL: pgsql/src/Makefile.global.in,v 1.242 2008/08/29 13:02:32 petere Exp $ +# $PostgreSQL: pgsql/src/Makefile.global.in,v 1.243 2008/09/05 12:11:18 petere Exp $ #------------------------------------------------------------------------------ # All PostgreSQL makefiles include this file and use the variables it sets, @@ -18,7 +18,7 @@ # # Meta configuration -.PHONY: all install install-strip installdirs uninstall clean distclean maintainer-clean distprep check installcheck maintainer-check +.PHONY: all install install-strip installdirs uninstall clean distclean maintainer-clean distprep check installcheck maintainer-check coverage .SILENT: installdirs # make `all' the default target @@ -165,6 +165,7 @@ enable_rpath = @enable_rpath@ enable_nls = @enable_nls@ enable_debug = @enable_debug@ enable_dtrace = @enable_dtrace@ +enable_coverage = @enable_coverage@ enable_thread_safety = @enable_thread_safety@ python_includespec = @python_includespec@ @@ -291,6 +292,17 @@ JADE = @JADE@ NSGMLS = @NSGMLS@ SGMLSPL = @SGMLSPL@ +# Code coverage + +GCOV = @GCOV@ +LCOV = @LCOV@ +GENHTML = @GENHTML@ + +ifeq ($(enable_coverage),yes) +# ccache loses .gcno files +export CCACHE_DISABLE = 1 +endif + # Feature settings DEF_PGPORT = @default_port@ @@ -574,3 +586,39 @@ include $(top_srcdir)/src/nls-global.mk endif # nls.mk endif # enable_nls + + +########################################################################## +# +# Coverage + +ifeq ($(enable_coverage), yes) + +# There is a strange interaction between lcov and existing .gcov +# output files. Hence the rm command and the ordering dependency. + +gcda_files := $(wildcard *.gcda) + +lcov.info: + rm -f *.gcov +ifneq (,$(gcda_files)) + $(LCOV) -d . -c -o $@ $(LCOVFLAGS) +endif + +%.c.gcov: %.gcda | lcov.info + $(GCOV) -b -f -p -o . $(GCOVFLAGS) $*.c >$*.c.gcov.out + + +# hook for clean-up +clean distclean maintainer-clean: clean-coverage + +.PHONY: clean-coverage +clean-coverage: + rm -f *.gcda *.gcno lcov.info *.gcov *.gcov.out + + +# User-callable target to reset counts between test runs +coverage-clean: + rm -f `find . -name '*.gcda' -print` + +endif # enable_coverage diff --git a/src/backend/common.mk b/src/backend/common.mk index 82a9ab6fc1..c617c67276 100644 --- a/src/backend/common.mk +++ b/src/backend/common.mk @@ -1,7 +1,7 @@ # # Common make rules for backend # -# $PostgreSQL: pgsql/src/backend/common.mk,v 1.7 2008/03/17 18:24:56 petere Exp $ +# $PostgreSQL: pgsql/src/backend/common.mk,v 1.8 2008/09/05 12:11:18 petere Exp $ # # When including this file, set OBJS to the object files created in @@ -46,3 +46,9 @@ ifdef SUBDIRS for dir in $(SUBDIRS); do $(MAKE) -C $$dir clean || exit; done endif rm -f $(subsysfilename) $(OBJS) + + +coverage: $(gcda_files:.gcda=.c.gcov) lcov.info +ifdef SUBDIRS + for dir in $(SUBDIRS); do $(MAKE) -C $$dir coverage || exit; done +endif