From 4c9864b9b4d87d02f07f40bb27976da737afdcab Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 11 Feb 2016 08:38:09 -0500 Subject: [PATCH] Add some isolation tests for deadlock detection and resolution. Previously, we had no test coverage for the deadlock detector. --- src/test/isolation/expected/deadlock-hard.out | 35 ++++++++++ .../isolation/expected/deadlock-simple.out | 11 +++ .../isolation/expected/deadlock-soft-2.out | 17 +++++ src/test/isolation/expected/deadlock-soft.out | 17 +++++ src/test/isolation/isolation_schedule | 4 ++ src/test/isolation/specs/deadlock-hard.spec | 70 +++++++++++++++++++ src/test/isolation/specs/deadlock-simple.spec | 29 ++++++++ src/test/isolation/specs/deadlock-soft-2.spec | 38 ++++++++++ src/test/isolation/specs/deadlock-soft.spec | 40 +++++++++++ 9 files changed, 261 insertions(+) create mode 100644 src/test/isolation/expected/deadlock-hard.out create mode 100644 src/test/isolation/expected/deadlock-simple.out create mode 100644 src/test/isolation/expected/deadlock-soft-2.out create mode 100644 src/test/isolation/expected/deadlock-soft.out create mode 100644 src/test/isolation/specs/deadlock-hard.spec create mode 100644 src/test/isolation/specs/deadlock-simple.spec create mode 100644 src/test/isolation/specs/deadlock-soft-2.spec create mode 100644 src/test/isolation/specs/deadlock-soft.spec diff --git a/src/test/isolation/expected/deadlock-hard.out b/src/test/isolation/expected/deadlock-hard.out new file mode 100644 index 0000000000..cb25f75d28 --- /dev/null +++ b/src/test/isolation/expected/deadlock-hard.out @@ -0,0 +1,35 @@ +Parsed test spec with 8 sessions + +starting permutation: s1a1 s2a2 s3a3 s4a4 s5a5 s6a6 s7a7 s8a8 s1a2 s2a3 s3a4 s4a5 s5a6 s6a7 s7a8 s8a1 s8c s7c s6c s5c s4c s3c s2c s1c +step s1a1: LOCK TABLE a1; +step s2a2: LOCK TABLE a2; +step s3a3: LOCK TABLE a3; +step s4a4: LOCK TABLE a4; +step s5a5: LOCK TABLE a5; +step s6a6: LOCK TABLE a6; +step s7a7: LOCK TABLE a7; +step s8a8: LOCK TABLE a8; +step s1a2: LOCK TABLE a2; +step s2a3: LOCK TABLE a3; +step s3a4: LOCK TABLE a4; +step s4a5: LOCK TABLE a5; +step s5a6: LOCK TABLE a6; +step s6a7: LOCK TABLE a7; +step s7a8: LOCK TABLE a8; +step s8a1: LOCK TABLE a1; +step s7a8: <... completed> +error in steps s8a1 s7a8: ERROR: deadlock detected +step s8c: COMMIT; +step s7c: COMMIT; +step s6a7: <... completed> +step s6c: COMMIT; +step s5a6: <... completed> +step s5c: COMMIT; +step s4a5: <... completed> +step s4c: COMMIT; +step s3a4: <... completed> +step s3c: COMMIT; +step s2a3: <... completed> +step s2c: COMMIT; +step s1a2: <... completed> +step s1c: COMMIT; diff --git a/src/test/isolation/expected/deadlock-simple.out b/src/test/isolation/expected/deadlock-simple.out new file mode 100644 index 0000000000..e0d2c4ef12 --- /dev/null +++ b/src/test/isolation/expected/deadlock-simple.out @@ -0,0 +1,11 @@ +Parsed test spec with 2 sessions + +starting permutation: s1as s2as s1ae s2ae s1c s2c +step s1as: LOCK TABLE a1 IN ACCESS SHARE MODE; +step s2as: LOCK TABLE a1 IN ACCESS SHARE MODE; +step s1ae: LOCK TABLE a1 IN ACCESS EXCLUSIVE MODE; +step s2ae: LOCK TABLE a1 IN ACCESS EXCLUSIVE MODE; +step s1ae: <... completed> +error in steps s2ae s1ae: ERROR: deadlock detected +step s1c: COMMIT; +step s2c: COMMIT; diff --git a/src/test/isolation/expected/deadlock-soft-2.out b/src/test/isolation/expected/deadlock-soft-2.out new file mode 100644 index 0000000000..14b0343ba4 --- /dev/null +++ b/src/test/isolation/expected/deadlock-soft-2.out @@ -0,0 +1,17 @@ +Parsed test spec with 4 sessions + +starting permutation: s1a s2a s2b s3a s4a s1b s1c s2c s3c s4c +step s1a: LOCK TABLE a1 IN SHARE UPDATE EXCLUSIVE MODE; +step s2a: LOCK TABLE a2 IN ACCESS SHARE MODE; +step s2b: LOCK TABLE a1 IN SHARE UPDATE EXCLUSIVE MODE; +step s3a: LOCK TABLE a2 IN ACCESS EXCLUSIVE MODE; +step s4a: LOCK TABLE a2 IN ACCESS EXCLUSIVE MODE; +step s1b: LOCK TABLE a2 IN SHARE UPDATE EXCLUSIVE MODE; +step s1b: <... completed> +step s1c: COMMIT; +step s2b: <... completed> +step s2c: COMMIT; +step s3a: <... completed> +step s3c: COMMIT; +step s4a: <... completed> +step s4c: COMMIT; diff --git a/src/test/isolation/expected/deadlock-soft.out b/src/test/isolation/expected/deadlock-soft.out new file mode 100644 index 0000000000..24a35daa7d --- /dev/null +++ b/src/test/isolation/expected/deadlock-soft.out @@ -0,0 +1,17 @@ +Parsed test spec with 4 sessions + +starting permutation: d1a1 d2a2 e1l e2l d1a2 d2a1 d1c e1c d2c e2c +step d1a1: LOCK TABLE a1 IN ACCESS SHARE MODE; +step d2a2: LOCK TABLE a2 IN ACCESS SHARE MODE; +step e1l: LOCK TABLE a1 IN ACCESS EXCLUSIVE MODE; +step e2l: LOCK TABLE a2 IN ACCESS EXCLUSIVE MODE; +step d1a2: LOCK TABLE a2 IN ACCESS SHARE MODE; +step d2a1: LOCK TABLE a1 IN ACCESS SHARE MODE; +step d1a2: <... completed> +step d1c: COMMIT; +step e1l: <... completed> +step e1c: COMMIT; +step d2a1: <... completed> +step d2c: COMMIT; +step e2l: <... completed> +step e2c: COMMIT; diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 09dc9d4730..fcde38ce45 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -10,6 +10,10 @@ test: partial-index test: two-ids test: multiple-row-versions test: index-only-scan +test: deadlock-simple +test: deadlock-hard +test: deadlock-soft +test: deadlock-soft-2 test: fk-contention test: fk-deadlock test: fk-deadlock2 diff --git a/src/test/isolation/specs/deadlock-hard.spec b/src/test/isolation/specs/deadlock-hard.spec new file mode 100644 index 0000000000..a6daba374a --- /dev/null +++ b/src/test/isolation/specs/deadlock-hard.spec @@ -0,0 +1,70 @@ +# This is a straightforward deadlock scenario. Since it involves more than +# two processes, the mainlock detector will find the problem and rollback the +# backend which discovers it. + +setup +{ + CREATE TABLE a1 (); + CREATE TABLE a2 (); + CREATE TABLE a3 (); + CREATE TABLE a4 (); + CREATE TABLE a5 (); + CREATE TABLE a6 (); + CREATE TABLE a7 (); + CREATE TABLE a8 (); +} + +teardown +{ + DROP TABLE a1, a2, a3, a4, a5, a6, a7, a8; +} + +session "s1" +setup { BEGIN; SET deadlock_timeout = '10s'; } +step "s1a1" { LOCK TABLE a1; } +step "s1a2" { LOCK TABLE a2; } +step "s1c" { COMMIT; } + +session "s2" +setup { BEGIN; SET deadlock_timeout = '10s'; } +step "s2a2" { LOCK TABLE a2; } +step "s2a3" { LOCK TABLE a3; } +step "s2c" { COMMIT; } + +session "s3" +setup { BEGIN; SET deadlock_timeout = '10s'; } +step "s3a3" { LOCK TABLE a3; } +step "s3a4" { LOCK TABLE a4; } +step "s3c" { COMMIT; } + +session "s4" +setup { BEGIN; SET deadlock_timeout = '10s'; } +step "s4a4" { LOCK TABLE a4; } +step "s4a5" { LOCK TABLE a5; } +step "s4c" { COMMIT; } + +session "s5" +setup { BEGIN; SET deadlock_timeout = '10s'; } +step "s5a5" { LOCK TABLE a5; } +step "s5a6" { LOCK TABLE a6; } +step "s5c" { COMMIT; } + +session "s6" +setup { BEGIN; SET deadlock_timeout = '10s'; } +step "s6a6" { LOCK TABLE a6; } +step "s6a7" { LOCK TABLE a7; } +step "s6c" { COMMIT; } + +session "s7" +setup { BEGIN; SET deadlock_timeout = '10s'; } +step "s7a7" { LOCK TABLE a7; } +step "s7a8" { LOCK TABLE a8; } +step "s7c" { COMMIT; } + +session "s8" +setup { BEGIN; SET deadlock_timeout = '10ms'; } +step "s8a8" { LOCK TABLE a8; } +step "s8a1" { LOCK TABLE a1; } +step "s8c" { COMMIT; } + +permutation "s1a1" "s2a2" "s3a3" "s4a4" "s5a5" "s6a6" "s7a7" "s8a8" "s1a2" "s2a3" "s3a4" "s4a5" "s5a6" "s6a7" "s7a8" "s8a1" "s8c" "s7c" "s6c" "s5c" "s4c" "s3c" "s2c" "s1c" diff --git a/src/test/isolation/specs/deadlock-simple.spec b/src/test/isolation/specs/deadlock-simple.spec new file mode 100644 index 0000000000..b1300ca77c --- /dev/null +++ b/src/test/isolation/specs/deadlock-simple.spec @@ -0,0 +1,29 @@ +# The deadlock detector has a special case for "simple" deadlocks. A simple +# deadlock occurs when we attempt a lock upgrade while another process waits +# for a lock upgrade on the same object; and the sought locks conflict with +# those already held, so that neither process can complete its upgrade until +# the other releases locks. Test this scenario. + +setup +{ + CREATE TABLE a1 (); +} + +teardown +{ + DROP TABLE a1; +} + +session "s1" +setup { BEGIN; } +step "s1as" { LOCK TABLE a1 IN ACCESS SHARE MODE; } +step "s1ae" { LOCK TABLE a1 IN ACCESS EXCLUSIVE MODE; } +step "s1c" { COMMIT; } + +session "s2" +setup { BEGIN; } +step "s2as" { LOCK TABLE a1 IN ACCESS SHARE MODE; } +step "s2ae" { LOCK TABLE a1 IN ACCESS EXCLUSIVE MODE; } +step "s2c" { COMMIT; } + +permutation "s1as" "s2as" "s1ae" "s2ae" "s1c" "s2c" diff --git a/src/test/isolation/specs/deadlock-soft-2.spec b/src/test/isolation/specs/deadlock-soft-2.spec new file mode 100644 index 0000000000..044921c05d --- /dev/null +++ b/src/test/isolation/specs/deadlock-soft-2.spec @@ -0,0 +1,38 @@ +# Soft deadlock requiring reversal of multiple wait-edges. s1 must +# jump over both s3 and s4 and acquire the lock on a2 immediately, +# since s3 and s4 are hard-blocked on a1. + +setup +{ + CREATE TABLE a1 (); + CREATE TABLE a2 (); +} + +teardown +{ + DROP TABLE a1, a2; +} + +session "s1" +setup { BEGIN; } +step "s1a" { LOCK TABLE a1 IN SHARE UPDATE EXCLUSIVE MODE; } +step "s1b" { LOCK TABLE a2 IN SHARE UPDATE EXCLUSIVE MODE; } +step "s1c" { COMMIT; } + +session "s2" +setup { BEGIN; } +step "s2a" { LOCK TABLE a2 IN ACCESS SHARE MODE; } +step "s2b" { LOCK TABLE a1 IN SHARE UPDATE EXCLUSIVE MODE; } +step "s2c" { COMMIT; } + +session "s3" +setup { BEGIN; } +step "s3a" { LOCK TABLE a2 IN ACCESS EXCLUSIVE MODE; } +step "s3c" { COMMIT; } + +session "s4" +setup { BEGIN; } +step "s4a" { LOCK TABLE a2 IN ACCESS EXCLUSIVE MODE; } +step "s4c" { COMMIT; } + +permutation "s1a" "s2a" "s2b" "s3a" "s4a" "s1b" "s1c" "s2c" "s3c" "s4c" diff --git a/src/test/isolation/specs/deadlock-soft.spec b/src/test/isolation/specs/deadlock-soft.spec new file mode 100644 index 0000000000..49d16e027c --- /dev/null +++ b/src/test/isolation/specs/deadlock-soft.spec @@ -0,0 +1,40 @@ +# Four-process deadlock with two hard edges and two soft edges. +# d2 waits for e1 (soft edge), e1 waits for d1 (hard edge), +# d1 waits for e2 (soft edge), e2 waits for d2 (hard edge). +# The deadlock detector resolves the deadlock by reversing the d1-e2 edge, +# unblocking d1. + +setup +{ + CREATE TABLE a1 (); + CREATE TABLE a2 (); +} + +teardown +{ + DROP TABLE a1, a2; +} + +session "d1" +setup { BEGIN; SET deadlock_timeout = '10s'; } +step "d1a1" { LOCK TABLE a1 IN ACCESS SHARE MODE; } +step "d1a2" { LOCK TABLE a2 IN ACCESS SHARE MODE; } +step "d1c" { COMMIT; } + +session "d2" +setup { BEGIN; SET deadlock_timeout = '10ms'; } +step "d2a2" { LOCK TABLE a2 IN ACCESS SHARE MODE; } +step "d2a1" { LOCK TABLE a1 IN ACCESS SHARE MODE; } +step "d2c" { COMMIT; } + +session "e1" +setup { BEGIN; SET deadlock_timeout = '10s'; } +step "e1l" { LOCK TABLE a1 IN ACCESS EXCLUSIVE MODE; } +step "e1c" { COMMIT; } + +session "e2" +setup { BEGIN; SET deadlock_timeout = '10s'; } +step "e2l" { LOCK TABLE a2 IN ACCESS EXCLUSIVE MODE; } +step "e2c" { COMMIT; } + +permutation "d1a1" "d2a2" "e1l" "e2l" "d1a2" "d2a1" "d1c" "e1c" "d2c" "e2c"