Move check for fsync=off so that pendingOps still gets cleared.

Commit 3eb77eba5a moved the loop and refactored it, and inadvertently
changed the effect of fsync=off so that it also skipped removing entries
from the pendingOps table. That was not intentional, and leads to an
assertion failure if you turn fsync on while the server is running and
reload the config.

Backpatch-through: 12-
Reviewed-By: Thomas Munro
Discussion: https://www.postgresql.org/message-id/3cbc7f4b-a5fa-56e9-9591-c886deb07513%40iki.fi

src/backend/storage/sync/sync.c

@@ -315,14 +315,6 @@ ProcessSyncRequests(void)
 	{
 		int			failures;
 
-		/*
-		 * If fsync is off then we don't have to bother opening the file at
-		 * all.  (We delay checking until this point so that changing fsync on
-		 * the fly behaves sensibly.)
-		 */
-		if (!enableFsync)
-			continue;
-
 		/*
 		 * If the entry is new then don't process it this time; it is new.
 		 * Note "continue" bypasses the hash-remove call at the bottom of the
@@ -334,12 +326,19 @@ ProcessSyncRequests(void)
 		/* Else assert we haven't missed it */
 		Assert((CycleCtr) (entry->cycle_ctr + 1) == sync_cycle_ctr);
 
+		/*
+		 * If fsync is off then we don't have to bother opening the file at
+		 * all.  (We delay checking until this point so that changing fsync on
+		 * the fly behaves sensibly.)
+		 */
+		if (enableFsync)
+		{
 			/*
 			 * If in checkpointer, we want to absorb pending requests every so
 			 * often to prevent overflow of the fsync request queue.  It is
 			 * unspecified whether newly-added entries will be visited by
-		 * hash_seq_search, but we don't care since we don't need to process
+			 * hash_seq_search, but we don't care since we don't need to
-		 * them anyway.
+			 * process them anyway.
 			 */
 			if (--absorb_counter <= 0)
 			{
@@ -348,14 +347,15 @@ ProcessSyncRequests(void)
 			}
 
 			/*
-		 * The fsync table could contain requests to fsync segments that have
+			 * The fsync table could contain requests to fsync segments that
-		 * been deleted (unlinked) by the time we get to them. Rather than
+			 * have been deleted (unlinked) by the time we get to them. Rather
-		 * just hoping an ENOENT (or EACCES on Windows) error can be ignored,
+			 * than just hoping an ENOENT (or EACCES on Windows) error can be
-		 * what we do on error is absorb pending requests and then retry.
+			 * ignored, what we do on error is absorb pending requests and
-		 * Since mdunlink() queues a "cancel" message before actually
+			 * then retry. Since mdunlink() queues a "cancel" message before
-		 * unlinking, the fsync request is guaranteed to be marked canceled
+			 * actually unlinking, the fsync request is guaranteed to be
-		 * after the absorb if it really was this case. DROP DATABASE likewise
+			 * marked canceled after the absorb if it really was this case.
-		 * has to tell us to forget fsync requests before it starts deletions.
+			 * DROP DATABASE likewise has to tell us to forget fsync requests
+			 * before it starts deletions.
 			 */
 			for (failures = 0; !entry->canceled; failures++)
 			{
@@ -385,9 +385,10 @@ ProcessSyncRequests(void)
 				}
 
 				/*
-			 * It is possible that the relation has been dropped or truncated
+				 * It is possible that the relation has been dropped or
-			 * since the fsync request was entered. Therefore, allow ENOENT,
+				 * truncated since the fsync request was entered. Therefore,
-			 * but only if we didn't fail already on this file.
+				 * allow ENOENT, but only if we didn't fail already on this
+				 * file.
 				 */
 				if (!FILE_POSSIBLY_DELETED(errno) || failures > 0)
 					ereport(data_sync_elevel(ERROR),
@@ -401,12 +402,13 @@ ProcessSyncRequests(void)
 									path)));
 
 				/*
-			 * Absorb incoming requests and check to see if a cancel arrived
+				 * Absorb incoming requests and check to see if a cancel
-			 * for this relation fork.
+				 * arrived for this relation fork.
 				 */
 				AbsorbSyncRequests();
 				absorb_counter = FSYNCS_PER_ABSORB; /* might as well... */
 			}					/* end retry loop */
+		}
 
 		/* We are done with this entry, remove it */
 		if (hash_search(pendingOps, &entry->tag, HASH_REMOVE, NULL) == NULL)