Handle domains when checking for recursive inclusion of composite types.

We need this now because we allow domains over arrays, and we'll probably
allow domains over composites pretty soon, which makes the problem even
more obvious.

Although domains over arrays also exist in previous versions, this does not
need to be back-patched, because the coding used in older versions
successfully "looked through" domains over arrays.  The problem is exposed
by not treating a domain as having a typelem.

Problem identified by Noah Misch, though I did not use his patch, since
it would require additional work to handle domains over composites that
way.  This approach is more future-proof.

src/backend/catalog/heap.c

@@ -485,12 +485,19 @@ CheckAttributeType(const char *attname,
 					 errmsg("column \"%s\" has pseudo-type %s",
 							attname, format_type_be(atttypid))));
 	}
+	else if (att_typtype == TYPTYPE_DOMAIN)
+	{
+		/*
+		 * If it's a domain, recurse to check its base type.
+		 */
+		CheckAttributeType(attname, getBaseType(atttypid), attcollation,
+						   containing_rowtypes,
+						   allow_system_table_mods);
+	}
 	else if (att_typtype == TYPTYPE_COMPOSITE)
 	{
 		/*
-		 * For a composite type, recurse into its attributes.  You might think
-		 * this isn't necessary, but since we allow system catalogs to break
-		 * the rule, we have to guard against the case.
+		 * For a composite type, recurse into its attributes.
 		 */
 		Relation	relation;
 		TupleDesc	tupdesc;

src/test/regress/expected/alter_table.out

@@ -1516,6 +1516,9 @@ alter table recur1 add column f2 recur1; -- fails
 ERROR:  composite type recur1 cannot be made a member of itself
 alter table recur1 add column f2 recur1[]; -- fails
 ERROR:  composite type recur1 cannot be made a member of itself
+create domain array_of_recur1 as recur1[];
+alter table recur1 add column f2 array_of_recur1; -- fails
+ERROR:  composite type recur1 cannot be made a member of itself
 create temp table recur2 (f1 int, f2 recur1);
 alter table recur1 add column f2 recur2; -- fails
 ERROR:  composite type recur1 cannot be made a member of itself

src/test/regress/sql/alter_table.sql

@@ -1128,6 +1128,8 @@ alter table tab1 alter column b type varchar; -- fails
 create temp table recur1 (f1 int);
 alter table recur1 add column f2 recur1; -- fails
 alter table recur1 add column f2 recur1[]; -- fails
+create domain array_of_recur1 as recur1[];
+alter table recur1 add column f2 array_of_recur1; -- fails
 create temp table recur2 (f1 int, f2 recur1);
 alter table recur1 add column f2 recur2; -- fails
 alter table recur1 add column f2 int;