Fill hole in recursive type soundness. Fixes PR 11372.

collects/tests/typed-racket/unit-tests/typecheck-tests.rkt

@@ -772,6 +772,11 @@
                        (if (number? x)
                            (begin (f) (add1 x))
                            12))]
+
+        [tc-err (ann 3 (Rec a a))]
+        [tc-err (ann 3 (Rec a (U a 3)))]
+        [tc-err (ann 3 (Rec a (Rec b a)))]
+
         #;
         [tc-err (lambda: ([x : Any])
                          (if (number? (not (not x)))

collects/typed-racket/private/parse-type.rkt

@@ -212,14 +212,21 @@
          (extend-tvars (list var)
            (let ([t* (parse-type #'t)])
              ;; is t in a productive position?
-             (unless (match t*
+             (define productive
-                       [(Union: es)
+               (let loop ((ty t*))
-                        (define seq-tvar (Type-seq tvar))
+                 (match ty
-                        (not (memf (λ (e) (eq? (Type-seq e) seq-tvar)) es))]
+                  [(Union: elems) (andmap loop elems)]
-                       [_ #t]) ; it's fine
+                  [(F: _) (not (equal? ty tvar))]
+                  [(App: rator rands stx)
+                   (loop (resolve-app rator rands stx))]
+                  [(Mu: _ body) (loop body)]
+                  [(Poly: names body) (loop body)]
+                  [(PolyDots: names body) (loop body)]
+                  [else #t])))
+             (unless productive
                (tc-error/stx
                 stx
-                "Recursive types are not allowed as members of unions directly inside their definition"))
+                "Recursive types are not allowed directly inside their definition"))
              (if (memq var (fv t*))
                  (make-Mu var t*)
                  t*))))]