From dc55c06aa30a09f54e96e4049659fb1e3b85c486 Mon Sep 17 00:00:00 2001
From: Marijn Haverbeke <marijnh@gmail.com>
Date: Fri, 27 Jan 2012 16:01:04 +0100
Subject: [PATCH] Properly check enum types for recursion

Closes #742
---
 src/comp/middle/ty.rs                   |  1 +
 src/comp/middle/typeck.rs               | 25 +++++++++++++++++++------
 src/test/compile-fail/recursive-enum.rs |  5 +++++
 3 files changed, 25 insertions(+), 6 deletions(-)
 create mode 100644 src/test/compile-fail/recursive-enum.rs

diff --git a/src/comp/middle/ty.rs b/src/comp/middle/ty.rs
index 160bb832f13..944614ad03e 100644
--- a/src/comp/middle/ty.rs
+++ b/src/comp/middle/ty.rs
@@ -172,6 +172,7 @@ export type_is_str;
 export type_is_unique;
 export type_is_enum;
 export type_is_c_like_enum;
+export type_structurally_contains;
 export type_structurally_contains_uniques;
 export type_autoderef;
 export type_param;
diff --git a/src/comp/middle/typeck.rs b/src/comp/middle/typeck.rs
index 1d54ec2befc..c1079466e73 100644
--- a/src/comp/middle/typeck.rs
+++ b/src/comp/middle/typeck.rs
@@ -259,10 +259,9 @@ fn ast_ty_to_ty(tcx: ty::ctxt, mode: mode, &&ast_ty: @ast::ty) -> ty::t {
     alt tcx.ast_ty_to_ty_cache.find(ast_ty) {
       some(some(ty)) { ret ty; }
       some(none) {
-        tcx.sess.span_fatal(ast_ty.span,
-                            "illegal recursive type \
-                              insert a enum in the cycle, \
-                              if this is desired)");
+        tcx.sess.span_fatal(ast_ty.span, "illegal recursive type. \
+                                          insert a enum in the cycle, \
+                                          if this is desired)");
       }
       none { }
     } /* go on */
@@ -2490,7 +2489,7 @@ fn check_const(ccx: @crate_ctxt, _sp: span, e: @ast::expr, id: ast::node_id) {
     demand::simple(fcx, e.span, declty, cty);
 }
 
-fn check_enum_variants(ccx: @crate_ctxt, _sp: span, vs: [ast::variant],
+fn check_enum_variants(ccx: @crate_ctxt, sp: span, vs: [ast::variant],
                       id: ast::node_id) {
     // FIXME: this is kinda a kludge; we manufacture a fake function context
     // and statement context for checking the initializer expression.
@@ -2512,7 +2511,7 @@ fn check_enum_variants(ccx: @crate_ctxt, _sp: span, vs: [ast::variant],
           some(e) {
             check_expr(fcx, e);
             let cty = expr_ty(fcx.ccx.tcx, e);
-            let declty =ty::mk_int(fcx.ccx.tcx);
+            let declty = ty::mk_int(fcx.ccx.tcx);
             demand::simple(fcx, e.span, declty, cty);
             // FIXME: issue #1417
             // Also, check_expr (from check_const pass) doesn't guarantee that
@@ -2537,6 +2536,20 @@ fn check_enum_variants(ccx: @crate_ctxt, _sp: span, vs: [ast::variant],
         disr_vals += [disr_val];
         disr_val += 1;
     }
+    let outer = true, did = local_def(id);
+    if ty::type_structurally_contains(ccx.tcx, rty, {|sty|
+        alt sty {
+          ty::ty_enum(id, _) if id == did {
+            if outer { outer = false; false }
+            else { true }
+          }
+          _ { false }
+        }
+    }) {
+        ccx.tcx.sess.span_fatal(sp, "illegal recursive enum type. \
+                                     wrap the inner value in a box to \
+                                     make it represenable");
+    }
 }
 
 // A generic function for checking the pred in a check
diff --git a/src/test/compile-fail/recursive-enum.rs b/src/test/compile-fail/recursive-enum.rs
new file mode 100644
index 00000000000..b3764036b99
--- /dev/null
+++ b/src/test/compile-fail/recursive-enum.rs
@@ -0,0 +1,5 @@
+// error-pattern: illegal recursive enum type
+
+enum list<T> { cons(T, list<T>), nil }
+
+fn main() {}