From e1dfd37f75a5af4262df229bfd9067af0aa8e418 Mon Sep 17 00:00:00 2001
From: Matthew Flatt <mflatt@racket-lang.org>
Date: Sat, 8 Oct 2011 16:39:00 -0600
Subject: [PATCH] fix compiler bug that could cause infinite inlining loop

The bug was that a procedure could be incorrectly marked as
a "leaf" procedure, which could in turn cause the compiler
to keep inlining a very small procedure that calls itself.

Closes PR 12270

Merge to 5.2
(cherry picked from commit 1bc80310e3ae54963568238a0595935b2f558708)
---
 collects/tests/racket/optimize.rktl | 10 ++++++++++
 src/racket/src/optimize.c           | 20 +++++++++++++-------
 2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/collects/tests/racket/optimize.rktl b/collects/tests/racket/optimize.rktl
index 4cfebc7c41..b7bf2a4028 100644
--- a/collects/tests/racket/optimize.rktl
+++ b/collects/tests/racket/optimize.rktl
@@ -1654,6 +1654,16 @@
           (begin (f) #f))
         #f))
 
+;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Make sure the compiler doesn't end up in an infinite inling loop:
+
+(module unc-small-self-call racket/base
+  (define unc1
+    (let ([x 1])
+      (lambda ()
+        (unc1))))
+  (unc1))
+
 ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (report-errs)
diff --git a/src/racket/src/optimize.c b/src/racket/src/optimize.c
index 86eaacfcff..f46924be71 100644
--- a/src/racket/src/optimize.c
+++ b/src/racket/src/optimize.c
@@ -914,9 +914,9 @@ Scheme_Object *optimize_for_inline(Optimize_Info *info, Scheme_Object *le, int a
   int offset = 0, single_use = 0, psize = 0;
   Scheme_Object *bad_app = NULL, *prev = NULL, *orig_le = le;
   intptr_t prev_offset = 0;
-  int nested_count = 0, outside_nested = 0, already_opt = optimized_rator;
+  int nested_count = 0, outside_nested = 0, already_opt = optimized_rator, nonleaf;
 
-  if (info->inline_fuel < 0)
+  if ((info->inline_fuel < 0) && info->has_nonleaf)
     return NULL;
 
   /* Move inside `let' bindings, so we can convert ((let (....) proc) arg ...)
@@ -996,7 +996,9 @@ Scheme_Object *optimize_for_inline(Optimize_Info *info, Scheme_Object *le, int a
       bad_app = le;
   }
 
-  if (le && SAME_TYPE(SCHEME_TYPE(le), scheme_compiled_unclosed_procedure_type)) {
+  nonleaf = 1;
+
+  if (le && SAME_TYPE(SCHEME_TYPE(le), scheme_compiled_unclosed_procedure_type) && (info->inline_fuel >= 0)) {
     Scheme_Closure_Data *data = (Scheme_Closure_Data *)le;
     int sz;
 
@@ -1048,25 +1050,28 @@ Scheme_Object *optimize_for_inline(Optimize_Info *info, Scheme_Object *le, int a
           return le;
 	} else {
           LOG_INLINE(fprintf(stderr, "No inline %s\n", scheme_write_to_string(data->name ? data->name : scheme_false, NULL)));
-          info->has_nonleaf = 1;
         }
       } else {
         LOG_INLINE(fprintf(stderr, "No fuel %s %d[%d]>%d@%d %d\n", scheme_write_to_string(data->name ? data->name : scheme_false, NULL),
                            sz, is_leaf, threshold,
                            info->inline_fuel, info->use_psize));
-        info->has_nonleaf = 1;
       }
     } else {
       /* Issue warning below */
       bad_app = (Scheme_Object *)data;
+      nonleaf = 0;
     }
   }
 
   if (le && SCHEME_PRIMP(le)) {
     int opt;
     opt = ((Scheme_Prim_Proc_Header *)le)->flags & SCHEME_PRIM_OPT_MASK;
-    if (opt >= SCHEME_PRIM_OPT_NONCM)
+    if (opt >= SCHEME_PRIM_OPT_NONCM) {
       *_flags = (CLOS_PRESERVES_MARKS | CLOS_SINGLE_RESULT);
+      if (opt >= SCHEME_PRIM_OPT_IMMEDIATE) {
+        nonleaf = 0;
+      }
+    }
   }
 
   if (le && SCHEME_PROCP(le) && (app || app2 || app3)) {
@@ -1074,6 +1079,7 @@ Scheme_Object *optimize_for_inline(Optimize_Info *info, Scheme_Object *le, int a
     a[0] = le;
     if (!scheme_check_proc_arity(NULL, argc, 0, 1, a))  {
       bad_app = le;
+      nonleaf = 0;
     }
   }
 
@@ -1083,7 +1089,7 @@ Scheme_Object *optimize_for_inline(Optimize_Info *info, Scheme_Object *le, int a
       info->psize += psize;
   }
 
-  if (!le)
+  if (nonleaf)
     info->has_nonleaf = 1;
 
   if (bad_app) {