1// Copyright 2009 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package base
6
7import (
8	"fmt"
9	"os"
10	"runtime"
11	"runtime/debug"
12	"runtime/metrics"
13)
14
15var atExitFuncs []func()
16
17func AtExit(f func()) {
18	atExitFuncs = append(atExitFuncs, f)
19}
20
21func Exit(code int) {
22	for i := len(atExitFuncs) - 1; i >= 0; i-- {
23		f := atExitFuncs[i]
24		atExitFuncs = atExitFuncs[:i]
25		f()
26	}
27	os.Exit(code)
28}
29
30// To enable tracing support (-t flag), set EnableTrace to true.
31const EnableTrace = false
32
33// forEachGC calls fn each GC cycle until it returns false.
34func forEachGC(fn func() bool) {
35	type T [32]byte // large enough to avoid runtime's tiny object allocator
36
37	var finalizer func(*T)
38	finalizer = func(p *T) {
39		if fn() {
40			runtime.SetFinalizer(p, finalizer)
41		}
42	}
43
44	finalizer(new(T))
45}
46
47// AdjustStartingHeap modifies GOGC so that GC should not occur until the heap
48// grows to the requested size.  This is intended but not promised, though it
49// is true-mostly, depending on when the adjustment occurs and on the
50// compiler's input and behavior.  Once this size is approximately reached
51// GOGC is reset to 100; subsequent GCs may reduce the heap below the requested
52// size, but this function does not affect that.
53//
54// -d=gcadjust=1 enables logging of GOGC adjustment events.
55//
56// NOTE: If you think this code would help startup time in your own
57// application and you decide to use it, please benchmark first to see if it
58// actually works for you (it may not: the Go compiler is not typical), and
59// whatever the outcome, please leave a comment on bug #56546.  This code
60// uses supported interfaces, but depends more than we like on
61// current+observed behavior of the garbage collector, so if many people need
62// this feature, we should consider/propose a better way to accomplish it.
63func AdjustStartingHeap(requestedHeapGoal uint64) {
64	logHeapTweaks := Debug.GCAdjust == 1
65	mp := runtime.GOMAXPROCS(0)
66	gcConcurrency := Flag.LowerC
67
68	const (
69		goal   = "/gc/heap/goal:bytes"
70		count  = "/gc/cycles/total:gc-cycles"
71		allocs = "/gc/heap/allocs:bytes"
72		frees  = "/gc/heap/frees:bytes"
73	)
74
75	sample := []metrics.Sample{{Name: goal}, {Name: count}, {Name: allocs}, {Name: frees}}
76	const (
77		GOAL   = 0
78		COUNT  = 1
79		ALLOCS = 2
80		FREES  = 3
81	)
82
83	// Assumptions and observations of Go's garbage collector, as of Go 1.17-1.20:
84
85	// - the initial heap goal is 4M, by fiat.  It is possible for Go to start
86	//   with a heap as small as 512k, so this may change in the future.
87
88	// - except for the first heap goal, heap goal is a function of
89	//   observed-live at the previous GC and current GOGC.  After the first
90	//   GC, adjusting GOGC immediately updates GOGC; before the first GC,
91	//   adjusting GOGC does not modify goal (but the change takes effect after
92	//   the first GC).
93
94	// - the before/after first GC behavior is not guaranteed anywhere, it's
95	//   just behavior, and it's a bad idea to rely on it.
96
97	// - we don't know exactly when GC will run, even after we adjust GOGC; the
98	//   first GC may not have happened yet, may have already happened, or may
99	//   be currently in progress, and GCs can start for several reasons.
100
101	// - forEachGC above will run the provided function at some delay after each
102	//   GC's mark phase terminates; finalizers are run after marking as the
103	//   spans containing finalizable objects are swept, driven by GC
104	//   background activity and allocation demand.
105
106	// - "live at last GC" is not available through the current metrics
107	//    interface. Instead, live is estimated by knowing the adjusted value of
108	//    GOGC and the new heap goal following a GC (this requires knowing that
109	//    at least one GC has occurred):
110	//		  estLive = 100 * newGoal / (100 + currentGogc)
111	//    this new value of GOGC
112	//		  newGogc = 100*requestedHeapGoal/estLive - 100
113	//    will result in the desired goal. The logging code checks that the
114	//    resulting goal is correct.
115
116	// There's a small risk that the finalizer will be slow to run after a GC
117	// that expands the goal to a huge value, and that this will lead to
118	// out-of-memory.  This doesn't seem to happen; in experiments on a variety
119	// of machines with a variety of extra loads to disrupt scheduling, the
120	// worst overshoot observed was 50% past requestedHeapGoal.
121
122	metrics.Read(sample)
123	for _, s := range sample {
124		if s.Value.Kind() == metrics.KindBad {
125			// Just return, a slightly slower compilation is a tolerable outcome.
126			if logHeapTweaks {
127				fmt.Fprintf(os.Stderr, "GCAdjust: Regret unexpected KindBad for metric %s\n", s.Name)
128			}
129			return
130		}
131	}
132
133	// Tinker with GOGC to make the heap grow rapidly at first.
134	currentGoal := sample[GOAL].Value.Uint64() // Believe this will be 4MByte or less, perhaps 512k
135	myGogc := 100 * requestedHeapGoal / currentGoal
136	if myGogc <= 150 {
137		return
138	}
139
140	if logHeapTweaks {
141		sample := append([]metrics.Sample(nil), sample...) // avoid races with GC callback
142		AtExit(func() {
143			metrics.Read(sample)
144			goal := sample[GOAL].Value.Uint64()
145			count := sample[COUNT].Value.Uint64()
146			oldGogc := debug.SetGCPercent(100)
147			if oldGogc == 100 {
148				fmt.Fprintf(os.Stderr, "GCAdjust: AtExit goal %d gogc %d count %d maxprocs %d gcConcurrency %d\n",
149					goal, oldGogc, count, mp, gcConcurrency)
150			} else {
151				inUse := sample[ALLOCS].Value.Uint64() - sample[FREES].Value.Uint64()
152				overPct := 100 * (int(inUse) - int(requestedHeapGoal)) / int(requestedHeapGoal)
153				fmt.Fprintf(os.Stderr, "GCAdjust: AtExit goal %d gogc %d count %d maxprocs %d gcConcurrency %d overPct %d\n",
154					goal, oldGogc, count, mp, gcConcurrency, overPct)
155
156			}
157		})
158	}
159
160	debug.SetGCPercent(int(myGogc))
161
162	adjustFunc := func() bool {
163
164		metrics.Read(sample)
165		goal := sample[GOAL].Value.Uint64()
166		count := sample[COUNT].Value.Uint64()
167
168		if goal <= requestedHeapGoal { // Stay the course
169			if logHeapTweaks {
170				fmt.Fprintf(os.Stderr, "GCAdjust: Reuse GOGC adjust, current goal %d, count is %d, current gogc %d\n",
171					goal, count, myGogc)
172			}
173			return true
174		}
175
176		// Believe goal has been adjusted upwards, else it would be less-than-or-equal than requestedHeapGoal
177		calcLive := 100 * goal / (100 + myGogc)
178
179		if 2*calcLive < requestedHeapGoal { // calcLive can exceed requestedHeapGoal!
180			myGogc = 100*requestedHeapGoal/calcLive - 100
181
182			if myGogc > 125 {
183				// Not done growing the heap.
184				oldGogc := debug.SetGCPercent(int(myGogc))
185
186				if logHeapTweaks {
187					// Check that the new goal looks right
188					inUse := sample[ALLOCS].Value.Uint64() - sample[FREES].Value.Uint64()
189					metrics.Read(sample)
190					newGoal := sample[GOAL].Value.Uint64()
191					pctOff := 100 * (int64(newGoal) - int64(requestedHeapGoal)) / int64(requestedHeapGoal)
192					// Check that the new goal is close to requested.  3% of make.bash fails this test.  Why, TBD.
193					if pctOff < 2 {
194						fmt.Fprintf(os.Stderr, "GCAdjust: Retry GOGC adjust, current goal %d, count is %d, gogc was %d, is now %d, calcLive %d pctOff %d\n",
195							goal, count, oldGogc, myGogc, calcLive, pctOff)
196					} else {
197						// The GC is being annoying and not giving us the goal that we requested, say more to help understand when/why.
198						fmt.Fprintf(os.Stderr, "GCAdjust: Retry GOGC adjust, current goal %d, count is %d, gogc was %d, is now %d, calcLive %d pctOff %d inUse %d\n",
199							goal, count, oldGogc, myGogc, calcLive, pctOff, inUse)
200					}
201				}
202				return true
203			}
204		}
205
206		// In this case we're done boosting GOGC, set it to 100 and don't set a new finalizer.
207		oldGogc := debug.SetGCPercent(100)
208		// inUse helps estimate how late the finalizer ran; at the instant the previous GC ended,
209		// it was (in theory) equal to the previous GC's heap goal.  In a growing heap it is
210		// expected to grow to the new heap goal.
211		inUse := sample[ALLOCS].Value.Uint64() - sample[FREES].Value.Uint64()
212		overPct := 100 * (int(inUse) - int(requestedHeapGoal)) / int(requestedHeapGoal)
213		if logHeapTweaks {
214			fmt.Fprintf(os.Stderr, "GCAdjust: Reset GOGC adjust, old goal %d, count is %d, gogc was %d, calcLive %d inUse %d overPct %d\n",
215				goal, count, oldGogc, calcLive, inUse, overPct)
216		}
217		return false
218	}
219
220	forEachGC(adjustFunc)
221}
222