compiler_builtins/
probestack.rs1#![cfg(not(feature = "mangled-names"))]
45#![cfg(not(any(windows, target_os = "cygwin")))]
47#![cfg(not(feature = "no-asm"))]
49#![cfg(any(target_arch = "x86_64", target_arch = "x86"))]
51
52unsafe extern "C" {
55 pub fn __rust_probestack();
56}
57
58#[cfg(not(any(target_vendor = "apple", target_os = "uefi")))]
64macro_rules! define_rust_probestack {
65 ($body: expr) => {
66 concat!(
67 "
68 .pushsection .text.__rust_probestack
69 .globl __rust_probestack
70 .type __rust_probestack, @function
71 .hidden __rust_probestack
72 __rust_probestack:
73 ",
74 $body,
75 "
76 .size __rust_probestack, . - __rust_probestack
77 .popsection
78 "
79 )
80 };
81}
82
83#[cfg(all(target_os = "uefi", target_arch = "x86_64"))]
84macro_rules! define_rust_probestack {
85 ($body: expr) => {
86 concat!(
87 "
88 .globl __rust_probestack
89 __rust_probestack:
90 ",
91 $body
92 )
93 };
94}
95
96#[cfg(target_vendor = "apple")]
99macro_rules! define_rust_probestack {
100 ($body: expr) => {
101 concat!(
102 "
103 .globl ___rust_probestack
104 ___rust_probestack:
105 ",
106 $body
107 )
108 };
109}
110
111#[cfg(all(target_os = "uefi", target_arch = "x86"))]
113macro_rules! define_rust_probestack {
114 ($body: expr) => {
115 concat!(
116 "
117 .globl ___rust_probestack
118 ___rust_probestack:
119 ",
120 $body
121 )
122 };
123}
124
125#[cfg(all(
133 target_arch = "x86_64",
134 not(all(target_env = "sgx", target_vendor = "fortanix"))
135))]
136core::arch::global_asm!(
137 define_rust_probestack!(
138 "
139 .cfi_startproc
140 pushq %rbp
141 .cfi_adjust_cfa_offset 8
142 .cfi_offset %rbp, -16
143 movq %rsp, %rbp
144 .cfi_def_cfa_register %rbp
145
146 mov %rax,%r11 // duplicate %rax as we're clobbering %r11
147
148 // Main loop, taken in one page increments. We're decrementing rsp by
149 // a page each time until there's less than a page remaining. We're
150 // guaranteed that this function isn't called unless there's more than a
151 // page needed.
152 //
153 // Note that we're also testing against `8(%rsp)` to account for the 8
154 // bytes pushed on the stack orginally with our return address. Using
155 // `8(%rsp)` simulates us testing the stack pointer in the caller's
156 // context.
157
158 // It's usually called when %rax >= 0x1000, but that's not always true.
159 // Dynamic stack allocation, which is needed to implement unsized
160 // rvalues, triggers stackprobe even if %rax < 0x1000.
161 // Thus we have to check %r11 first to avoid segfault.
162 cmp $0x1000,%r11
163 jna 3f
1642:
165 sub $0x1000,%rsp
166 test %rsp,8(%rsp)
167 sub $0x1000,%r11
168 cmp $0x1000,%r11
169 ja 2b
170
1713:
172 // Finish up the last remaining stack space requested, getting the last
173 // bits out of r11
174 sub %r11,%rsp
175 test %rsp,8(%rsp)
176
177 // Restore the stack pointer to what it previously was when entering
178 // this function. The caller will readjust the stack pointer after we
179 // return.
180 add %rax,%rsp
181
182 leave
183 .cfi_def_cfa_register %rsp
184 .cfi_adjust_cfa_offset -8
185 ret
186 .cfi_endproc
187 "
188 ),
189 options(att_syntax)
190);
191
192#[cfg(all(
197 target_arch = "x86_64",
198 all(target_env = "sgx", target_vendor = "fortanix")
199))]
200core::arch::global_asm!(
201 define_rust_probestack!(
202 "
203 .cfi_startproc
204 pushq %rbp
205 .cfi_adjust_cfa_offset 8
206 .cfi_offset %rbp, -16
207 movq %rsp, %rbp
208 .cfi_def_cfa_register %rbp
209
210 mov %rax,%r11 // duplicate %rax as we're clobbering %r11
211
212 // Main loop, taken in one page increments. We're decrementing rsp by
213 // a page each time until there's less than a page remaining. We're
214 // guaranteed that this function isn't called unless there's more than a
215 // page needed.
216 //
217 // Note that we're also testing against `8(%rsp)` to account for the 8
218 // bytes pushed on the stack orginally with our return address. Using
219 // `8(%rsp)` simulates us testing the stack pointer in the caller's
220 // context.
221
222 // It's usually called when %rax >= 0x1000, but that's not always true.
223 // Dynamic stack allocation, which is needed to implement unsized
224 // rvalues, triggers stackprobe even if %rax < 0x1000.
225 // Thus we have to check %r11 first to avoid segfault.
226 cmp $0x1000,%r11
227 jna 3f
2282:
229 sub $0x1000,%rsp
230 test %rsp,8(%rsp)
231 sub $0x1000,%r11
232 cmp $0x1000,%r11
233 ja 2b
234
2353:
236 // Finish up the last remaining stack space requested, getting the last
237 // bits out of r11
238 sub %r11,%rsp
239 test %rsp,8(%rsp)
240
241 // Restore the stack pointer to what it previously was when entering
242 // this function. The caller will readjust the stack pointer after we
243 // return.
244 add %rax,%rsp
245
246 leave
247 .cfi_def_cfa_register %rsp
248 .cfi_adjust_cfa_offset -8
249 pop %r11
250 lfence
251 jmp *%r11
252 .cfi_endproc
253 "
254 ),
255 options(att_syntax)
256);
257
258#[cfg(all(target_arch = "x86", not(target_os = "uefi")))]
259core::arch::global_asm!(
265 define_rust_probestack!(
266 "
267 .cfi_startproc
268 push %ebp
269 .cfi_adjust_cfa_offset 4
270 .cfi_offset %ebp, -8
271 mov %esp, %ebp
272 .cfi_def_cfa_register %ebp
273 push %ecx
274 mov %eax,%ecx
275
276 cmp $0x1000,%ecx
277 jna 3f
2782:
279 sub $0x1000,%esp
280 test %esp,8(%esp)
281 sub $0x1000,%ecx
282 cmp $0x1000,%ecx
283 ja 2b
284
2853:
286 sub %ecx,%esp
287 test %esp,8(%esp)
288
289 add %eax,%esp
290 pop %ecx
291 leave
292 .cfi_def_cfa_register %esp
293 .cfi_adjust_cfa_offset -4
294 ret
295 .cfi_endproc
296 "
297 ),
298 options(att_syntax)
299);
300
301#[cfg(all(target_arch = "x86", target_os = "uefi"))]
302core::arch::global_asm!(
313 define_rust_probestack!(
314 "
315 .cfi_startproc
316 push %ebp
317 .cfi_adjust_cfa_offset 4
318 .cfi_offset %ebp, -8
319 mov %esp, %ebp
320 .cfi_def_cfa_register %ebp
321 push %ecx
322 push %edx
323 mov %eax,%ecx
324
325 cmp $0x1000,%ecx
326 jna 3f
3272:
328 sub $0x1000,%esp
329 test %esp,8(%esp)
330 sub $0x1000,%ecx
331 cmp $0x1000,%ecx
332 ja 2b
333
3343:
335 sub %ecx,%esp
336 test %esp,8(%esp)
337 mov 4(%ebp),%edx
338 mov %edx, 12(%esp)
339 add %eax,%esp
340 pop %edx
341 pop %ecx
342 leave
343
344 sub %eax, %esp
345 .cfi_def_cfa_register %esp
346 .cfi_adjust_cfa_offset -4
347 ret
348 .cfi_endproc
349 "
350 ),
351 options(att_syntax)
352);