@@ -37,6 +37,7 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
37
37
align : Align ,
38
38
slot_size : Align ,
39
39
allow_higher_align : bool ,
40
+ force_right_adjust : bool ,
40
41
) -> ( & ' ll Value , Align ) {
41
42
let va_list_ty = bx. type_ptr ( ) ;
42
43
let va_list_addr = list. immediate ( ) ;
@@ -54,7 +55,10 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
54
55
let next = bx. inbounds_ptradd ( addr, full_direct_size) ;
55
56
bx. store ( next, va_list_addr, bx. tcx ( ) . data_layout . pointer_align . abi ) ;
56
57
57
- if size. bytes ( ) < slot_size. bytes ( ) && bx. tcx ( ) . sess . target . endian == Endian :: Big {
58
+ if size. bytes ( ) < slot_size. bytes ( )
59
+ && bx. tcx ( ) . sess . target . endian == Endian :: Big
60
+ && force_right_adjust
61
+ {
58
62
let adjusted_size = bx. cx ( ) . const_i32 ( ( slot_size. bytes ( ) - size. bytes ( ) ) as i32 ) ;
59
63
let adjusted = bx. inbounds_ptradd ( addr, adjusted_size) ;
60
64
( adjusted, addr_align)
@@ -63,14 +67,40 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
63
67
}
64
68
}
65
69
70
+ enum PassMode {
71
+ Direct ,
72
+ Indirect ,
73
+ }
74
+
75
+ enum SlotSize {
76
+ Bytes8 = 8 ,
77
+ Bytes4 = 4 ,
78
+ }
79
+
80
+ enum AllowHigherAlign {
81
+ No ,
82
+ Yes ,
83
+ }
84
+
85
+ enum ForceRightAdjust {
86
+ No ,
87
+ Yes ,
88
+ }
89
+
66
90
fn emit_ptr_va_arg < ' ll , ' tcx > (
67
91
bx : & mut Builder < ' _ , ' ll , ' tcx > ,
68
92
list : OperandRef < ' tcx , & ' ll Value > ,
69
93
target_ty : Ty < ' tcx > ,
70
- indirect : bool ,
71
- slot_size : Align ,
72
- allow_higher_align : bool ,
94
+ pass_mode : PassMode ,
95
+ slot_size : SlotSize ,
96
+ allow_higher_align : AllowHigherAlign ,
97
+ force_right_adjust : ForceRightAdjust ,
73
98
) -> & ' ll Value {
99
+ let indirect = matches ! ( pass_mode, PassMode :: Indirect ) ;
100
+ let allow_higher_align = matches ! ( allow_higher_align, AllowHigherAlign :: Yes ) ;
101
+ let force_right_adjust = matches ! ( force_right_adjust, ForceRightAdjust :: Yes ) ;
102
+ let slot_size = Align :: from_bytes ( slot_size as u64 ) . unwrap ( ) ;
103
+
74
104
let layout = bx. cx . layout_of ( target_ty) ;
75
105
let ( llty, size, align) = if indirect {
76
106
(
@@ -81,8 +111,15 @@ fn emit_ptr_va_arg<'ll, 'tcx>(
81
111
} else {
82
112
( layout. llvm_type ( bx. cx ) , layout. size , layout. align )
83
113
} ;
84
- let ( addr, addr_align) =
85
- emit_direct_ptr_va_arg ( bx, list, size, align. abi , slot_size, allow_higher_align) ;
114
+ let ( addr, addr_align) = emit_direct_ptr_va_arg (
115
+ bx,
116
+ list,
117
+ size,
118
+ align. abi ,
119
+ slot_size,
120
+ allow_higher_align,
121
+ force_right_adjust,
122
+ ) ;
86
123
if indirect {
87
124
let tmp_ret = bx. load ( llty, addr, addr_align) ;
88
125
bx. load ( bx. cx . layout_of ( target_ty) . llvm_type ( bx. cx ) , tmp_ret, align. abi )
@@ -179,8 +216,15 @@ fn emit_aapcs_va_arg<'ll, 'tcx>(
179
216
180
217
// On Stack block
181
218
bx. switch_to_block ( on_stack) ;
182
- let stack_value =
183
- emit_ptr_va_arg ( bx, list, target_ty, false , Align :: from_bytes ( 8 ) . unwrap ( ) , true ) ;
219
+ let stack_value = emit_ptr_va_arg (
220
+ bx,
221
+ list,
222
+ target_ty,
223
+ PassMode :: Direct ,
224
+ SlotSize :: Bytes8 ,
225
+ AllowHigherAlign :: Yes ,
226
+ ForceRightAdjust :: No ,
227
+ ) ;
184
228
bx. br ( end) ;
185
229
186
230
bx. switch_to_block ( end) ;
@@ -386,29 +430,67 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
386
430
// Determine the va_arg implementation to use. The LLVM va_arg instruction
387
431
// is lacking in some instances, so we should only use it as a fallback.
388
432
let target = & bx. cx . tcx . sess . target ;
389
- let arch = & bx. cx . tcx . sess . target . arch ;
433
+ let arch = & target. arch ;
434
+
390
435
match & * * arch {
391
436
// Windows x86
392
- "x86" if target. is_like_windows => {
393
- emit_ptr_va_arg ( bx, addr, target_ty, false , Align :: from_bytes ( 4 ) . unwrap ( ) , false )
394
- }
437
+ "x86" if target. is_like_windows => emit_ptr_va_arg (
438
+ bx,
439
+ addr,
440
+ target_ty,
441
+ PassMode :: Direct ,
442
+ SlotSize :: Bytes4 ,
443
+ AllowHigherAlign :: No ,
444
+ ForceRightAdjust :: No ,
445
+ ) ,
395
446
// Generic x86
396
- "x86" => emit_ptr_va_arg ( bx, addr, target_ty, false , Align :: from_bytes ( 4 ) . unwrap ( ) , true ) ,
447
+ "x86" => emit_ptr_va_arg (
448
+ bx,
449
+ addr,
450
+ target_ty,
451
+ PassMode :: Direct ,
452
+ SlotSize :: Bytes4 ,
453
+ AllowHigherAlign :: Yes ,
454
+ ForceRightAdjust :: No ,
455
+ ) ,
397
456
// Windows AArch64
398
- "aarch64" | "arm64ec" if target. is_like_windows => {
399
- emit_ptr_va_arg ( bx, addr, target_ty, false , Align :: from_bytes ( 8 ) . unwrap ( ) , false )
400
- }
457
+ "aarch64" | "arm64ec" if target. is_like_windows => emit_ptr_va_arg (
458
+ bx,
459
+ addr,
460
+ target_ty,
461
+ PassMode :: Direct ,
462
+ SlotSize :: Bytes8 ,
463
+ AllowHigherAlign :: No ,
464
+ ForceRightAdjust :: No ,
465
+ ) ,
401
466
// macOS / iOS AArch64
402
- "aarch64" if target. is_like_darwin => {
403
- emit_ptr_va_arg ( bx, addr, target_ty, false , Align :: from_bytes ( 8 ) . unwrap ( ) , true )
404
- }
467
+ "aarch64" if target. is_like_darwin => emit_ptr_va_arg (
468
+ bx,
469
+ addr,
470
+ target_ty,
471
+ PassMode :: Direct ,
472
+ SlotSize :: Bytes8 ,
473
+ AllowHigherAlign :: Yes ,
474
+ ForceRightAdjust :: No ,
475
+ ) ,
405
476
"aarch64" => emit_aapcs_va_arg ( bx, addr, target_ty) ,
406
477
"s390x" => emit_s390x_va_arg ( bx, addr, target_ty) ,
407
478
// Windows x86_64
408
479
"x86_64" if target. is_like_windows => {
409
480
let target_ty_size = bx. cx . size_of ( target_ty) . bytes ( ) ;
410
- let indirect: bool = target_ty_size > 8 || !target_ty_size. is_power_of_two ( ) ;
411
- emit_ptr_va_arg ( bx, addr, target_ty, indirect, Align :: from_bytes ( 8 ) . unwrap ( ) , false )
481
+ emit_ptr_va_arg (
482
+ bx,
483
+ addr,
484
+ target_ty,
485
+ if target_ty_size > 8 || !target_ty_size. is_power_of_two ( ) {
486
+ PassMode :: Indirect
487
+ } else {
488
+ PassMode :: Direct
489
+ } ,
490
+ SlotSize :: Bytes8 ,
491
+ AllowHigherAlign :: No ,
492
+ ForceRightAdjust :: No ,
493
+ )
412
494
}
413
495
"xtensa" => emit_xtensa_va_arg ( bx, addr, target_ty) ,
414
496
// For all other architecture/OS combinations fall back to using
0 commit comments