1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139

// Originally inspired by Tifflin OS.

use core::{
    arch::asm,
    sync::atomic::{compiler_fence, Ordering},
};

/// A guard type for withholding regular interrupts on the current CPU.
///
/// When dropped, interrupts are returned to their prior state rather than
/// just blindly re-enabled. For example, if interrupts were enabled
/// when [`hold_interrupts()`] was invoked, interrupts will be re-enabled
/// when this type is dropped.
#[derive(Default)]
pub struct HeldInterrupts(bool);

impl !Send for HeldInterrupts {}

/// Prevents regular interrupts from occurring until the returned
/// `HeldInterrupts` object is dropped.
///
/// This function only affects *regular* IRQs;
/// it does not affect NMIs or fast interrupts (FIQs on aarch64).
pub fn hold_interrupts() -> HeldInterrupts {
    let enabled = interrupts_enabled();
    let retval = HeldInterrupts(enabled);
    disable_interrupts();
    // trace!("hold_interrupts(): disabled interrupts, were {}", enabled);
    retval
}

impl Drop for HeldInterrupts {
    fn drop(&mut self) {
        // trace!("hold_interrupts(): enabling interrupts? {}", self.0);
        if self.0 {
            enable_interrupts();
        }
    }
}

/// Unconditionally enables *regular* interrupts (IRQs),
/// not NMIs or fast interrupts (FIQs on aarch64).
///
/// To enable fast interrupts (FIQs) on aarch64,
/// use the [`enable_fast_interrupts()`] interrupts.
#[inline(always)]
pub fn enable_interrupts() {
    compiler_fence(Ordering::SeqCst);
    unsafe {
        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
        asm!("sti", options(nomem, nostack));

        #[cfg(target_arch = "aarch64")]
        // Clear the I bit, which is bit 1 of the DAIF bitset.
        asm!("msr daifclr, #2", options(nomem, nostack, preserves_flags));

        #[cfg(target_arch = "arm")]
        asm!("cpsie i", options(nomem, nostack, preserves_flags));
    }
}

/// Unconditionally disables *regular* interrupts (IRQs),
/// not NMIs or fast interrupts (FIQs on aarch64).
///
/// To disable fast interrupts (FIQs) on aarch64,
/// use the [`disable_fast_interrupts()`] interrupts.
#[inline(always)]
pub fn disable_interrupts() {
    unsafe {
        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
        asm!("cli", options(nomem, nostack));

        #[cfg(target_arch = "aarch64")]
        // Set the I bit, which is bit 1 of the DAIF bitset.
        asm!("msr daifset, #2", options(nomem, nostack, preserves_flags));

        #[cfg(target_arch = "arm")]
        asm!("cpsid i", options(nomem, nostack, preserves_flags));
    }
    compiler_fence(Ordering::SeqCst);
}

/// Unconditionally enables fast interrupts (FIQs); aarch64-only.
///
/// On aarch64, NMIs are only available as a hardware extension,
/// therefore we only deal with FIQs here, which are widely supported.
#[inline(always)]
#[cfg(target_arch = "aarch64")]
pub fn enable_fast_interrupts() {
    compiler_fence(Ordering::SeqCst);
    unsafe {
        // Clear the F bit, which is bit 0 of the DAIF bitset.
        asm!("msr daifclr, #1", options(nomem, nostack, preserves_flags));
    }
}

/// Unconditionally disables fast interrupts (FIQs); aarch64-only.
///
/// On aarch64, NMIs are only available as a hardware extension,
/// therefore we only deal with FIQs here, which are widely supported.
#[inline(always)]
#[cfg(target_arch = "aarch64")]
pub fn disable_fast_interrupts() {
    unsafe {
        // Clear the F bit, which is bit 0 of the DAIF bitset.
        asm!("msr daifset, #1", options(nomem, nostack, preserves_flags));
    }
    compiler_fence(Ordering::SeqCst);
}

/// Returns whether regular interrupts are enabled on the current CPU.
///
/// This only checks whether *regular* interrupts are enabled,
/// not NMIs or fast interrupts (FIQs on aarch64).
#[inline(always)]
pub fn interrupts_enabled() -> bool {
    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
    unsafe {
        // we only need the lower 16 bits of the eflags/rflags register
        let flags: usize;
        asm!("pushfq; pop {}", out(reg) flags, options(nomem, preserves_flags));
        (flags & 0x0200) != 0
    }

    #[cfg(target_arch = "aarch64")]
    unsafe {
        let daif: usize;
        asm!("mrs {}, daif", out(reg) daif, options(nomem, nostack, preserves_flags));
        // PSTATE flags of interest are in bits [6:9]; we only care about I, stored in bit 7.
        (daif & (1 << 7)) == 0
    }

    #[cfg(target_arch = "arm")]
    unsafe {
        let primask: u32;
        asm!("mrs {}, primask", out(reg) primask, options(nomem, nostack, preserves_flags));
        primask & (1 << 0) != (1 << 0)
    }
}