1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
//! The Receive Queue (RQ) object holds the descriptor ring used to hold incoming packets.
//! The descriptor ring is referred to as a Work Queue Buffer.
//! This module defines the layout of an RQ and the context used to initialize a RQ.
//! 
//! (PRM Section 8.13: Receive Queue)

use zerocopy::{U32, FromBytes};
use volatile::Volatile;
use byteorder::BigEndian;
use memory::{MappedPages, create_contiguous_mapping, BorrowedSliceMappedPages, Mutable, MMIO_FLAGS};
use core::fmt;
use num_enum::TryFromPrimitive;
use core::convert::TryFrom;
use alloc::vec::Vec;
use nic_buffers::ReceiveBuffer;

#[allow(unused_imports)]
use crate::{Rqn, Lkey, CQN_MASK, command_queue::CommandOpcode, work_queue::WorkQueueEntryReceive, completion_queue::CompletionQueue};


/// The Transport Interface Receive (TIR) object is responsible for performing 
/// all transport related operations on the receive side. TIR performs the
/// packet processing and reassembly and is also responsible for demultiplexing
/// packets into different RQs.
#[derive(FromBytes, Default)]
#[repr(C)]
pub(crate) struct TransportInterfaceReceiveContext {
    _padding1:              [u8; 28],
    /// RQ number that packets will directly be delivered to
    inline_rqn:             Volatile<U32<BigEndian>>,
    _padding2:              u32,
    /// transport domain ID
    transport_domain:       Volatile<U32<BigEndian>>,
    _padding3:              [u8; 32],
    _padding4:              [u8; 20],
}

const _: () = assert!(core::mem::size_of::<TransportInterfaceReceiveContext>() == 92);

impl TransportInterfaceReceiveContext {
    /// Initialize the TIR object
    /// 
    /// # Arguments
    /// * `rqn`: RQ number
    /// * `td`: transport domain ID 
    pub fn init(rqn: u32, td: u32) -> TransportInterfaceReceiveContext {
        let mut ctxt = TransportInterfaceReceiveContext::default();
        ctxt.inline_rqn.write(U32::new(rqn));
        ctxt.transport_domain.write(U32::new(td));
        ctxt
    }

    /// Offset that this context is written to in the mailbox buffer
    pub(crate) fn mailbox_offset() -> usize { 0x10 }
}

/// The possible states the RQ can be in.
#[derive(Debug, TryFromPrimitive)]
#[repr(u8)]
pub enum ReceiveQueueState {
    Reset = 0x0,
    Ready = 0x1,
    Error = 0x3
}

/// The bitmask for the state in the [`ReceiveQueueContext`]
const STATE_MASK:   u32 = 0xF0_0000;
/// The bit shift for the state in the [`ReceiveQueueContext`]
const STATE_SHIFT:  u32 = 20;

/// The data structure containing RQ initialization parameters.
/// It is passed to the HCA at the time of RQ creation.
#[derive(FromBytes, Default)]
#[repr(C, packed)]
pub(crate) struct ReceiveQueueContext {
    /// A multi-part field:
    /// * `rlky`: when set the reserved LKey can be used on the RQ, occupies bit 31
    /// * `vlan_strip_disable`: if set, VLAN is not stripped from incoming frames, occupies bit 28
    /// * `state`: RQ state, occupies bits [23:20]
    /// * `flush_in_error_en`: if set, and when RQ transitions into error state, the hardware will flush in error WQEs that were posted, occupies bit 18
    rlky_state:                         Volatile<U32<BigEndian>>,
    /// an opaque identifier which software sets, which is reported to the Completion Queue
    user_index:                         Volatile<U32<BigEndian>>,
    /// number of the CQ associated with this RQ
    cqn:                                Volatile<U32<BigEndian>>,
    /// set of counters in which statistics on this RQ are collected
    counter_set_id:                     Volatile<U32<BigEndian>>,
    /// remote memory pool number (only when enabled)
    rmpn:                               Volatile<U32<BigEndian>>,
    hairpin_peer_sq:                    Volatile<U32<BigEndian>>,
    hairpin_peer_vhca:                  Volatile<U32<BigEndian>>,
    _padding1:                          [u8; 20],
}

const _: () = assert!(core::mem::size_of::<ReceiveQueueContext>() == 48);

impl fmt::Debug for ReceiveQueueContext {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.debug_struct("ReceiveQueueContext")
            .field("rlky_state", &self.rlky_state.read().get())
            .field("user_index", &self.user_index.read().get())
            .field("cqn", &self.cqn.read().get())
            .field("counter_set_id", &self.counter_set_id.read().get())
            .field("rmpn", &self.rmpn.read().get())
            .field("hairpin_peer_rq", &self.hairpin_peer_sq.read().get())
            .field("hairpin_peer_vhca",&self.hairpin_peer_vhca.read().get())
            .finish()
    }
}

#[allow(unused)]
impl ReceiveQueueContext {
    /// Create and initialize the fields of the RQ context.
    /// The RQ context is then passed to the HCA when creating the RQ.
    /// 
    /// # Arguments
    /// * `cqn`: number of CQ associated with this RQ 
    pub fn init(cqn: u32) -> ReceiveQueueContext {
        const ENABLE_RLKEY:             u32 = 1 << 31;
        const VLAN_STRIP_DISABLE:       u32 = 1 << 28;
        
        // set all fields to zero
        let mut ctxt = ReceiveQueueContext::default();
        ctxt.rlky_state.write(U32::new(ENABLE_RLKEY | VLAN_STRIP_DISABLE)); 
        ctxt.cqn.write(U32::new(cqn & CQN_MASK));
        ctxt
    }

    /// set state of the RQ in the RQ context to `next_state`
    pub fn set_state(&mut self, next_state: ReceiveQueueState) {
        let state = self.rlky_state.read().get() & !STATE_MASK;
        self.rlky_state.write(U32::new(state | ((next_state as u32) << STATE_SHIFT))); 
    }

    /// Find the state of the RQ from the RQ context 
    pub fn get_state(&self) -> Result<ReceiveQueueState, &'static str> {
        let state = (self.rlky_state.read().get() & STATE_MASK) >> STATE_SHIFT;
        ReceiveQueueState::try_from(state as u8).map_err(|_e| "Invalid value in the RQ state")
    }

    /// Offset that this context is written to in the mailbox buffer
    pub(crate) fn mailbox_offset() -> usize { 0x10 }
}

/// A data structure that contains the RQ ring of descriptors 
/// and is used to interact with the RQ once initialized.
#[allow(dead_code)]
pub struct ReceiveQueue {
    /// physically-contiguous RQ descriptors
    entries: BorrowedSliceMappedPages<WorkQueueEntryReceive, Mutable>, 
    /// the packet buffers in use by the descriptors
    packet_buffers: Vec<ReceiveBuffer>,
    /// The size of a receive buffers in bytes. 
    /// It should be set to the MTU.
    buffer_size_bytes: u32,
    /// Rx buffer pool 
    pool: &'static mpmc::Queue<ReceiveBuffer>,
    /// The number of WQEs that have been completed.
    /// From this we also calculate the next descriptor to use
    wqe_counter: u16,
    /// completion queue index of the next completed packet
    cqe_counter: u16,
    /// CQE ownership value that indicates SW owned
    owner: u8,
    /// RQ number that is returned by the [`CommandOpcode::CreateRq`] command
    rqn: Rqn,
    /// the lkey used by the SQ
    lkey: Lkey,
    /// completion queue associated with this receive queue
    cq: CompletionQueue
}

impl ReceiveQueue {
    /// Creates a RQ by mapping the buffer as a slice of [`WorkQueueEntryReceive`]s.
    /// Each WQE is set to an initial state.
    /// 
    /// # Arguments
    /// * `entries_mp`: memory that is to be transformed into a slice of WQEs. 
    /// The starting physical address should have been passed to the HCA when creating the SQ.
    /// * `num_entries`: number of entries in the RQ
    /// * `mtu`: size of the receive buffers in bytes
    /// * `buffer_pool`: receive buffer pool 
    /// * `rqn`: SQ number returned by the HCA
    /// * `lkey`: the lkey used by the RQ
    pub fn create(
        entries_mp: MappedPages, 
        num_entries: usize,
        mtu: u32,
        pool: &'static mpmc::Queue<ReceiveBuffer>, 
        rqn: Rqn, 
        lkey: Lkey,
        cq: CompletionQueue
    ) -> Result<ReceiveQueue, &'static str> {
        // map the descriptor ring and initialize
        let mut entries = entries_mp.into_borrowed_slice_mut::<WorkQueueEntryReceive>(0, num_entries)
            .map_err(|(_mp, err)| err)?;
        for entry in entries.iter_mut() {
            entry.init()
        }
        Ok(ReceiveQueue {
            entries, 
            packet_buffers: Vec::new(), 
            buffer_size_bytes: mtu,
            pool,
            wqe_counter: 0,
            cqe_counter: 0, 
            owner: 0,
            rqn, 
            lkey,
            cq
        })
    }

    /// Refills the receive queue by updating WQEs with new packet buffers.
    /// Right now we assume that this function is only called once at the point of initialization.
    /// 
    /// TODO:
    /// this function can be shifted to nic_initialization if we remove intel specific actions from those functions
    pub fn refill(&mut self) -> Result<(), &'static str> {
        let buffer_size = self.buffer_size_bytes;
        let mem_pool = self.pool;

        // now that we've created the rx descriptors, we can fill them in with initial values
        let mut rx_bufs_in_use: Vec<ReceiveBuffer> = Vec::with_capacity(self.entries.len());
        for wqe in self.entries.iter_mut()
        {
            // obtain or create a receive buffer for each rx_desc
            let rx_buf = self.pool.pop()
                .ok_or("Couldn't obtain a ReceiveBuffer from the pool")
                .or_else(|_e| {
                    create_contiguous_mapping(buffer_size as usize, MMIO_FLAGS)
                        .and_then(|(buf_mapped, buf_paddr)|
                            ReceiveBuffer::new(buf_mapped, buf_paddr, buffer_size as u16, mem_pool)
                        )
                })?;
            let paddr_buf = rx_buf.phys_addr();
            rx_bufs_in_use.push(rx_buf); 

            wqe.update_buffer_info(self.lkey.0, paddr_buf, self.buffer_size_bytes); 
        }
        self.packet_buffers = rx_bufs_in_use;
        Ok(())
    }

}