#![no_std]
#![allow(clippy::type_complexity)]
#![allow(dead_code)] #![feature(rustc_private)]
#![feature(abi_x86_interrupt)]
#[macro_use] extern crate log;
#[macro_use] extern crate lazy_static;
extern crate volatile;
extern crate zerocopy;
extern crate alloc;
extern crate spin;
extern crate sync_irq;
extern crate kernel_config;
extern crate memory;
extern crate pci;
extern crate interrupts;
extern crate x86_64;
extern crate mpmc;
extern crate intel_ethernet;
extern crate nic_buffers;
extern crate nic_queues;
extern crate nic_initialization;
extern crate net;
extern crate deferred_interrupt_tasks;
extern crate task;
pub mod test_e1000_driver;
mod regs;
use regs::*;
use spin::Once;
use alloc::{collections::VecDeque, format, sync::Arc, vec::Vec};
use sync_irq::IrqSafeMutex;
use memory::{PhysicalAddress, BorrowedMappedPages, BorrowedSliceMappedPages, Mutable, map_frame_range, MMIO_FLAGS};
use pci::{PciDevice, PciConfigSpaceAccessMechanism};
use kernel_config::memory::PAGE_SIZE;
use interrupts::{eoi, InterruptNumber};
use x86_64::structures::idt::InterruptStackFrame;
use nic_initialization::{init_rx_buf_pool, init_rx_queue, init_tx_queue};
use intel_ethernet::descriptors::{LegacyRxDescriptor, LegacyTxDescriptor};
use nic_buffers::{TransmitBuffer, ReceiveBuffer, ReceivedFrame};
use nic_queues::{RxQueue, TxQueue, RxQueueRegisters, TxQueueRegisters};
pub const INTEL_VEND: u16 = 0x8086; pub const E1000_DEV: u16 = 0x100E; const E1000_NUM_RX_DESC: u16 = 8;
const E1000_NUM_TX_DESC: u16 = 8;
const E1000_RX_BUFFER_SIZE_IN_BYTES: u16 = PAGE_SIZE as u16;
const INT_LSC: u32 = 0x04;
const INT_RX: u32 = 0x80;
static E1000_NIC: Once<IrqSafeMutex<E1000Nic>> = Once::new();
pub fn get_e1000_nic() -> Option<&'static IrqSafeMutex<E1000Nic>> {
E1000_NIC.get()
}
const RX_BUFFER_POOL_SIZE: usize = 256;
lazy_static! {
static ref RX_BUFFER_POOL: mpmc::Queue<ReceiveBuffer> = mpmc::Queue::with_capacity(RX_BUFFER_POOL_SIZE);
}
struct E1000RxQueueRegisters(BorrowedMappedPages<E1000RxRegisters, Mutable>);
impl RxQueueRegisters for E1000RxQueueRegisters {
fn set_rdbal(&mut self, value: u32) {
self.0.rx_regs.rdbal.write(value);
}
fn set_rdbah(&mut self, value: u32) {
self.0.rx_regs.rdbah.write(value);
}
fn set_rdlen(&mut self, value: u32) {
self.0.rx_regs.rdlen.write(value);
}
fn set_rdh(&mut self, value: u32) {
self.0.rx_regs.rdh.write(value);
}
fn set_rdt(&mut self, value: u32) {
self.0.rx_regs.rdt.write(value);
}
}
struct E1000TxQueueRegisters(BorrowedMappedPages<E1000TxRegisters, Mutable>);
impl TxQueueRegisters for E1000TxQueueRegisters {
fn set_tdbal(&mut self, value: u32) {
self.0.tx_regs.tdbal.write(value);
}
fn set_tdbah(&mut self, value: u32) {
self.0.tx_regs.tdbah.write(value);
}
fn set_tdlen(&mut self, value: u32) {
self.0.tx_regs.tdlen.write(value);
}
fn set_tdh(&mut self, value: u32) {
self.0.tx_regs.tdh.write(value);
}
fn set_tdt(&mut self, value: u32) {
self.0.tx_regs.tdt.write(value);
}
}
pub struct E1000Nic {
bar_type: u8,
interrupt_num: InterruptNumber,
mac_hardware: [u8; 6],
mac_spoofed: Option<[u8; 6]>,
mem_base: PhysicalAddress,
rx_queue: RxQueue<E1000RxQueueRegisters, LegacyRxDescriptor>,
tx_queue: TxQueue<E1000TxQueueRegisters, LegacyTxDescriptor>,
regs: BorrowedMappedPages<E1000Registers, Mutable>,
mac_regs: BorrowedMappedPages<E1000MacRegisters, Mutable>,
deferred_task: Option<task::JoinableTaskRef>,
}
impl E1000Nic {
pub fn init(e1000_pci_dev: &PciDevice) -> Result<&'static IrqSafeMutex<E1000Nic>, &'static str> {
use interrupts::IRQ_BASE_OFFSET;
let interrupt_num = match e1000_pci_dev.pci_get_intx_info() {
Ok((Some(irq), _pin)) => (irq + IRQ_BASE_OFFSET) as InterruptNumber,
_ => return Err("e1000: PCI device had no interrupt number (IRQ vector)"),
};
let bar0 = e1000_pci_dev.bars[0];
let bar_type = (bar0 as u8) & 0x1;
if bar_type == PciConfigSpaceAccessMechanism::IoPort as u8 {
error!("e1000::init(): BAR0 is of I/O type");
return Err("e1000::init(): BAR0 is of I/O type")
}
let mem_base = e1000_pci_dev.determine_mem_base(0)?;
e1000_pci_dev.pci_set_command_bus_master_bit();
let (mut mapped_registers, rx_registers, tx_registers, mut mac_registers) = Self::map_e1000_regs(e1000_pci_dev, mem_base)?;
let mut rx_registers = E1000RxQueueRegisters(rx_registers);
let mut tx_registers = E1000TxQueueRegisters(tx_registers);
Self::start_link(&mut mapped_registers);
let mac_addr_hardware = Self::read_mac_address_from_nic(&mut mac_registers);
init_rx_buf_pool(RX_BUFFER_POOL_SIZE, E1000_RX_BUFFER_SIZE_IN_BYTES, &RX_BUFFER_POOL)?;
let (rx_descs, rx_buffers) = Self::rx_init(&mut mapped_registers, &mut rx_registers)?;
let rxq = RxQueue {
id: 0,
regs: rx_registers,
rx_descs,
num_rx_descs: E1000_NUM_RX_DESC,
rx_cur: 0,
rx_bufs_in_use: rx_buffers,
rx_buffer_size_bytes: E1000_RX_BUFFER_SIZE_IN_BYTES,
received_frames: VecDeque::new(),
cpu_id: None,
rx_buffer_pool: &RX_BUFFER_POOL,
filter_num: None
};
let tx_descs = Self::tx_init(&mut mapped_registers, &mut tx_registers)?;
let txq = TxQueue {
id: 0,
regs: tx_registers,
tx_descs,
num_tx_descs: E1000_NUM_TX_DESC,
tx_cur: 0,
cpu_id: None,
};
let e1000_nic = E1000Nic {
bar_type,
mem_base,
interrupt_num,
mac_hardware: mac_addr_hardware,
mac_spoofed: None,
rx_queue: rxq,
tx_queue: txq,
regs: mapped_registers,
mac_regs: mac_registers,
deferred_task: None,
};
let nic_ref = E1000_NIC.call_once(|| IrqSafeMutex::new(e1000_nic));
Ok(nic_ref)
}
pub fn init_interrupts(
&mut self,
interface: Arc<net::NetworkInterface>,
) -> Result<(), &'static str> {
self.enable_interrupts();
let deferred_task = deferred_interrupt_tasks::register_interrupt_handler(
self.interrupt_num,
e1000_handler,
poll_interface,
interface,
Some(format!("e1000_deferred_task_irq_{:#X}", self.interrupt_num)),
)
.map_err(|error| {
error!("error registering e1000 handler: {:?}", error);
"e1000 interrupt number was already in use! Sharing IRQs is currently unsupported."
})?;
self.deferred_task = Some(deferred_task);
Ok(())
}
fn map_e1000_regs(
_device: &PciDevice,
mem_base: PhysicalAddress,
) -> Result<(
BorrowedMappedPages<E1000Registers, Mutable>,
BorrowedMappedPages<E1000RxRegisters, Mutable>,
BorrowedMappedPages<E1000TxRegisters, Mutable>,
BorrowedMappedPages<E1000MacRegisters, Mutable>
), &'static str> {
const GENERAL_REGISTERS_SIZE_BYTES: usize = 8192;
const RX_REGISTERS_SIZE_BYTES: usize = 4096;
const TX_REGISTERS_SIZE_BYTES: usize = 4096;
const MAC_REGISTERS_SIZE_BYTES: usize = 114_688;
let mut physical_addr = mem_base;
let nic_regs_mapped_page = map_frame_range(physical_addr, GENERAL_REGISTERS_SIZE_BYTES, MMIO_FLAGS)?;
let regs = nic_regs_mapped_page.into_borrowed_mut(0).map_err(|(_mp, err)| err)?;
physical_addr += GENERAL_REGISTERS_SIZE_BYTES;
let nic_rx_regs_mapped_page = map_frame_range(physical_addr, RX_REGISTERS_SIZE_BYTES, MMIO_FLAGS)?;
let rx_regs = nic_rx_regs_mapped_page.into_borrowed_mut(0).map_err(|(_mp, err)| err)?;
physical_addr += RX_REGISTERS_SIZE_BYTES;
let nic_tx_regs_mapped_page = map_frame_range(physical_addr, TX_REGISTERS_SIZE_BYTES, MMIO_FLAGS)?;
let tx_regs = nic_tx_regs_mapped_page.into_borrowed_mut(0).map_err(|(_mp, err)| err)?;
physical_addr += TX_REGISTERS_SIZE_BYTES;
let nic_mac_regs_mapped_page = map_frame_range(physical_addr, MAC_REGISTERS_SIZE_BYTES, MMIO_FLAGS)?;
let mac_regs = nic_mac_regs_mapped_page.into_borrowed_mut(0).map_err(|(_mp, err)| err)?;
Ok((regs, rx_regs, tx_regs, mac_regs))
}
pub fn spoof_mac(&mut self, spoofed_mac_addr: [u8; 6]) {
self.mac_spoofed = Some(spoofed_mac_addr);
}
fn read_mac_address_from_nic(regs: &mut E1000MacRegisters) -> [u8; 6] {
let mac_32_low = regs.ral.read();
let mac_32_high = regs.rah.read();
let mut mac_addr = [0; 6];
mac_addr[0] = mac_32_low as u8;
mac_addr[1] = (mac_32_low >> 8) as u8;
mac_addr[2] = (mac_32_low >> 16) as u8;
mac_addr[3] = (mac_32_low >> 24) as u8;
mac_addr[4] = mac_32_high as u8;
mac_addr[5] = (mac_32_high >> 8) as u8;
debug!("E1000: read hardware MAC address: {:02x?}", mac_addr);
mac_addr
}
fn start_link(regs: &mut E1000Registers) {
let val = regs.ctrl.read();
regs.ctrl.write(val | 0x40 | 0x20);
let val = regs.ctrl.read();
regs.ctrl.write(val & !(regs::CTRL_LRST) & !(regs::CTRL_ILOS) & !(regs::CTRL_VME) & !(regs::CTRL_PHY_RST));
debug!("e1000::start_link(): REG_CTRL: {:#X}", regs.ctrl.read());
}
fn rx_init(
regs: &mut E1000Registers,
rx_regs: &mut E1000RxQueueRegisters
) -> Result<(
BorrowedSliceMappedPages<LegacyRxDescriptor, Mutable>,
Vec<ReceiveBuffer>
), &'static str> {
let (rx_descs, rx_bufs_in_use) = init_rx_queue(E1000_NUM_RX_DESC as usize, &RX_BUFFER_POOL, E1000_RX_BUFFER_SIZE_IN_BYTES as usize, rx_regs)?;
rx_regs.set_rdt((E1000_NUM_RX_DESC - 1) as u32);
regs.rctl.write(regs::RCTL_EN| regs::RCTL_SBP | regs::RCTL_LBM_NONE | regs::RTCL_RDMTS_HALF | regs::RCTL_BAM | regs::RCTL_SECRC | regs::RCTL_BSIZE_2048);
Ok((rx_descs, rx_bufs_in_use))
}
fn tx_init(
regs: &mut E1000Registers,
tx_regs: &mut E1000TxQueueRegisters
) -> Result<BorrowedSliceMappedPages<LegacyTxDescriptor, Mutable>, &'static str> {
let tx_descs = init_tx_queue(E1000_NUM_TX_DESC as usize, tx_regs)?;
regs.tctl.write(regs::TCTL_EN | regs::TCTL_PSP);
Ok(tx_descs)
}
fn enable_interrupts(&mut self) {
self.regs.ims.write(INT_LSC | INT_RX);
self.regs.icr.read();
}
fn clear_interrupt_status(&self) -> u32 {
self.regs.icr.read()
}
fn handle_interrupt(&mut self) -> Result<(), &'static str> {
let status = self.clear_interrupt_status();
let mut handled = false;
if (status & INT_LSC) == INT_LSC {
debug!("e1000::handle_interrupt(): link status changed");
Self::start_link(&mut self.regs);
handled = true;
}
if (status & INT_RX) == INT_RX {
self.rx_queue.poll_queue_and_store_received_packets()?;
handled = true;
}
if !handled {
error!("e1000::handle_interrupt(): unhandled interrupt! status: {:#X}", status);
} else if let Some(ref deferred_task) = self.deferred_task {
let _ = deferred_task
.unblock()
.expect("BUG: e1000::handle_interrupt(): couldn't unblock deferred task");
} else {
error!("e1000::handle_interrupt(): no deferred task");
}
Ok(())
}
}
impl net::NetworkDevice for E1000Nic {
fn send(&mut self, buf: TransmitBuffer) {
self.tx_queue.send_on_queue(buf);
}
fn receive(&mut self) -> Option<ReceivedFrame> {
self.rx_queue.received_frames.pop_front()
}
fn mac_address(&self) -> [u8; 6] {
self.mac_spoofed.unwrap_or(self.mac_hardware)
}
}
extern "x86-interrupt" fn e1000_handler(_stack_frame: InterruptStackFrame) {
if let Some(e1000_nic_ref) = E1000_NIC.get() {
let mut e1000_nic = e1000_nic_ref.lock();
if let Err(e) = e1000_nic.handle_interrupt() {
error!("e1000_handler(): error handling interrupt: {:?}", e);
}
eoi(e1000_nic.interrupt_num);
} else {
error!("BUG: e1000_handler(): E1000 NIC hasn't yet been initialized!");
}
}
fn poll_interface(interface: &Arc<net::NetworkInterface>) -> Result<(), ()> {
interface.poll();
Ok(())
}