/rust/registry/src/index.crates.io-6f17d22bba15001f/virtio-queue-0.12.0/src/queue.rs

Source (jump to first uncovered line)
// Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// Copyright (C) 2020-2021 Alibaba Cloud. All rights reserved.
// Copyright © 2019 Intel Corporation.
// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE-BSD-3-Clause file.
//
// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause

use std::mem::size_of;
use std::num::Wrapping;
use std::ops::Deref;
use std::sync::atomic::{fence, Ordering};

use vm_memory::{Address, Bytes, GuestAddress, GuestMemory};

use crate::defs::{
    DEFAULT_AVAIL_RING_ADDR, DEFAULT_DESC_TABLE_ADDR, DEFAULT_USED_RING_ADDR,
    VIRTQ_AVAIL_ELEMENT_SIZE, VIRTQ_AVAIL_RING_HEADER_SIZE, VIRTQ_AVAIL_RING_META_SIZE,
    VIRTQ_USED_ELEMENT_SIZE, VIRTQ_USED_RING_HEADER_SIZE, VIRTQ_USED_RING_META_SIZE,
};
use crate::{
    error, Descriptor, DescriptorChain, Error, QueueGuard, QueueOwnedT, QueueState, QueueT,
    VirtqUsedElem,
};
use virtio_bindings::bindings::virtio_ring::VRING_USED_F_NO_NOTIFY;

/// The maximum queue size as defined in the Virtio Spec.
pub const MAX_QUEUE_SIZE: u16 = 32768;

/// Struct to maintain information and manipulate a virtio queue.
///
/// # Example
///
/// ```rust
/// use virtio_queue::{Queue, QueueOwnedT, QueueT};
/// use vm_memory::{Bytes, GuestAddress, GuestAddressSpace, GuestMemoryMmap};
///
/// let m = GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
/// let mut queue = Queue::new(1024).unwrap();
///
/// // First, the driver sets up the queue; this set up is done via writes on the bus (PCI, MMIO).
/// queue.set_size(8);
/// queue.set_desc_table_address(Some(0x1000), None);
/// queue.set_avail_ring_address(Some(0x2000), None);
/// queue.set_used_ring_address(Some(0x3000), None);
/// queue.set_event_idx(true);
/// queue.set_ready(true);
/// // The user should check if the queue is valid before starting to use it.
/// assert!(queue.is_valid(&m));
///
/// // Here the driver would add entries in the available ring and then update the `idx` field of
/// // the available ring (address = 0x2000 + 2).
/// m.write_obj(3, GuestAddress(0x2002));
///
/// loop {
///     queue.disable_notification(&m).unwrap();
///
///     // Consume entries from the available ring.
///     while let Some(chain) = queue.iter(&m).unwrap().next() {
///         // Process the descriptor chain, and then add an entry in the used ring and optionally
///         // notify the driver.
///         queue.add_used(&m, chain.head_index(), 0x100).unwrap();
///
///         if queue.needs_notification(&m).unwrap() {
///             // Here we would notify the driver it has new entries in the used ring to consume.
///         }
///     }
///     if !queue.enable_notification(&m).unwrap() {
///         break;
///     }
/// }
///
/// // We can reset the queue at some point.
/// queue.reset();
/// // The queue should not be ready after reset.
/// assert!(!queue.ready());
/// ```
#[derive(Debug, Default, PartialEq, Eq)]
pub struct Queue {
    /// The maximum size in elements offered by the device.
    max_size: u16,

    /// Tail position of the available ring.
    next_avail: Wrapping<u16>,

    /// Head position of the used ring.
    next_used: Wrapping<u16>,

    /// VIRTIO_F_RING_EVENT_IDX negotiated.
    event_idx_enabled: bool,

    /// The number of descriptor chains placed in the used ring via `add_used`
    /// since the last time `needs_notification` was called on the associated queue.
    num_added: Wrapping<u16>,

    /// The queue size in elements the driver selected.
    size: u16,

    /// Indicates if the queue is finished with configuration.
    ready: bool,

    /// Guest physical address of the descriptor table.
    desc_table: GuestAddress,

    /// Guest physical address of the available ring.
    avail_ring: GuestAddress,

    /// Guest physical address of the used ring.
    used_ring: GuestAddress,
}

impl Queue {
    /// Equivalent of [`QueueT::set_size`] returning an error in case of invalid size.
    ///
    /// This should not be directly used, as the preferred method is part of the [`QueueT`]
    /// interface. This is a convenience function for implementing save/restore capabilities.
    pub fn try_set_size(&mut self, size: u16) -> Result<(), Error> {
        if size > self.max_size() || size == 0 || (size & (size - 1)) != 0 {
            return Err(Error::InvalidSize);
        }
        self.size = size;
        Ok(())
    }

    /// Tries to set the descriptor table address. In case of an invalid value, the address is
    /// not updated.
    ///
    /// This should not be directly used, as the preferred method is
    /// [`QueueT::set_desc_table_address`]. This is a convenience function for implementing
    /// save/restore capabilities.
    pub fn try_set_desc_table_address(&mut self, desc_table: GuestAddress) -> Result<(), Error> {
        if desc_table.mask(0xf) != 0 {
            return Err(Error::InvalidDescTableAlign);
        }
        self.desc_table = desc_table;

        Ok(())
    }

    /// Tries to update the available ring address. In case of an invalid value, the address is
    /// not updated.
    ///
    /// This should not be directly used, as the preferred method is
    /// [`QueueT::set_avail_ring_address`]. This is a convenience function for implementing
    /// save/restore capabilities.
    pub fn try_set_avail_ring_address(&mut self, avail_ring: GuestAddress) -> Result<(), Error> {
        if avail_ring.mask(0x1) != 0 {
            return Err(Error::InvalidAvailRingAlign);
        }
        self.avail_ring = avail_ring;
        Ok(())
    }

    /// Tries to update the used ring address. In cae of an invalid value, the address is not
    /// updated.
    ///
    /// This should not be directly used, as the preferred method is
    /// [`QueueT::set_used_ring_address`]. This is a convenience function for implementing
    /// save/restore capabilities.
    pub fn try_set_used_ring_address(&mut self, used_ring: GuestAddress) -> Result<(), Error> {
        if used_ring.mask(0x3) != 0 {
            return Err(Error::InvalidUsedRingAlign);
        }
        self.used_ring = used_ring;
        Ok(())
    }

    /// Returns the state of the `Queue`.
    ///
    /// This is useful for implementing save/restore capabilities.
    /// The state does not have support for serialization, but this can be
    /// added by VMMs locally through the use of a
    /// [remote type](https://serde.rs/remote-derive.html).
    ///
    /// Alternatively, a version aware and serializable/deserializable QueueState
    /// is available in the `virtio-queue-ser` crate.
    pub fn state(&self) -> QueueState {
        QueueState {
            max_size: self.max_size,
            next_avail: self.next_avail(),
            next_used: self.next_used(),
            event_idx_enabled: self.event_idx_enabled,
            size: self.size,
            ready: self.ready,
            desc_table: self.desc_table(),
            avail_ring: self.avail_ring(),
            used_ring: self.used_ring(),
        }
    }

    // Helper method that writes `val` to the `avail_event` field of the used ring, using
    // the provided ordering.
    fn set_avail_event<M: GuestMemory>(
        &self,
        mem: &M,
        val: u16,
        order: Ordering,
    ) -> Result<(), Error> {
        // This can not overflow an u64 since it is working with relatively small numbers compared
        // to u64::MAX.
        let avail_event_offset =
            VIRTQ_USED_RING_HEADER_SIZE + VIRTQ_USED_ELEMENT_SIZE * u64::from(self.size);
        let addr = self
            .used_ring
            .checked_add(avail_event_offset)
            .ok_or(Error::AddressOverflow)?;

        mem.store(u16::to_le(val), addr, order)
            .map_err(Error::GuestMemory)
    }

    // Set the value of the `flags` field of the used ring, applying the specified ordering.
    fn set_used_flags<M: GuestMemory>(
        &mut self,
        mem: &M,
        val: u16,
        order: Ordering,
    ) -> Result<(), Error> {
        mem.store(u16::to_le(val), self.used_ring, order)
            .map_err(Error::GuestMemory)
    }

    // Write the appropriate values to enable or disable notifications from the driver.
    //
    // Every access in this method uses `Relaxed` ordering because a fence is added by the caller
    // when appropriate.
    fn set_notification<M: GuestMemory>(&mut self, mem: &M, enable: bool) -> Result<(), Error> {
        if enable {
            if self.event_idx_enabled {
                // We call `set_avail_event` using the `next_avail` value, instead of reading
                // and using the current `avail_idx` to avoid missing notifications. More
                // details in `enable_notification`.
                self.set_avail_event(mem, self.next_avail.0, Ordering::Relaxed)
            } else {
                self.set_used_flags(mem, 0, Ordering::Relaxed)
            }
        } else if !self.event_idx_enabled {
            self.set_used_flags(mem, VRING_USED_F_NO_NOTIFY as u16, Ordering::Relaxed)
        } else {
            // Notifications are effectively disabled by default after triggering once when
            // `VIRTIO_F_EVENT_IDX` is negotiated, so we don't do anything in that case.
            Ok(())
        }
    }

    // Return the value present in the used_event field of the avail ring.
    //
    // If the VIRTIO_F_EVENT_IDX feature bit is not negotiated, the flags field in the available
    // ring offers a crude mechanism for the driver to inform the device that it doesn’t want
    // interrupts when buffers are used. Otherwise virtq_avail.used_event is a more performant
    // alternative where the driver specifies how far the device can progress before interrupting.
    //
    // Neither of these interrupt suppression methods are reliable, as they are not synchronized
    // with the device, but they serve as useful optimizations. So we only ensure access to the
    // virtq_avail.used_event is atomic, but do not need to synchronize with other memory accesses.
    fn used_event<M: GuestMemory>(&self, mem: &M, order: Ordering) -> Result<Wrapping<u16>, Error> {
        // This can not overflow an u64 since it is working with relatively small numbers compared
        // to u64::MAX.
        let used_event_offset =
            VIRTQ_AVAIL_RING_HEADER_SIZE + u64::from(self.size) * VIRTQ_AVAIL_ELEMENT_SIZE;
        let used_event_addr = self
            .avail_ring
            .checked_add(used_event_offset)
            .ok_or(Error::AddressOverflow)?;

        mem.load(used_event_addr, order)
            .map(u16::from_le)
            .map(Wrapping)
            .map_err(Error::GuestMemory)
    }
}

impl<'a> QueueGuard<'a> for Queue {
    type G = &'a mut Self;
}

impl QueueT for Queue {
    fn new(max_size: u16) -> Result<Self, Error> {
        // We need to check that the max size is a power of 2 because we're setting this as the
        // queue size, and the valid queue sizes are a power of 2 as per the specification.
        if max_size == 0 || max_size > MAX_QUEUE_SIZE || (max_size & (max_size - 1)) != 0 {
            return Err(Error::InvalidMaxSize);
        }
        Ok(Queue {
            max_size,
            size: max_size,
            ready: false,
            desc_table: GuestAddress(DEFAULT_DESC_TABLE_ADDR),
            avail_ring: GuestAddress(DEFAULT_AVAIL_RING_ADDR),
            used_ring: GuestAddress(DEFAULT_USED_RING_ADDR),
            next_avail: Wrapping(0),
            next_used: Wrapping(0),
            event_idx_enabled: false,
            num_added: Wrapping(0),
        })
    }

    fn is_valid<M: GuestMemory>(&self, mem: &M) -> bool {
        let queue_size = self.size as u64;
        let desc_table = self.desc_table;
        // The multiplication can not overflow an u64 since we are multiplying an u16 with a
        // small number.
        let desc_table_size = size_of::<Descriptor>() as u64 * queue_size;
        let avail_ring = self.avail_ring;
        // The operations below can not overflow an u64 since they're working with relatively small
        // numbers compared to u64::MAX.
        let avail_ring_size = VIRTQ_AVAIL_RING_META_SIZE + VIRTQ_AVAIL_ELEMENT_SIZE * queue_size;
        let used_ring = self.used_ring;
        let used_ring_size = VIRTQ_USED_RING_META_SIZE + VIRTQ_USED_ELEMENT_SIZE * queue_size;

        if !self.ready {
            error!("attempt to use virtio queue that is not marked ready");
            false
        } else if desc_table
            .checked_add(desc_table_size)
            .map_or(true, |v| !mem.address_in_range(v))
        {
            error!(
                "virtio queue descriptor table goes out of bounds: start:0x{:08x} size:0x{:08x}",
                desc_table.raw_value(),
                desc_table_size
            );
            false
        } else if avail_ring
            .checked_add(avail_ring_size)
            .map_or(true, |v| !mem.address_in_range(v))
        {
            error!(
                "virtio queue available ring goes out of bounds: start:0x{:08x} size:0x{:08x}",
                avail_ring.raw_value(),
                avail_ring_size
            );
            false
        } else if used_ring
            .checked_add(used_ring_size)
            .map_or(true, |v| !mem.address_in_range(v))
        {
            error!(
                "virtio queue used ring goes out of bounds: start:0x{:08x} size:0x{:08x}",
                used_ring.raw_value(),
                used_ring_size
            );
            false
        } else {
            true
        }
    }

    fn reset(&mut self) {
        self.ready = false;
        self.size = self.max_size;
        self.desc_table = GuestAddress(DEFAULT_DESC_TABLE_ADDR);
        self.avail_ring = GuestAddress(DEFAULT_AVAIL_RING_ADDR);
        self.used_ring = GuestAddress(DEFAULT_USED_RING_ADDR);
        self.next_avail = Wrapping(0);
        self.next_used = Wrapping(0);
        self.num_added = Wrapping(0);
        self.event_idx_enabled = false;
    }

    fn lock(&mut self) -> <Self as QueueGuard>::G {
        self
    }

    fn max_size(&self) -> u16 {
        self.max_size
    }

    fn size(&self) -> u16 {
        self.size
    }

    fn set_size(&mut self, size: u16) {
        if self.try_set_size(size).is_err() {
            error!("virtio queue with invalid size: {}", size);
        }
    }

    fn ready(&self) -> bool {
        self.ready
    }

    fn set_ready(&mut self, ready: bool) {
        self.ready = ready;
    }

    fn set_desc_table_address(&mut self, low: Option<u32>, high: Option<u32>) {
        let low = low.unwrap_or(self.desc_table.0 as u32) as u64;
        let high = high.unwrap_or((self.desc_table.0 >> 32) as u32) as u64;

        let desc_table = GuestAddress((high << 32) | low);
        if self.try_set_desc_table_address(desc_table).is_err() {
            error!("virtio queue descriptor table breaks alignment constraints");
        }
    }

    fn set_avail_ring_address(&mut self, low: Option<u32>, high: Option<u32>) {
        let low = low.unwrap_or(self.avail_ring.0 as u32) as u64;
        let high = high.unwrap_or((self.avail_ring.0 >> 32) as u32) as u64;

        let avail_ring = GuestAddress((high << 32) | low);
        if self.try_set_avail_ring_address(avail_ring).is_err() {
            error!("virtio queue available ring breaks alignment constraints");
        }
    }

    fn set_used_ring_address(&mut self, low: Option<u32>, high: Option<u32>) {
        let low = low.unwrap_or(self.used_ring.0 as u32) as u64;
        let high = high.unwrap_or((self.used_ring.0 >> 32) as u32) as u64;

        let used_ring = GuestAddress((high << 32) | low);
        if self.try_set_used_ring_address(used_ring).is_err() {
            error!("virtio queue used ring breaks alignment constraints");
        }
    }

    fn set_event_idx(&mut self, enabled: bool) {
        self.event_idx_enabled = enabled;
    }

    fn avail_idx<M>(&self, mem: &M, order: Ordering) -> Result<Wrapping<u16>, Error>
    where
        M: GuestMemory + ?Sized,
    {
        let addr = self
            .avail_ring
            .checked_add(2)
            .ok_or(Error::AddressOverflow)?;

        mem.load(addr, order)
            .map(u16::from_le)
            .map(Wrapping)
            .map_err(Error::GuestMemory)
    }

    fn used_idx<M: GuestMemory>(&self, mem: &M, order: Ordering) -> Result<Wrapping<u16>, Error> {
        let addr = self
            .used_ring
            .checked_add(2)
            .ok_or(Error::AddressOverflow)?;

        mem.load(addr, order)
            .map(u16::from_le)
            .map(Wrapping)
            .map_err(Error::GuestMemory)
    }

    fn add_used<M: GuestMemory>(
        &mut self,
        mem: &M,
        head_index: u16,
        len: u32,
    ) -> Result<(), Error> {
        if head_index >= self.size {
            error!(
                "attempted to add out of bounds descriptor to used ring: {}",
                head_index
            );
            return Err(Error::InvalidDescriptorIndex);
        }

        let next_used_index = u64::from(self.next_used.0 % self.size);
        // This can not overflow an u64 since it is working with relatively small numbers compared
        // to u64::MAX.
        let offset = VIRTQ_USED_RING_HEADER_SIZE + next_used_index * VIRTQ_USED_ELEMENT_SIZE;
        let addr = self
            .used_ring
            .checked_add(offset)
            .ok_or(Error::AddressOverflow)?;
        mem.write_obj(VirtqUsedElem::new(head_index.into(), len), addr)
            .map_err(Error::GuestMemory)?;

        self.next_used += Wrapping(1);
        self.num_added += Wrapping(1);

        mem.store(
            u16::to_le(self.next_used.0),
            self.used_ring
                .checked_add(2)
                .ok_or(Error::AddressOverflow)?,
            Ordering::Release,
        )
        .map_err(Error::GuestMemory)
    }

    // TODO: Turn this into a doc comment/example.
    // With the current implementation, a common way of consuming entries from the available ring
    // while also leveraging notification suppression is to use a loop, for example:
    //
    // loop {
    //     // We have to explicitly disable notifications if `VIRTIO_F_EVENT_IDX` has not been
    //     // negotiated.
    //     self.disable_notification()?;
    //
    //     for chain in self.iter()? {
    //         // Do something with each chain ...
    //         // Let's assume we process all available chains here.
    //     }
    //
    //     // If `enable_notification` returns `true`, the driver has added more entries to the
    //     // available ring.
    //     if !self.enable_notification()? {
    //         break;
    //     }
    // }
    fn enable_notification<M: GuestMemory>(&mut self, mem: &M) -> Result<bool, Error> {
        self.set_notification(mem, true)?;
        // Ensures the following read is not reordered before any previous write operation.
        fence(Ordering::SeqCst);

        // We double check here to avoid the situation where the available ring has been updated
        // just before we re-enabled notifications, and it's possible to miss one. We compare the
        // current `avail_idx` value to `self.next_avail` because it's where we stopped processing
        // entries. There are situations where we intentionally avoid processing everything in the
        // available ring (which will cause this method to return `true`), but in that case we'll
        // probably not re-enable notifications as we already know there are pending entries.
        self.avail_idx(mem, Ordering::Relaxed)
            .map(|idx| idx != self.next_avail)
    }

    fn disable_notification<M: GuestMemory>(&mut self, mem: &M) -> Result<(), Error> {
        self.set_notification(mem, false)
    }

    fn needs_notification<M: GuestMemory>(&mut self, mem: &M) -> Result<bool, Error> {
        let used_idx = self.next_used;

        // Complete all the writes in add_used() before reading the event.
        fence(Ordering::SeqCst);

        // The VRING_AVAIL_F_NO_INTERRUPT flag isn't supported yet.

        // When the `EVENT_IDX` feature is negotiated, the driver writes into `used_event`
        // a value that's used by the device to determine whether a notification must
        // be submitted after adding a descriptor chain to the used ring. According to the
        // standard, the notification must be sent when `next_used == used_event + 1`, but
        // various device model implementations rely on an inequality instead, most likely
        // to also support use cases where a bunch of descriptor chains are added to the used
        // ring first, and only afterwards the `needs_notification` logic is called. For example,
        // the approach based on `num_added` below is taken from the Linux Kernel implementation
        // (i.e. https://elixir.bootlin.com/linux/v5.15.35/source/drivers/virtio/virtio_ring.c#L661)

        // The `old` variable below is used to determine the value of `next_used` from when
        // `needs_notification` was called last (each `needs_notification` call resets `num_added`
        // to zero, while each `add_used` called increments it by one). Then, the logic below
        // uses wrapped arithmetic to see whether `used_event` can be found between `old` and
        // `next_used` in the circular sequence space of the used ring.
        if self.event_idx_enabled {
            let used_event = self.used_event(mem, Ordering::Relaxed)?;
            let old = used_idx - self.num_added;
            self.num_added = Wrapping(0);

            return Ok(used_idx - used_event - Wrapping(1) < used_idx - old);
        }

        Ok(true)
    }

    fn next_avail(&self) -> u16 {
        self.next_avail.0
    }

    fn set_next_avail(&mut self, next_avail: u16) {
        self.next_avail = Wrapping(next_avail);
    }

    fn next_used(&self) -> u16 {
        self.next_used.0
    }

    fn set_next_used(&mut self, next_used: u16) {
        self.next_used = Wrapping(next_used);
    }

    fn desc_table(&self) -> u64 {
        self.desc_table.0
    }

    fn avail_ring(&self) -> u64 {
        self.avail_ring.0
    }

    fn used_ring(&self) -> u64 {
        self.used_ring.0
    }

    fn event_idx_enabled(&self) -> bool {
        self.event_idx_enabled
    }

    fn pop_descriptor_chain<M>(&mut self, mem: M) -> Option<DescriptorChain<M>>
    where
        M: Clone + Deref,
        M::Target: GuestMemory,
    {
        // Default, iter-based impl. Will be subsequently improved.
        match self.iter(mem) {
            Ok(mut iter) => iter.next(),
            Err(e) => {
                error!("Iterator error {}", e);
                None
            }
        }
    }
}

impl QueueOwnedT for Queue {
    fn iter<M>(&mut self, mem: M) -> Result<AvailIter<'_, M>, Error>
    where
        M: Deref,
        M::Target: GuestMemory,
    {
        // We're checking here that a reset did not happen without re-initializing the queue.
        // TODO: In the future we might want to also check that the other parameters in the
        // queue are valid.
        if !self.ready || self.avail_ring == GuestAddress(0) {
            return Err(Error::QueueNotReady);
        }

        self.avail_idx(mem.deref(), Ordering::Acquire)
            .map(move |idx| AvailIter::new(mem, idx, self))?
    }

    fn go_to_previous_position(&mut self) {
        self.next_avail -= Wrapping(1);
    }
}

/// Consuming iterator over all available descriptor chain heads in the queue.
///
/// # Example
///
/// ```rust
/// # use virtio_bindings::bindings::virtio_ring::{VRING_DESC_F_NEXT, VRING_DESC_F_WRITE};
/// # use virtio_queue::mock::MockSplitQueue;
/// use virtio_queue::{Descriptor, Queue, QueueOwnedT};
/// use vm_memory::{GuestAddress, GuestMemoryMmap};
///
/// # fn populate_queue(m: &GuestMemoryMmap) -> Queue {
/// #    let vq = MockSplitQueue::new(m, 16);
/// #    let mut q: Queue = vq.create_queue().unwrap();
/// #
/// #    // The chains are (0, 1), (2, 3, 4) and (5, 6).
/// #    let mut descs = Vec::new();
/// #    for i in 0..7 {
/// #        let flags = match i {
/// #            1 | 6 => 0,
/// #            2 | 5 => VRING_DESC_F_NEXT | VRING_DESC_F_WRITE,
/// #            4 => VRING_DESC_F_WRITE,
/// #            _ => VRING_DESC_F_NEXT,
/// #        };
/// #
/// #        descs.push(Descriptor::new((0x1000 * (i + 1)) as u64, 0x1000, flags as u16, i + 1));
/// #    }
/// #
/// #    vq.add_desc_chains(&descs, 0).unwrap();
/// #    q
/// # }
/// let m = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
/// // Populate the queue with descriptor chains and update the available ring accordingly.
/// let mut queue = populate_queue(m);
/// let mut i = queue.iter(m).unwrap();
///
/// {
///     let mut c = i.next().unwrap();
///     let _first_head_index = c.head_index();
///     // We should have two descriptors in the first chain.
///     let _desc1 = c.next().unwrap();
///     let _desc2 = c.next().unwrap();
/// }
///
/// {
///     let c = i.next().unwrap();
///     let _second_head_index = c.head_index();
///
///     let mut iter = c.writable();
///     // We should have two writable descriptors in the second chain.
///     let _desc1 = iter.next().unwrap();
///     let _desc2 = iter.next().unwrap();
/// }
///
/// {
///     let c = i.next().unwrap();
///     let _third_head_index = c.head_index();
///
///     let mut iter = c.readable();
///     // We should have one readable descriptor in the third chain.
///     let _desc1 = iter.next().unwrap();
/// }
/// // Let's go back one position in the available ring.
/// i.go_to_previous_position();
/// // We should be able to access again the third descriptor chain.
/// let c = i.next().unwrap();
/// let _third_head_index = c.head_index();
/// ```
#[derive(Debug)]
pub struct AvailIter<'b, M> {
    mem: M,
    desc_table: GuestAddress,
    avail_ring: GuestAddress,
    queue_size: u16,
    last_index: Wrapping<u16>,
    next_avail: &'b mut Wrapping<u16>,
}

impl<'b, M> AvailIter<'b, M>
where
    M: Deref,
    M::Target: GuestMemory,
{
    /// Create a new instance of `AvailInter`.
    ///
    /// # Arguments
    /// * `mem` - the `GuestMemory` object that can be used to access the queue buffers.
    /// * `idx` - the index of the available ring entry where the driver would put the next
    ///           available descriptor chain.
    /// * `queue` - the `Queue` object from which the needed data to create the `AvailIter` can
    ///             be retrieved.
    pub(crate) fn new(mem: M, idx: Wrapping<u16>, queue: &'b mut Queue) -> Result<Self, Error> {
        // The number of descriptor chain heads to process should always
        // be smaller or equal to the queue size, as the driver should
        // never ask the VMM to process a available ring entry more than
        // once. Checking and reporting such incorrect driver behavior
        // can prevent potential hanging and Denial-of-Service from
        // happening on the VMM side.
        if (idx - queue.next_avail).0 > queue.size {
            return Err(Error::InvalidAvailRingIndex);
        }

        Ok(AvailIter {
            mem,
            desc_table: queue.desc_table,
            avail_ring: queue.avail_ring,
            queue_size: queue.size,
            last_index: idx,
            next_avail: &mut queue.next_avail,
        })
    }

    /// Goes back one position in the available descriptor chain offered by the driver.
    ///
    /// Rust does not support bidirectional iterators. This is the only way to revert the effect
    /// of an iterator increment on the queue.
    ///
    /// Note: this method assumes there's only one thread manipulating the queue, so it should only
    /// be invoked in single-threaded context.
    pub fn go_to_previous_position(&mut self) {
        *self.next_avail -= Wrapping(1);
    }
}

impl<'b, M> Iterator for AvailIter<'b, M>
where
    M: Clone + Deref,
    M::Target: GuestMemory,
{
    type Item = DescriptorChain<M>;

    fn next(&mut self) -> Option<Self::Item> {
        if *self.next_avail == self.last_index {
            return None;
        }

        // These two operations can not overflow an u64 since they're working with relatively small
        // numbers compared to u64::MAX.
        let elem_off =
            u64::from(self.next_avail.0.checked_rem(self.queue_size)?) * VIRTQ_AVAIL_ELEMENT_SIZE;
        let offset = VIRTQ_AVAIL_RING_HEADER_SIZE + elem_off;

        let addr = self.avail_ring.checked_add(offset)?;
        let head_index: u16 = self
            .mem
            .load(addr, Ordering::Acquire)
            .map(u16::from_le)
            .map_err(|_| error!("Failed to read from memory {:x}", addr.raw_value()))
            .ok()?;

        *self.next_avail += Wrapping(1);

        Some(DescriptorChain::new(
            self.mem.clone(),
            self.desc_table,
            self.queue_size,
            head_index,
        ))
    }
}

#[cfg(any(test, feature = "test-utils"))]
// It is convenient for tests to implement `PartialEq`, but it is not a
// proper implementation as `GuestMemory` errors cannot implement `PartialEq`.
impl PartialEq for Error {
    fn eq(&self, other: &Self) -> bool {
        format!("{}", &self) == format!("{}", other)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::defs::{DEFAULT_AVAIL_RING_ADDR, DEFAULT_DESC_TABLE_ADDR, DEFAULT_USED_RING_ADDR};
    use crate::mock::MockSplitQueue;
    use crate::Descriptor;
    use virtio_bindings::bindings::virtio_ring::{
        VRING_DESC_F_NEXT, VRING_DESC_F_WRITE, VRING_USED_F_NO_NOTIFY,
    };

    use vm_memory::{Address, Bytes, GuestAddress, GuestMemoryMmap};

    #[test]
    fn test_queue_is_valid() {
        let m = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
        let vq = MockSplitQueue::new(m, 16);
        let mut q: Queue = vq.create_queue().unwrap();

        // q is currently valid
        assert!(q.is_valid(m));

        // shouldn't be valid when not marked as ready
        q.set_ready(false);
        assert!(!q.ready());
        assert!(!q.is_valid(m));
        q.set_ready(true);

        // shouldn't be allowed to set a size > max_size
        q.set_size(q.max_size() << 1);
        assert_eq!(q.size, q.max_size());

        // or set the size to 0
        q.set_size(0);
        assert_eq!(q.size, q.max_size());

        // or set a size which is not a power of 2
        q.set_size(11);
        assert_eq!(q.size, q.max_size());

        // but should be allowed to set a size if 0 < size <= max_size and size is a power of two
        q.set_size(4);
        assert_eq!(q.size, 4);
        q.size = q.max_size();

        // shouldn't be allowed to set an address that breaks the alignment constraint
        q.set_desc_table_address(Some(0xf), None);
        assert_eq!(q.desc_table.0, vq.desc_table_addr().0);
        // should be allowed to set an aligned out of bounds address
        q.set_desc_table_address(Some(0xffff_fff0), None);
        assert_eq!(q.desc_table.0, 0xffff_fff0);
        // but shouldn't be valid
        assert!(!q.is_valid(m));
        // but should be allowed to set a valid description table address
        q.set_desc_table_address(Some(0x10), None);
        assert_eq!(q.desc_table.0, 0x10);
        assert!(q.is_valid(m));
        let addr = vq.desc_table_addr().0;
        q.set_desc_table_address(Some(addr as u32), Some((addr >> 32) as u32));

        // shouldn't be allowed to set an address that breaks the alignment constraint
        q.set_avail_ring_address(Some(0x1), None);
        assert_eq!(q.avail_ring.0, vq.avail_addr().0);
        // should be allowed to set an aligned out of bounds address
        q.set_avail_ring_address(Some(0xffff_fffe), None);
        assert_eq!(q.avail_ring.0, 0xffff_fffe);
        // but shouldn't be valid
        assert!(!q.is_valid(m));
        // but should be allowed to set a valid available ring address
        q.set_avail_ring_address(Some(0x2), None);
        assert_eq!(q.avail_ring.0, 0x2);
        assert!(q.is_valid(m));
        let addr = vq.avail_addr().0;
        q.set_avail_ring_address(Some(addr as u32), Some((addr >> 32) as u32));

        // shouldn't be allowed to set an address that breaks the alignment constraint
        q.set_used_ring_address(Some(0x3), None);
        assert_eq!(q.used_ring.0, vq.used_addr().0);
        // should be allowed to set an aligned out of bounds address
        q.set_used_ring_address(Some(0xffff_fffc), None);
        assert_eq!(q.used_ring.0, 0xffff_fffc);
        // but shouldn't be valid
        assert!(!q.is_valid(m));
        // but should be allowed to set a valid used ring address
        q.set_used_ring_address(Some(0x4), None);
        assert_eq!(q.used_ring.0, 0x4);
        let addr = vq.used_addr().0;
        q.set_used_ring_address(Some(addr as u32), Some((addr >> 32) as u32));
        assert!(q.is_valid(m));
    }

    #[test]
    fn test_add_used() {
        let mem = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
        let vq = MockSplitQueue::new(mem, 16);
        let mut q: Queue = vq.create_queue().unwrap();

        assert_eq!(q.used_idx(mem, Ordering::Acquire).unwrap(), Wrapping(0));
        assert_eq!(u16::from_le(vq.used().idx().load()), 0);

        // index too large
        assert!(q.add_used(mem, 16, 0x1000).is_err());
        assert_eq!(u16::from_le(vq.used().idx().load()), 0);

        // should be ok
        q.add_used(mem, 1, 0x1000).unwrap();
        assert_eq!(q.next_used, Wrapping(1));
        assert_eq!(q.used_idx(mem, Ordering::Acquire).unwrap(), Wrapping(1));
        assert_eq!(u16::from_le(vq.used().idx().load()), 1);

        let x = vq.used().ring().ref_at(0).unwrap().load();
        assert_eq!(x.id(), 1);
        assert_eq!(x.len(), 0x1000);
    }

    #[test]
    fn test_reset_queue() {
        let m = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
        let vq = MockSplitQueue::new(m, 16);
        let mut q: Queue = vq.create_queue().unwrap();

        q.set_size(8);
        // The address set by `MockSplitQueue` for the descriptor table is DEFAULT_DESC_TABLE_ADDR,
        // so let's change it for testing the reset.
        q.set_desc_table_address(Some(0x5000), None);
        // Same for `event_idx_enabled`, `next_avail` `next_used` and `signalled_used`.
        q.set_event_idx(true);
        q.set_next_avail(2);
        q.set_next_used(4);
        q.num_added = Wrapping(15);
        assert_eq!(q.size, 8);
        // `create_queue` also marks the queue as ready.
        assert!(q.ready);
        assert_ne!(q.desc_table, GuestAddress(DEFAULT_DESC_TABLE_ADDR));
        assert_ne!(q.avail_ring, GuestAddress(DEFAULT_AVAIL_RING_ADDR));
        assert_ne!(q.used_ring, GuestAddress(DEFAULT_USED_RING_ADDR));
        assert_ne!(q.next_avail, Wrapping(0));
        assert_ne!(q.next_used, Wrapping(0));
        assert_ne!(q.num_added, Wrapping(0));
        assert!(q.event_idx_enabled);

        q.reset();
        assert_eq!(q.size, 16);
        assert!(!q.ready);
        assert_eq!(q.desc_table, GuestAddress(DEFAULT_DESC_TABLE_ADDR));
        assert_eq!(q.avail_ring, GuestAddress(DEFAULT_AVAIL_RING_ADDR));
        assert_eq!(q.used_ring, GuestAddress(DEFAULT_USED_RING_ADDR));
        assert_eq!(q.next_avail, Wrapping(0));
        assert_eq!(q.next_used, Wrapping(0));
        assert_eq!(q.num_added, Wrapping(0));
        assert!(!q.event_idx_enabled);
    }

    #[test]
    fn test_needs_notification() {
        let mem = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
        let qsize = 16;
        let vq = MockSplitQueue::new(mem, qsize);
        let mut q: Queue = vq.create_queue().unwrap();
        let avail_addr = vq.avail_addr();

        // It should always return true when EVENT_IDX isn't enabled.
        for i in 0..qsize {
            q.next_used = Wrapping(i);
            assert!(q.needs_notification(mem).unwrap());
        }

        mem.write_obj::<u16>(
            u16::to_le(4),
            avail_addr.unchecked_add(4 + qsize as u64 * 2),
        )
        .unwrap();
        q.set_event_idx(true);

        // Incrementing up to this value causes an `u16` to wrap back to 0.
        let wrap = u32::from(u16::MAX) + 1;

        for i in 0..wrap + 12 {
            q.next_used = Wrapping(i as u16);
            // Let's test wrapping around the maximum index value as well.
            // `num_added` needs to be at least `1` to represent the fact that new descriptor
            // chains have be added to the used ring since the last time `needs_notification`
            // returned.
            q.num_added = Wrapping(1);
            let expected = i == 5 || i == (5 + wrap);
            assert_eq!((q.needs_notification(mem).unwrap(), i), (expected, i));
        }

        mem.write_obj::<u16>(
            u16::to_le(8),
            avail_addr.unchecked_add(4 + qsize as u64 * 2),
        )
        .unwrap();

        // Returns `false` because the current `used_event` value is behind both `next_used` and
        // the value of `next_used` at the time when `needs_notification` last returned (which is
        // computed based on `num_added` as described in the comments for `needs_notification`.
        assert!(!q.needs_notification(mem).unwrap());

        mem.write_obj::<u16>(
            u16::to_le(15),
            avail_addr.unchecked_add(4 + qsize as u64 * 2),
        )
        .unwrap();

        q.num_added = Wrapping(1);
        assert!(!q.needs_notification(mem).unwrap());

        q.next_used = Wrapping(15);
        q.num_added = Wrapping(1);
        assert!(!q.needs_notification(mem).unwrap());

        q.next_used = Wrapping(16);
        q.num_added = Wrapping(1);
        assert!(q.needs_notification(mem).unwrap());

        // Calling `needs_notification` again immediately returns `false`.
        assert!(!q.needs_notification(mem).unwrap());

        mem.write_obj::<u16>(
            u16::to_le(u16::MAX - 3),
            avail_addr.unchecked_add(4 + qsize as u64 * 2),
        )
        .unwrap();
        q.next_used = Wrapping(u16::MAX - 2);
        q.num_added = Wrapping(1);
        // Returns `true` because, when looking at circular sequence of indices of the used ring,
        // the value we wrote in the `used_event` appears between the "old" value of `next_used`
        // (i.e. `next_used` - `num_added`) and the current `next_used`, thus suggesting that we
        // need to notify the driver.
        assert!(q.needs_notification(mem).unwrap());
    }

    #[test]
    fn test_enable_disable_notification() {
        let mem = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
        let vq = MockSplitQueue::new(mem, 16);

        let mut q: Queue = vq.create_queue().unwrap();
        let used_addr = vq.used_addr();

        assert!(!q.event_idx_enabled);

        q.enable_notification(mem).unwrap();
        let v = mem.read_obj::<u16>(used_addr).map(u16::from_le).unwrap();
        assert_eq!(v, 0);

        q.disable_notification(mem).unwrap();
        let v = mem.read_obj::<u16>(used_addr).map(u16::from_le).unwrap();
        assert_eq!(v, VRING_USED_F_NO_NOTIFY as u16);

        q.enable_notification(mem).unwrap();
        let v = mem.read_obj::<u16>(used_addr).map(u16::from_le).unwrap();
        assert_eq!(v, 0);

        q.set_event_idx(true);
        let avail_addr = vq.avail_addr();
        mem.write_obj::<u16>(u16::to_le(2), avail_addr.unchecked_add(2))
            .unwrap();

        assert!(q.enable_notification(mem).unwrap());
        q.next_avail = Wrapping(2);
        assert!(!q.enable_notification(mem).unwrap());

        mem.write_obj::<u16>(u16::to_le(8), avail_addr.unchecked_add(2))
            .unwrap();

        assert!(q.enable_notification(mem).unwrap());
        q.next_avail = Wrapping(8);
        assert!(!q.enable_notification(mem).unwrap());
    }

    #[test]
    fn test_consume_chains_with_notif() {
        let mem = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
        let vq = MockSplitQueue::new(mem, 16);

        let mut q: Queue = vq.create_queue().unwrap();

        // q is currently valid.
        assert!(q.is_valid(mem));

        // The chains are (0, 1), (2, 3, 4), (5, 6), (7, 8), (9, 10, 11, 12).
        let mut descs = Vec::new();
        for i in 0..13 {
            let flags = match i {
                1 | 4 | 6 | 8 | 12 => 0,
                _ => VRING_DESC_F_NEXT,
            };

            descs.push(Descriptor::new(
                (0x1000 * (i + 1)) as u64,
                0x1000,
                flags as u16,
                i + 1,
            ));
        }

        vq.add_desc_chains(&descs, 0).unwrap();
        // Update the index of the chain that can be consumed to not be the last one.
        // This enables us to consume chains in multiple iterations as opposed to consuming
        // all the driver written chains at once.
        vq.avail().idx().store(u16::to_le(2));
        // No descriptor chains are consumed at this point.
        assert_eq!(q.next_avail(), 0);

        let mut i = 0;

        loop {
            i += 1;
            q.disable_notification(mem).unwrap();

            while let Some(chain) = q.iter(mem).unwrap().next() {
                // Process the descriptor chain, and then add entries to the
                // used ring.
                let head_index = chain.head_index();
                let mut desc_len = 0;
                chain.for_each(|d| {
                    if d.flags() as u32 & VRING_DESC_F_WRITE == VRING_DESC_F_WRITE {
                        desc_len += d.len();
                    }
                });
                q.add_used(mem, head_index, desc_len).unwrap();
            }
            if !q.enable_notification(mem).unwrap() {
                break;
            }
        }
        // The chains should be consumed in a single loop iteration because there's nothing updating
        // the `idx` field of the available ring in the meantime.
        assert_eq!(i, 1);
        // The next chain that can be consumed should have index 2.
        assert_eq!(q.next_avail(), 2);
        assert_eq!(q.next_used(), 2);
        // Let the device know it can consume one more chain.
        vq.avail().idx().store(u16::to_le(3));
        i = 0;

        loop {
            i += 1;
            q.disable_notification(mem).unwrap();

            while let Some(chain) = q.iter(mem).unwrap().next() {
                // Process the descriptor chain, and then add entries to the
                // used ring.
                let head_index = chain.head_index();
                let mut desc_len = 0;
                chain.for_each(|d| {
                    if d.flags() as u32 & VRING_DESC_F_WRITE == VRING_DESC_F_WRITE {
                        desc_len += d.len();
                    }
                });
                q.add_used(mem, head_index, desc_len).unwrap();
            }

            // For the simplicity of the test we are updating here the `idx` value of the available
            // ring. Ideally this should be done on a separate thread.
            // Because of this update, the loop should be iterated again to consume the new
            // available descriptor chains.
            vq.avail().idx().store(u16::to_le(4));
            if !q.enable_notification(mem).unwrap() {
                break;
            }
        }
        assert_eq!(i, 2);
        // The next chain that can be consumed should have index 4.
        assert_eq!(q.next_avail(), 4);
        assert_eq!(q.next_used(), 4);

        // Set an `idx` that is bigger than the number of entries added in the ring.
        // This is an allowed scenario, but the indexes of the chain will have unexpected values.
        vq.avail().idx().store(u16::to_le(7));
        loop {
            q.disable_notification(mem).unwrap();

            while let Some(chain) = q.iter(mem).unwrap().next() {
                // Process the descriptor chain, and then add entries to the
                // used ring.
                let head_index = chain.head_index();
                let mut desc_len = 0;
                chain.for_each(|d| {
                    if d.flags() as u32 & VRING_DESC_F_WRITE == VRING_DESC_F_WRITE {
                        desc_len += d.len();
                    }
                });
                q.add_used(mem, head_index, desc_len).unwrap();
            }
            if !q.enable_notification(mem).unwrap() {
                break;
            }
        }
        assert_eq!(q.next_avail(), 7);
        assert_eq!(q.next_used(), 7);
    }

    #[test]
    fn test_invalid_avail_idx() {
        // This is a negative test for the following MUST from the spec: `A driver MUST NOT
        // decrement the available idx on a virtqueue (ie. there is no way to “unexpose” buffers).`.
        // We validate that for this misconfiguration, the device does not panic.
        let mem = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
        let vq = MockSplitQueue::new(mem, 16);

        let mut q: Queue = vq.create_queue().unwrap();

        // q is currently valid.
        assert!(q.is_valid(mem));

        // The chains are (0, 1), (2, 3, 4), (5, 6).
        let mut descs = Vec::new();
        for i in 0..7 {
            let flags = match i {
                1 | 4 | 6 => 0,
                _ => VRING_DESC_F_NEXT,
            };

            descs.push(Descriptor::new(
                (0x1000 * (i + 1)) as u64,
                0x1000,
                flags as u16,
                i + 1,
            ));
        }

        vq.add_desc_chains(&descs, 0).unwrap();
        // Let the device know it can consume chains with the index < 2.
        vq.avail().idx().store(u16::to_le(3));
        // No descriptor chains are consumed at this point.
        assert_eq!(q.next_avail(), 0);
        assert_eq!(q.next_used(), 0);

        loop {
            q.disable_notification(mem).unwrap();

            while let Some(chain) = q.iter(mem).unwrap().next() {
                // Process the descriptor chain, and then add entries to the
                // used ring.
                let head_index = chain.head_index();
                let mut desc_len = 0;
                chain.for_each(|d| {
                    if d.flags() as u32 & VRING_DESC_F_WRITE == VRING_DESC_F_WRITE {
                        desc_len += d.len();
                    }
                });
                q.add_used(mem, head_index, desc_len).unwrap();
            }
            if !q.enable_notification(mem).unwrap() {
                break;
            }
        }
        // The next chain that can be consumed should have index 3.
        assert_eq!(q.next_avail(), 3);
        assert_eq!(q.avail_idx(mem, Ordering::Acquire).unwrap(), Wrapping(3));
        assert_eq!(q.next_used(), 3);
        assert_eq!(q.used_idx(mem, Ordering::Acquire).unwrap(), Wrapping(3));
        assert!(q.lock().ready());

        // Decrement `idx` which should be forbidden. We don't enforce this thing, but we should
        // test that we don't panic in case the driver decrements it.
        vq.avail().idx().store(u16::to_le(1));
        // Invalid available ring index
        assert!(q.iter(mem).is_err());
    }

    #[test]
    fn test_iterator_and_avail_idx() {
        // This test ensures constructing a descriptor chain iterator succeeds
        // with valid available ring indexes while produces an error with invalid
        // indexes.
        let queue_size = 2;
        let mem = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
        let vq = MockSplitQueue::new(mem, queue_size);

        let mut q: Queue = vq.create_queue().unwrap();

        // q is currently valid.
        assert!(q.is_valid(mem));

        // Create descriptors to fill up the queue
        let mut descs = Vec::new();
        for i in 0..queue_size {
            descs.push(Descriptor::new(
                (0x1000 * (i + 1)) as u64,
                0x1000,
                0_u16,
                i + 1,
            ));
        }
        vq.add_desc_chains(&descs, 0).unwrap();

        // Set the 'next_available' index to 'u16:MAX' to test the wrapping scenarios
        q.set_next_avail(u16::MAX);

        // When the number of chains exposed by the driver is equal to or less than the queue
        // size, the available ring index is valid and constructs an iterator successfully.
        let avail_idx = Wrapping(q.next_avail()) + Wrapping(queue_size);
        vq.avail().idx().store(u16::to_le(avail_idx.0));
        assert!(q.iter(mem).is_ok());
        let avail_idx = Wrapping(q.next_avail()) + Wrapping(queue_size - 1);
        vq.avail().idx().store(u16::to_le(avail_idx.0));
        assert!(q.iter(mem).is_ok());

        // When the number of chains exposed by the driver is larger than the queue size, the
        // available ring index is invalid and produces an error from constructing an iterator.
        let avail_idx = Wrapping(q.next_avail()) + Wrapping(queue_size + 1);
        vq.avail().idx().store(u16::to_le(avail_idx.0));
        assert!(q.iter(mem).is_err());
    }

    #[test]
    fn test_descriptor_and_iterator() {
        let m = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
        let vq = MockSplitQueue::new(m, 16);

        let mut q: Queue = vq.create_queue().unwrap();

        // q is currently valid
        assert!(q.is_valid(m));

        // the chains are (0, 1), (2, 3, 4) and (5, 6)
        let mut descs = Vec::new();
        for j in 0..7 {
            let flags = match j {
                1 | 6 => 0,
                2 | 5 => VRING_DESC_F_NEXT | VRING_DESC_F_WRITE,
                4 => VRING_DESC_F_WRITE,
                _ => VRING_DESC_F_NEXT,
            };

            descs.push(Descriptor::new(
                (0x1000 * (j + 1)) as u64,
                0x1000,
                flags as u16,
                j + 1,
            ));
        }

        vq.add_desc_chains(&descs, 0).unwrap();

        let mut i = q.iter(m).unwrap();

        {
            let c = i.next().unwrap();
            assert_eq!(c.head_index(), 0);

            let mut iter = c;
            assert!(iter.next().is_some());
            assert!(iter.next().is_some());
            assert!(iter.next().is_none());
            assert!(iter.next().is_none());
        }

        {
            let c = i.next().unwrap();
            assert_eq!(c.head_index(), 2);

            let mut iter = c.writable();
            assert!(iter.next().is_some());
            assert!(iter.next().is_some());
            assert!(iter.next().is_none());
            assert!(iter.next().is_none());
        }

        {
            let c = i.next().unwrap();
            assert_eq!(c.head_index(), 5);

            let mut iter = c.readable();
            assert!(iter.next().is_some());
            assert!(iter.next().is_none());
            assert!(iter.next().is_none());
        }
    }

    #[test]
    fn test_iterator() {
        let m = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
        let vq = MockSplitQueue::new(m, 16);

        let mut q: Queue = vq.create_queue().unwrap();

        q.size = q.max_size;
        q.desc_table = vq.desc_table_addr();
        q.avail_ring = vq.avail_addr();
        q.used_ring = vq.used_addr();
        assert!(q.is_valid(m));

        {
            // an invalid queue should return an iterator with no next
            q.ready = false;
            assert!(q.iter(m).is_err());
        }

        q.ready = true;

        // now let's create two simple descriptor chains
        // the chains are (0, 1) and (2, 3, 4)
        {
            let mut descs = Vec::new();
            for j in 0..5u16 {
                let flags = match j {
                    1 | 4 => 0,
                    _ => VRING_DESC_F_NEXT,
                };

                descs.push(Descriptor::new(
                    (0x1000 * (j + 1)) as u64,
                    0x1000,
                    flags as u16,
                    j + 1,
                ));
            }
            vq.add_desc_chains(&descs, 0).unwrap();

            let mut i = q.iter(m).unwrap();

            {
                let mut c = i.next().unwrap();
                assert_eq!(c.head_index(), 0);

                c.next().unwrap();
                assert!(c.next().is_some());
                assert!(c.next().is_none());
                assert_eq!(c.head_index(), 0);
            }

            {
                let mut c = i.next().unwrap();
                assert_eq!(c.head_index(), 2);

                c.next().unwrap();
                c.next().unwrap();
                c.next().unwrap();
                assert!(c.next().is_none());
                assert_eq!(c.head_index(), 2);
            }

            // also test go_to_previous_position() works as expected
            {
                assert!(i.next().is_none());
                i.go_to_previous_position();
                let mut c = q.iter(m).unwrap().next().unwrap();
                c.next().unwrap();
                c.next().unwrap();
                c.next().unwrap();
                assert!(c.next().is_none());
            }
        }

        // Test that iterating some broken descriptor chain does not exceed
        // 2^32 bytes in total (VIRTIO spec version 1.2, 2.7.5.2:
        // Drivers MUST NOT add a descriptor chain longer than 2^32 bytes in
        // total)
        {
            let descs = vec![
                Descriptor::new(0x1000, 0xffff_ffff, VRING_DESC_F_NEXT as u16, 1),
                Descriptor::new(0x1000, 0x1234_5678, 0, 2),
            ];
            vq.add_desc_chains(&descs, 0).unwrap();
            let mut yielded_bytes_by_iteration = 0_u32;
            for d in q.iter(m).unwrap().next().unwrap() {
                yielded_bytes_by_iteration = yielded_bytes_by_iteration
                    .checked_add(d.len())
                    .expect("iterator should not yield more than 2^32 bytes");
            }
        }

        // Same as above, but test with a descriptor which is self-referential
        {
            let descs = vec![Descriptor::new(
                0x1000,
                0xffff_ffff,
                VRING_DESC_F_NEXT as u16,
                0,
            )];
            vq.add_desc_chains(&descs, 0).unwrap();
            let mut yielded_bytes_by_iteration = 0_u32;
            for d in q.iter(m).unwrap().next().unwrap() {
                yielded_bytes_by_iteration = yielded_bytes_by_iteration
                    .checked_add(d.len())
                    .expect("iterator should not yield more than 2^32 bytes");
            }
        }
    }

    #[test]
    fn test_regression_iterator_division() {
        // This is a regression test that tests that the iterator does not try to divide
        // by 0 when the queue size is 0
        let m = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
        let vq = MockSplitQueue::new(m, 1);
        // This input was generated by the fuzzer, both for the QueueS and the Descriptor
        let descriptors: Vec<Descriptor> = vec![Descriptor::new(
            14178673876262995140,
            3301229764,
            50372,
            50372,
        )];
        vq.build_desc_chain(&descriptors).unwrap();

        let mut q = Queue {
            max_size: 38,
            next_avail: Wrapping(0),
            next_used: Wrapping(0),
            event_idx_enabled: false,
            num_added: Wrapping(0),
            size: 0,
            ready: false,
            desc_table: GuestAddress(12837708984796196),
            avail_ring: GuestAddress(0),
            used_ring: GuestAddress(9943947977301164032),
        };

        assert!(q.pop_descriptor_chain(m).is_none());
    }

    #[test]
    fn test_setters_error_cases() {
        assert_eq!(Queue::new(15).unwrap_err(), Error::InvalidMaxSize);
        let mut q = Queue::new(16).unwrap();

        let expected_val = q.desc_table.0;
        assert_eq!(
            q.try_set_desc_table_address(GuestAddress(0xf)).unwrap_err(),
            Error::InvalidDescTableAlign
        );
        assert_eq!(q.desc_table(), expected_val);

        let expected_val = q.avail_ring.0;
        assert_eq!(
            q.try_set_avail_ring_address(GuestAddress(0x1)).unwrap_err(),
            Error::InvalidAvailRingAlign
        );
        assert_eq!(q.avail_ring(), expected_val);

        let expected_val = q.used_ring.0;
        assert_eq!(
            q.try_set_used_ring_address(GuestAddress(0x3)).unwrap_err(),
            Error::InvalidUsedRingAlign
        );
        assert_eq!(q.used_ring(), expected_val);

        let expected_val = q.size;
        assert_eq!(q.try_set_size(15).unwrap_err(), Error::InvalidSize);
        assert_eq!(q.size(), expected_val)
    }

    #[test]
    // This is a regression test for a fuzzing finding. If the driver requests a reset of the
    // device, but then does not re-initializes the queue then a subsequent call to process
    // a request should yield no descriptors to process. Before this fix we were processing
    // descriptors that were added to the queue before, and we were ending up processing 255
    // descriptors per chain.
    fn test_regression_timeout_after_reset() {
        // The input below was generated by libfuzzer and adapted for this test.
        let m = &GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0x0), 0x10000)]).unwrap();
        let vq = MockSplitQueue::new(m, 1024);

        // This input below was generated by the fuzzer.
        let descriptors: Vec<Descriptor> = vec![
            Descriptor::new(21508325467, 0, 1, 4),
            Descriptor::new(2097152, 4096, 3, 0),
            Descriptor::new(18374686479672737792, 4294967295, 65535, 29),
            Descriptor::new(76842670169653248, 1114115, 0, 0),
            Descriptor::new(16, 983040, 126, 3),
            Descriptor::new(897648164864, 0, 0, 0),
            Descriptor::new(111669149722, 0, 0, 0),
        ];
        vq.build_multiple_desc_chains(&descriptors).unwrap();

        let mut q: Queue = vq.create_queue().unwrap();

        // Setting the queue to ready should not allow consuming descriptors after reset.
        q.reset();
        q.set_ready(true);
        let mut counter = 0;
        while let Some(mut desc_chain) = q.pop_descriptor_chain(m) {
            // this empty loop is here to check that there are no side effects
            // in terms of memory & execution time.
            while desc_chain.next().is_some() {
                counter += 1;
            }
        }
        assert_eq!(counter, 0);

        // Setting the avail_addr to valid should not allow consuming descriptors after reset.
        q.reset();
        q.set_avail_ring_address(Some(0x1000), None);
        assert_eq!(q.avail_ring, GuestAddress(0x1000));
        counter = 0;
        while let Some(mut desc_chain) = q.pop_descriptor_chain(m) {
            // this empty loop is here to check that there are no side effects
            // in terms of memory & execution time.
            while desc_chain.next().is_some() {
                counter += 1;
            }
        }
        assert_eq!(counter, 0);
    }
}

Coverage Report

Created: 2024-05-21 06:19