reactos/drivers/storage/ide/atapi/scsi.c

/*
 * PROJECT:     ReactOS ATA Port Driver
 * LICENSE:     MIT (https://spdx.org/licenses/MIT)
 * PURPOSE:     SCSI I/O queue and requests handling
 * COPYRIGHT:   Copyright 2026 Dmitry Borisov <[email protected]>
 */

/* INCLUDES *******************************************************************/

#include "atapi.h"

/* GLOBALS ********************************************************************/

SLIST_HEADER AtapCompletionQueueList;
KDPC AtapCompletionDpc;

static DRIVER_CANCEL AtaReqDeviceQueueCancelIo;
static DRIVER_LIST_CONTROL AtaReqPreparePrdTable;
static DRIVER_CONTROL AtaReqCallSendRequestSerialized;

static
NTSTATUS
AtaReqDeviceQueueAddSrb(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PSCSI_REQUEST_BLOCK Srb);

static
VOID
AtaReqDispatchRequest(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PATA_DEVICE_REQUEST Request,
    _In_ BOOLEAN DoReleaseDeviceQueueLock);

static
BOOLEAN
AtaReqDeviceQueueDispatchNextRequest(
    _In_ PATAPORT_IO_CONTEXT Device);

/* FUNCTIONS ******************************************************************/

static
NTSTATUS
AtaSrbStatusToNtStatus(
    _In_ UCHAR SrbStatus)
{
    ASSERT(SRB_STATUS(SrbStatus) != SRB_STATUS_PENDING);

    /* We translate only the values that are actually used by the driver */
    switch (SRB_STATUS(SrbStatus))
    {
        case SRB_STATUS_SUCCESS:
            return STATUS_SUCCESS;

        case SRB_STATUS_TIMEOUT:
            return STATUS_IO_TIMEOUT;

        case SRB_STATUS_NO_DEVICE:
            return STATUS_DEVICE_DOES_NOT_EXIST;

        case SRB_STATUS_DATA_OVERRUN:
            return STATUS_BUFFER_OVERFLOW;

        case SRB_STATUS_SELECTION_TIMEOUT:
            return STATUS_DEVICE_NOT_CONNECTED;

        default:
            break;
    }

    return STATUS_IO_DEVICE_ERROR;
}

static
VOID
AtaDeviceCheckPowerState(
    _In_ PATAPORT_IO_CONTEXT Device)
{
    PATAPORT_DEVICE_EXTENSION DevExt;
    NTSTATUS Status;
    POWER_STATE PowerState;

   /*
    * The device might have been powered down as a part of an idle detection logic,
    * so we have to power up the device again.
    */
    if (!(Device->QueueFlags & QUEUE_FLAG_FROZEN_POWER))
        return;

    DevExt = CONTAINING_RECORD(Device, ATAPORT_DEVICE_EXTENSION, Device);
    if (DevExt->Common.DevicePowerState == PowerDeviceD0)
        return;

    if (DevExt->Common.SystemPowerState != PowerSystemWorking)
        return;

    INFO("Powering up idle device\n");

    PowerState.DeviceState = PowerDeviceD0;
    Status = PoRequestPowerIrp(DevExt->Common.Self,
                               IRP_MN_SET_POWER,
                               PowerState,
                               NULL,
                               NULL,
                               NULL);
    if (!NT_SUCCESS(Status))
    {
        ERR("Failed to power up device '%s' %lx\n", DevExt->FriendlyName, Status);
    }
}

static
VOID
AtaDeviceQueueEmptyEvent(
    _In_ PATAPORT_IO_CONTEXT Device)
{
    PSCSI_REQUEST_BLOCK Srb;

    KeSetEvent(&Device->QueueStoppedEvent, 0, FALSE);

    Srb = Device->QuiescenceSrb;
    if (Srb)
    {
        PIRP Irp = Srb->OriginalRequest;

        Device->QuiescenceSrb = NULL;

        KeReleaseSpinLockFromDpcLevel(&Device->QueueLock);

        ASSERT(Srb->OriginalRequest == Irp);

        Srb->SrbStatus = SRB_STATUS_SUCCESS;
        Irp->IoStatus.Status = STATUS_SUCCESS;
        IoCompleteRequest(Irp, IO_DISK_INCREMENT);

        KeAcquireSpinLockAtDpcLevel(&Device->QueueLock);
    }
}

static
NTSTATUS
AtaReqRequeueRequest(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PATA_DEVICE_REQUEST Request,
    _Out_ PUCHAR ResultSrbStatus)
{
    UCHAR SrbStatus;

    SrbStatus = SRB_STATUS(*ResultSrbStatus);

    /* Only requeue busy requests */
    if ((SrbStatus != SRB_STATUS_BUSY) && (SrbStatus != SRB_STATUS_INSUFFICIENT_RESOURCES))
        return COMPLETE_IRP;

    /* Retry at least once if the request was rejected due to lack of resources */
    if (SrbStatus == SRB_STATUS_INSUFFICIENT_RESOURCES)
    {
        if (SRB_GET_FLAGS(Request->Srb) & SRB_FLAG_LOW_MEM_RETRY)
        {
            /* We have no chance to dispatch it */
            return COMPLETE_IRP;
        }

        WARN("Retrying operation\n");

        SRB_SET_FLAGS(Request->Srb, SRB_FLAG_LOW_MEM_RETRY);
    }

    ASSERT(!(Request->Flags & (REQUEST_FLAG_HAS_SG_LIST |
                               REQUEST_FLAG_HAS_MDL |
                               REQUEST_FLAG_HAS_RESERVED_MAPPING)));

    /* Place the Srb back into the queue */
    if (AtaReqDeviceQueueAddSrb(Device, Request->Srb) != STATUS_PENDING)
    {
        /* We failed because of the IRP was cancelled, just update the status */
        *ResultSrbStatus = Request->Srb->SrbStatus;

        return COMPLETE_IRP;
    }

    return COMPLETE_NO_IRP;
}

_Requires_lock_held_(PortData->QueueLock)
static
BOOLEAN
AtaReqPortQueueListDispatchNextRequest(
    _In_ PATAPORT_PORT_DATA PortData)
{
    PATAPORT_IO_CONTEXT Device;
    PATA_DEVICE_REQUEST Request;
    PLIST_ENTRY Entry;

    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);

    if (IsListEmpty(&PortData->PortQueueList))
        return FALSE;

    Entry = RemoveHeadList(&PortData->PortQueueList);

    Request = CONTAINING_RECORD(Entry, ATA_DEVICE_REQUEST, PortEntry);
    ASSERT_REQUEST(Request);

    Device = (PATAPORT_IO_CONTEXT)Request->Device;

    /* Try to unfreeze the device queue */
    Device->QueueFlags &= ~QUEUE_FLAG_FROZEN_SLOT;
    AtaReqDispatchRequest(Device, Request, FALSE);

    return TRUE;
}

static
VOID
AtaReqReleaseResources(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PATA_DEVICE_REQUEST Request)
{
    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);

    if (Request->Flags & REQUEST_FLAG_HAS_SG_LIST)
    {
        PDMA_ADAPTER DmaAdapter = Device->PortData->DmaAdapter;
        PDMA_OPERATIONS DmaOperations = DmaAdapter->DmaOperations;

        ASSERT(Request->SgList);
        ASSERT(Request->Flags & (REQUEST_FLAG_DATA_IN | REQUEST_FLAG_DATA_OUT));

        DmaOperations->PutScatterGatherList(DmaAdapter,
                                            Request->SgList,
                                            !!(Request->Flags & REQUEST_FLAG_DATA_IN));
    }
    else if (Request->Flags & REQUEST_FLAG_HAS_RESERVED_MAPPING)
    {
        MmUnmapReservedMapping(Device->PortData->ReservedVaSpace, ATAPORT_TAG, Request->Mdl);
        _InterlockedExchange(&Device->PortData->ReservedMappingLock, 0);
    }

    if (Request->Flags & REQUEST_FLAG_HAS_MDL)
    {
        ASSERT(Request->Mdl);

        IoFreeMdl(Request->Mdl);
    }

#if DBG
    Request->Mdl = NULL;
    Request->SgList = NULL;
#endif

    Request->Flags &= ~(REQUEST_FLAG_HAS_SG_LIST |
                        REQUEST_FLAG_HAS_MDL |
                        REQUEST_FLAG_HAS_RESERVED_MAPPING);
}

static
VOID
AtaReqCompleteRequest(
    _In_ PATA_DEVICE_REQUEST Request)
{
    PATAPORT_IO_CONTEXT Device = (PATAPORT_IO_CONTEXT)Request->Device;
    PSCSI_REQUEST_BLOCK Srb;
    PIRP Irp;
    UCHAR SrbStatus;
    ATA_COMPLETION_ACTION CompletionAction = COMPLETE_IRP;

    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);

    if (Request->InternalState != REQUEST_STATE_NOT_STARTED)
    {
        if (Request->Complete)
        {
            if (Request->Flags & REQUEST_FLAG_INTERNAL)
            {
                Request->Complete(Request);
                return;
            }
            else
            {
                KeAcquireSpinLockAtDpcLevel(&Device->QueueLock);
                CompletionAction = Request->Complete(Request);
                KeReleaseSpinLockFromDpcLevel(&Device->QueueLock);
            }
        }

        if (Device->PortData->PortFlags & PORT_FLAG_IS_SIMPLEX)
        {
            IoFreeController(Device->PortData->HwSyncObject);
        }
    }

    AtaReqReleaseResources(Device, Request);

    Srb = Request->Srb;

    Irp = Request->Irp;
    Irp->IoStatus.Information = Request->DataTransferLength;

    KeAcquireSpinLockAtDpcLevel(&Device->QueueLock);

    /* Release the request back to the device queue */
    ASSERT(!(Device->FreeRequestsBitmap & Request->Tag));
    Device->FreeRequestsBitmap |= Request->Tag;

    /* Release the slot back to the port queue */
    if (Request->InternalState != REQUEST_STATE_NOT_STARTED)
    {
        PATAPORT_PORT_DATA PortData = Device->PortData;

        KeAcquireSpinLockAtDpcLevel(&PortData->QueueLock);

        /* Stop the timer tied to the completed slot */
        PortData->ActiveTimersBitmap &= ~(1 << Request->Slot);

        /* Release exclusive access to the port queue */
        if (Request->Flags & REQUEST_EXCLUSIVE_ACCESS_FLAGS)
            PortData->QueueFlags &= ~PORT_QUEUE_FLAG_EXCLUSIVE_MODE;

        /* Release the slot */
        ASSERT(!(PortData->FreeSlotsBitmap & (1 << Request->Slot)));
        PortData->FreeSlotsBitmap |= (1 << Request->Slot);
#if DBG
        PortData->Slots[Request->Slot] = NULL;
#endif
        if (Request->Flags & REQUEST_FLAG_NCQ)
            --PortData->AllocatedSlots;
        else
            ++PortData->AllocatedSlots;

        PortData->AllocateSlot(PortData->ChannelContext, Request, FALSE);

        if ((PortData->QueueFlags & PORT_QUEUE_FLAG_SIGNAL_STOP) && AtaPortQueueEmpty(PortData))
        {
            PortData->QueueFlags &= ~PORT_QUEUE_FLAG_SIGNAL_STOP;
            KeSetEvent(&PortData->QueueStoppedEvent, 0, FALSE);
        }

        /* Start the next request on the port queue */
        if (!AtaReqPortQueueListDispatchNextRequest(PortData))
            KeReleaseSpinLockFromDpcLevel(&PortData->QueueLock);
    }

    /* Check if we need to retry later */
    if (Request->InternalState == REQUEST_STATE_NOT_STARTED ||
        Request->InternalState == REQUEST_STATE_REQUEUE)
    {
        CompletionAction = AtaReqRequeueRequest(Device, Request, &Request->SrbStatus);
    }

#if DBG
    if (CompletionAction == COMPLETE_IRP)
        ++Device->Statistics.RequestsCompleted;
#endif

    /* Freeze the Srb queue in case of device error */
    if ((Request->InternalState == REQUEST_STATE_FREEZE_QUEUE) &&
        !(Srb->SrbFlags & SRB_FLAGS_NO_QUEUE_FREEZE))
    {
        ASSERT(SRB_STATUS(Request->SrbStatus) != SRB_STATUS_INSUFFICIENT_RESOURCES);

        Device->QueueFlags |= QUEUE_FLAG_FROZEN_QUEUE_FREEZE;
        Request->SrbStatus |= SRB_STATUS_QUEUE_FROZEN;
        INFO("Freeze queue\n");
    }

    /* Signal the event when the list of active IRPs is empty */
    if ((Device->QueueFlags & QUEUE_FLAG_SIGNAL_STOP) &&
        (Device->FreeRequestsBitmap == Device->MaxRequestsBitmap))
    {
        Device->QueueFlags &= ~QUEUE_FLAG_SIGNAL_STOP;
        AtaDeviceQueueEmptyEvent(Device);
    }

    ASSERT(!(Request->Flags & (REQUEST_FLAG_HAS_SG_LIST |
                               REQUEST_FLAG_HAS_MDL |
                               REQUEST_FLAG_HAS_RESERVED_MAPPING)));

    SrbStatus = Request->SrbStatus;

    /*
     * Start the next request on the device queue.
     * It's important to do this before actually completing an IRP.
     */
    if (!AtaReqDeviceQueueDispatchNextRequest(Device))
        KeReleaseSpinLockFromDpcLevel(&Device->QueueLock);

    /*
     * Complete the IRP outside of the spinlock to avoid deadlocks.
     * A new SCSI device I/O request might sent immediately after the IRP completed.
     */
    if (CompletionAction == COMPLETE_IRP)
    {
        if (SRB_STATUS(SrbStatus) == SRB_STATUS_INSUFFICIENT_RESOURCES)
        {
            /* Special status for the upper class driver */
            Srb->SrbStatus = SRB_STATUS_INTERNAL_ERROR;
            Srb->InternalStatus = STATUS_INSUFFICIENT_RESOURCES;
            Irp->IoStatus.Status = STATUS_INSUFFICIENT_RESOURCES;
        }
        else
        {
            Srb->SrbStatus = SrbStatus;
            Irp->IoStatus.Status = AtaSrbStatusToNtStatus(SrbStatus);
        }

        IoCompleteRequest(Irp, IO_DISK_INCREMENT);
    }
}

UCHAR
AtaReqSetFixedSenseData(
    _In_ PSCSI_REQUEST_BLOCK Srb,
    _In_ SCSI_SENSE_CODE SenseCode)
{
    SENSE_DATA SenseData;

    if ((Srb->SenseInfoBuffer == NULL) || (Srb->SenseInfoBufferLength == 0))
        return SenseCode.SrbStatus;

    ASSERT(!(SenseCode.SrbStatus & SRB_STATUS_AUTOSENSE_VALID));

    Srb->ScsiStatus = SCSISTAT_CHECK_CONDITION;

    // TODO: D_SENSE (fixed or descriptor format) is not supported yet
    RtlZeroMemory(&SenseData, sizeof(SenseData));
    SenseData.Valid = 1;
    SenseData.ErrorCode = SCSI_SENSE_ERRORCODE_FIXED_CURRENT;
    SenseData.SenseKey = SenseCode.SenseKey;
    SenseData.AdditionalSenseCode = SenseCode.AdditionalSenseCode;
    SenseData.AdditionalSenseCodeQualifier = SenseCode.AdditionalSenseCodeQualifier;
    SenseData.AdditionalSenseLength =
        sizeof(SenseData) - RTL_SIZEOF_THROUGH_FIELD(SENSE_DATA, AdditionalSenseLength);

    RtlCopyMemory(Srb->SenseInfoBuffer,
                  &SenseData,
                  min(Srb->SenseInfoBufferLength, sizeof(SenseData)));

    return SenseCode.SrbStatus | SRB_STATUS_AUTOSENSE_VALID;
}

VOID
AtaReqSetLbaInformation(
    _In_ PSCSI_REQUEST_BLOCK Srb,
    _In_ ULONG64 Lba)
{
    PSENSE_DATA SenseData;

    if ((Srb->SenseInfoBuffer == NULL) || (Srb->SenseInfoBufferLength == 0))
        return;

    SenseData = Srb->SenseInfoBuffer;

    if (RTL_CONTAINS_FIELD(SenseData, Srb->SenseInfoBufferLength, Information))
    {
        ASSERT(SenseData->Valid);

        SenseData->Information[0] = (UCHAR)(Lba >> 24);
        SenseData->Information[1] = (UCHAR)(Lba >> 16);
        SenseData->Information[2] = (UCHAR)(Lba >> 8);
        SenseData->Information[3] = (UCHAR)(Lba >> 0);
    }

    // TODO: D_SENSE (fixed or descriptor format) is not supported yet
}

BOOLEAN
AtaReqAllocateMdl(
    _In_ PATA_DEVICE_REQUEST Request)
{
    PMDL Mdl;

    Mdl = IoAllocateMdl(Request->DataBuffer,
                        Request->DataTransferLength,
                        FALSE,
                        FALSE,
                        NULL);
    if (!Mdl)
        return FALSE;

    MmBuildMdlForNonPagedPool(Mdl);

    Request->Mdl = Mdl;
    Request->Flags |= REQUEST_FLAG_HAS_MDL;

    return TRUE;
}

static
VOID
AtaReqStartIo(
    _In_ PATA_DEVICE_REQUEST Request)
{
    PATAPORT_IO_CONTEXT Device = (PATAPORT_IO_CONTEXT)Request->Device;
    PATAPORT_PORT_DATA PortData = Device->PortData;
    KIRQL OldIrql;

    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
    ASSERT(PortData->Slots[Request->Slot] == Request);

    PortData->PrepareIo(PortData->ChannelContext, Request);

    OldIrql = KeAcquireInterruptSpinLock(PortData->InterruptObject);

    ASSERT(!(PortData->ActiveSlotsBitmap & (1 << Request->Slot)));

    if ((PortData->InterruptFlags & PORT_INT_FLAG_IS_IO_ACTIVE) ||
        (Request->Flags & REQUEST_FLAG_INTERNAL))
    {
        PortData->ActiveSlotsBitmap |= 1 << Request->Slot;

        if (!PortData->StartIo(PortData->ChannelContext, Request))
        {
            PortData->TimerCount[Request->Slot] = Request->TimeOut;
        }

        KeReleaseInterruptSpinLock(PortData->InterruptObject, OldIrql);
    }
    else
    {
        PATAPORT_IO_CONTEXT Device = (PATAPORT_IO_CONTEXT)Request->Device;

        /*
         * Needed to prevent an infinite loop that happens
         * when the queue manager keeps retrying the SRB at dispatch level.
         */
        _InterlockedOr(&Device->QueueFlags, QUEUE_FLAG_FROZEN_PORT_BUSY);

        KeReleaseInterruptSpinLock(PortData->InterruptObject, OldIrql);

        /* If the queue is frozen, then force queue manager to requeue the SRB */
        Request->SrbStatus = SRB_STATUS_BUSY;
        Request->InternalState = REQUEST_STATE_REQUEUE;

        /* Defer the completion to a DPC to avoid the recursive call in some cases */
        AtaReqStartCompletionDpc(Request);
    }
}

static
VOID
NTAPI
AtaReqPreparePrdTable(
    _In_ PDEVICE_OBJECT DeviceObject,
    _In_ PIRP Irp,
    _In_ PSCATTER_GATHER_LIST SgList,
    _In_ PVOID Context)
{
    PATA_DEVICE_REQUEST Request = Context;
    PATAPORT_IO_CONTEXT Device = (PATAPORT_IO_CONTEXT)Request->Device;
    PATAPORT_PORT_DATA PortData = Device->PortData;

    UNREFERENCED_PARAMETER(DeviceObject);
    UNREFERENCED_PARAMETER(Irp);

    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);

    Request->SgList = SgList;
    Request->Flags |= REQUEST_FLAG_HAS_SG_LIST;

    PortData->PreparePrdTable(PortData->ChannelContext, Request, SgList);
    AtaReqStartIo(Request);
}

static
PVOID
AtaReqMapBuffer(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PATA_DEVICE_REQUEST Request)
{
    PATAPORT_PORT_DATA PortData;
    PVOID BaseAddress;
    ULONG PagesNeeded;

    BaseAddress = MmGetSystemAddressForMdlSafe(Request->Mdl, HighPagePriority);
    if (BaseAddress)
        return BaseAddress;

    PortData = Device->PortData;
    if (!PortData->ReservedVaSpace)
        return NULL;

    /* The system is low resources, handle it in a non-fatal way */
    PagesNeeded = ADDRESS_AND_SIZE_TO_SPAN_PAGES(MmGetMdlVirtualAddress(Request->Mdl),
                                                 MmGetMdlByteCount(Request->Mdl));
    if (PagesNeeded > ATA_RESERVED_PAGES)
        return NULL;

    /* Utilize the reserved mapping to overcome memory issues */
    if (!_InterlockedCompareExchange(&PortData->ReservedMappingLock, 1, 0))
    {
        BaseAddress = MmMapLockedPagesWithReservedMapping(PortData->ReservedVaSpace,
                                                          ATAPORT_TAG,
                                                          Request->Mdl,
                                                          MmCached);
        if (BaseAddress)
            Request->Flags |= REQUEST_FLAG_HAS_RESERVED_MAPPING;
        else
            _InterlockedExchange(&PortData->ReservedMappingLock, 0);
    }

    return BaseAddress;
}

static
BOOLEAN
AtaReqGetScatterGatherList(
    _In_ PATAPORT_PORT_DATA PortData,
    _In_ PATA_DEVICE_REQUEST Request)
{
    PDMA_ADAPTER DmaAdapter = PortData->DmaAdapter;
    PDMA_OPERATIONS DmaOperations = DmaAdapter->DmaOperations;
    NTSTATUS Status;

    ASSERT(Request->Mdl);

    Status = DmaOperations->GetScatterGatherList(DmaAdapter,
                                                 PortData->ChannelObject,
                                                 Request->Mdl,
                                                 Request->DataBuffer,
                                                 Request->DataTransferLength,
                                                 AtaReqPreparePrdTable,
                                                 Request,
                                                 !!(Request->Flags & REQUEST_FLAG_DATA_IN));
    if (NT_SUCCESS(Status))
        return TRUE;

    WARN("Failed to get the S/G list with status %lx\n", Status);
    return FALSE;
}

VOID
AtaReqSendRequest(
    _In_ PATA_DEVICE_REQUEST Request)
{
    PATAPORT_IO_CONTEXT Device = (PATAPORT_IO_CONTEXT)Request->Device;

    PVOID BaseAddress;
    ULONG_PTR Offset;

    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);

    /* The channel can only perform DMA I/O and PIO is not supported */
    if (Device->DeviceFlags & DEVICE_PIO_VIA_DMA)
        Request->Flags |= REQUEST_FLAG_PROGRAM_DMA;

    if (Request->Flags & REQUEST_FLAG_DMA)
        ASSERT(Request->Flags & REQUEST_FLAG_PROGRAM_DMA);

    if (!(Request->Flags & REQUEST_FLAG_NO_KEEP_AWAKE))
    {
        PULONG PowerIdleCounter = Device->PowerIdleCounter;

        if (PowerIdleCounter)
            PoSetDeviceBusy(PowerIdleCounter);
    }

    /* Local buffer transfer */
    if (Request->Flags & REQUEST_FLAG_HAS_LOCAL_BUFFER)
    {
        ASSERT(Request->Flags & REQUEST_FLAG_INTERNAL);

        if (Request->Flags & REQUEST_FLAG_PROGRAM_DMA)
        {
            /* DMA transfer */
            Device->PortData->LocalSgList.Elements[0].Length = Request->DataTransferLength;
            AtaReqPreparePrdTable(NULL, NULL, &Device->PortData->LocalSgList, Request);
        }
        else
        {
            /* PIO data transfer */
            Request->DataBuffer = Device->LocalBuffer;
            AtaReqStartIo(Request);
        }
        return;
    }

    /* No data transfer */
    if (!(Request->Flags & (REQUEST_FLAG_DATA_IN | REQUEST_FLAG_DATA_OUT)))
    {
        AtaReqStartIo(Request);
        return;
    }

    /* DMA transfer, get the S/G list for the MDL */
    if (Request->Flags & REQUEST_FLAG_PROGRAM_DMA)
    {
        if (AtaReqGetScatterGatherList(Device->PortData, Request))
            return;

        /* This channel can only perform DMA I/O and PIO is not supported */
        if (Device->DeviceFlags & DEVICE_PIO_VIA_DMA)
            goto CompleteNoMemory;

        /* S/G list construction failed, attempt to fall back to PIO mode */
        if (!AtaReqDmaTransferToPioTransfer(Request))
            goto CompleteNoMemory;
    }

    /* PIO data transfer path */
    ASSERT(!(Device->DeviceFlags & DEVICE_PIO_VIA_DMA));

    BaseAddress = AtaReqMapBuffer(Device, Request);
    if (!BaseAddress)
        goto CompleteNoMemory;

    /* Calculate the offset within DataBuffer */
    Offset = (ULONG_PTR)BaseAddress +
             (ULONG_PTR)Request->DataBuffer -
             (ULONG_PTR)MmGetMdlVirtualAddress(Request->Mdl);
    Request->DataBuffer = (PVOID)Offset;

    AtaReqStartIo(Request);
    return;

CompleteNoMemory:
    Request->SrbStatus = SRB_STATUS_INSUFFICIENT_RESOURCES;
    Request->InternalState = REQUEST_STATE_REQUEUE;

    /*
     * Defer the completion to a DPC.
     * We cannot complete request with the queue spinlock held.
     */
    AtaReqStartCompletionDpc(Request);
}

static
IO_ALLOCATION_ACTION
NTAPI
AtaReqCallSendRequestSerialized(
    _In_ PDEVICE_OBJECT DeviceObject,
    _Inout_ PIRP Irp,
    _In_ PVOID MapRegisterBase,
    _In_ PVOID Context)
{
    PATA_DEVICE_REQUEST Request = Context;

    UNREFERENCED_PARAMETER(DeviceObject);
    UNREFERENCED_PARAMETER(Irp);
    UNREFERENCED_PARAMETER(MapRegisterBase);

    AtaReqSendRequest(Request);
    return KeepObject;
}

VOID
NTAPI
AtaReqCompletionDpc(
    _In_ PKDPC Dpc,
    _In_opt_ PVOID DeferredContext,
    _In_opt_ PVOID SystemArgument1,
    _In_opt_ PVOID SystemArgument2)
{
    PSLIST_ENTRY CurrentEntry, NextEntry;

    UNREFERENCED_PARAMETER(Dpc);
    UNREFERENCED_PARAMETER(DeferredContext);
    UNREFERENCED_PARAMETER(SystemArgument1);
    UNREFERENCED_PARAMETER(SystemArgument2);

    CurrentEntry = ExInterlockedFlushSList(&AtapCompletionQueueList);
    while (CurrentEntry)
    {
        PATA_DEVICE_REQUEST Request;

        NextEntry = CurrentEntry->Next;

        Request = CONTAINING_RECORD(CurrentEntry, ATA_DEVICE_REQUEST, CompletionEntry);
        ASSERT_REQUEST(Request);

        AtaReqCompleteRequest(Request);

        CurrentEntry = NextEntry;
    }
}

VOID
AtaReqStartCompletionDpc(
    _In_ PATA_DEVICE_REQUEST Request)
{
    ASSERT_REQUEST(Request);

    InterlockedPushEntrySList(&AtapCompletionQueueList, &Request->CompletionEntry);
    KeInsertQueueDpc(&AtapCompletionDpc, NULL, NULL);
}

/*
 * The control flow is designed such that
 * we can just return FALSE without having to undo the failed slot allocation later.
 */
static
BOOLEAN
AtaReqAllocateSlot(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PATAPORT_PORT_DATA PortData,
    _In_ PATA_DEVICE_REQUEST Request)
{
    ULONG SlotMask, SlotNumber, SlotsBitmap;

    if (PortData->QueueFlags & PORT_QUEUE_FLAG_EXCLUSIVE_MODE)
        return FALSE;

    /* Gain exclusive access to the slot queue */
    if (Request->Flags & REQUEST_EXCLUSIVE_ACCESS_FLAGS)
    {
        /* Check if we have any outstanding commands */
        if (PortData->AllocatedSlots != 0)
            return FALSE;
    }

    /* Queued command */
    if (Request->Flags & REQUEST_FLAG_NCQ)
    {
        ULONG QueueDepth;

        /* Check if we have any outstanding non-queued commands */
        if (PortData->AllocatedSlots < 0)
            return FALSE;

        QueueDepth = Device->TransportFlags & DEVICE_QUEUE_DEPTH_MASK;
        QueueDepth >>= DEVICE_QUEUE_DEPTH_SHIFT;

        /* The device capacity may be less than the total HBA capacity */
        if (PortData->AllocatedSlots >= QueueDepth)
            return FALSE;
    }
    else
    {
        /* Check if we have any outstanding native queued commands */
        if (PortData->AllocatedSlots > 0)
            return FALSE;
    }

    SlotsBitmap = PortData->FreeSlotsBitmap;

    /* Mask off previously issued commands */
    SlotMask = ~(0xFFFFFFFF >> (MAX_SLOTS - (PortData->LastUsedSlot + 1)));
    if (!(SlotsBitmap & SlotMask))
        SlotMask = 0xFFFFFFFF;

    /* Allocate slot in a circular fashion. This is required to maintain CCS update */
    if (!_BitScanForward(&SlotNumber, SlotsBitmap & SlotMask))
        return FALSE;

    if (!PortData->AllocateSlot(PortData->ChannelContext, Request, TRUE))
        return FALSE;

    /* The slot can safely be consumed at this point */
    if (Request->Flags & REQUEST_FLAG_NCQ)
        ++PortData->AllocatedSlots;
    else
        --PortData->AllocatedSlots;

    if (Request->Flags & REQUEST_EXCLUSIVE_ACCESS_FLAGS)
        PortData->QueueFlags |= PORT_QUEUE_FLAG_EXCLUSIVE_MODE;

    ASSERT(PortData->LastUsedSlot < MAX_SLOTS);
    ASSERT(IsPowerOfTwo(SlotsBitmap) || (SlotNumber != PortData->LastUsedSlot));
    ASSERT(PortData->FreeSlotsBitmap & (1 << SlotNumber));
    ASSERT(PortData->Slots[SlotNumber] == NULL);

    Request->Slot = SlotNumber;

    PortData->LastUsedSlot = SlotNumber;
    PortData->FreeSlotsBitmap &= ~(1 << SlotNumber);
    PortData->Slots[SlotNumber] = Request;
    return TRUE;
}

static
VOID
AtaReqDispatchRequest(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PATA_DEVICE_REQUEST Request,
    _In_ BOOLEAN DoReleaseDeviceQueueLock)
{
    PATAPORT_PORT_DATA PortData = Device->PortData;
    BOOLEAN Success;

    Success = AtaReqAllocateSlot(Device, PortData, Request);
    if (Success)
    {
        PortData->ActiveTimersBitmap |= 1 << Request->Slot;
    }
    else
    {
        /*
         * If all slots are busy, overloading the port can starve other incoming I/O requests:
         * - An IDE channel can only deal with one active request at a time.
         * - A SATA Port Multiplier shares the bandwidth with up to 15 devices.
         * In order to avoid starvation of the device,
         * put the request on the high-priority port queue.
         */
        InsertTailList(&PortData->PortQueueList, &Request->PortEntry);

        TRACE("Freeze slot queue\n");

        /*
         * Freeze the device queue to ensure
         * that only one request per device is queued to the port queue.
         */
        ASSERT(!(Device->QueueFlags & QUEUE_FLAG_FROZEN_SLOT));
        Device->QueueFlags |= QUEUE_FLAG_FROZEN_SLOT;
    }

    KeReleaseSpinLockFromDpcLevel(&PortData->QueueLock);

    if (DoReleaseDeviceQueueLock)
        KeReleaseSpinLockFromDpcLevel(&Device->QueueLock);

    if (Success)
    {
        if (PortData->PortFlags & PORT_FLAG_IS_SIMPLEX)
        {
            IoAllocateController(PortData->HwSyncObject,
                                 PortData->ChannelObject,
                                 AtaReqCallSendRequestSerialized,
                                 Request);
        }
        else
        {
            AtaReqSendRequest(Request);
        }
    }
}

static
ULONG
AtaReqTranslateRequest(
    _In_ PATAPORT_DEVICE_EXTENSION DevExt,
    _In_ PATA_DEVICE_REQUEST Request,
    _In_ PSCSI_REQUEST_BLOCK Srb)
{
    UCHAR SrbStatus;

    Request->Srb = Srb;
    Request->DataBuffer = Srb->DataBuffer;
    Request->DataTransferLength = Srb->DataTransferLength;
    Request->Irp = Srb->OriginalRequest;
    Request->Mdl = ((PIRP)(Srb->OriginalRequest))->MdlAddress;
    Request->TimeOut = Srb->TimeOutValue;
    Request->Complete = NULL;
    Request->Flags = 0;
    Request->State = 0;

    switch (Srb->Function)
    {
        case SRB_FUNCTION_EXECUTE_SCSI:
        {
            SrbStatus = AtaReqExecuteScsi(DevExt, Request, Srb);
            break;
        }

        case SRB_FUNCTION_IO_CONTROL:
        {
            SrbStatus = AtaReqSmartIoControl(DevExt, Request, Srb);
            break;
        }

        case SRB_FUNCTION_SHUTDOWN:
        case SRB_FUNCTION_FLUSH:
        {
            SrbStatus = SRB_STATUS_SUCCESS;
            break;
        }

        default:
        {
            ASSERT(FALSE);
            UNREACHABLE;
        }
    }

    return SrbStatus;
}

static
VOID
AtaReqStartRequest(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PATA_DEVICE_REQUEST Request,
    _In_ PSCSI_REQUEST_BLOCK Srb)
{
    PATAPORT_DEVICE_EXTENSION DevExt;
    UCHAR SrbStatus;

    DevExt = CONTAINING_RECORD(Device, ATAPORT_DEVICE_EXTENSION, Device);

    SrbStatus = AtaReqTranslateRequest(DevExt, Request, Srb);

    if (SrbStatus != SRB_STATUS_PENDING)
    {
        KeReleaseSpinLockFromDpcLevel(&Device->QueueLock);

        Request->SrbStatus = SrbStatus;
        Request->InternalState = REQUEST_STATE_NOT_STARTED;

        /* Defer the completion to a DPC to avoid the recursive call in some cases */
        AtaReqStartCompletionDpc(Request);
    }
    else
    {
        Request->Flags |= Srb->SrbFlags & REQUEST_FLAG_NO_KEEP_AWAKE;

        KeAcquireSpinLockAtDpcLevel(&Device->PortData->QueueLock);

        AtaReqDispatchRequest(Device, Request, TRUE);
    }
}

static
PATA_DEVICE_REQUEST
AtaReqRemovePortRequest(
    _In_ PATAPORT_IO_CONTEXT Device)
{
    PATAPORT_PORT_DATA PortData = Device->PortData;
    PLIST_ENTRY Entry;
    PATA_DEVICE_REQUEST Request, Result = NULL;

    if (!(Device->QueueFlags & QUEUE_FLAG_FROZEN_SLOT))
        return NULL;

    Device->QueueFlags &= ~QUEUE_FLAG_FROZEN_SLOT;

    KeAcquireSpinLockAtDpcLevel(&PortData->QueueLock);

    for (Entry = PortData->PortQueueList.Flink;
         Entry != &PortData->PortQueueList;
         Entry = Entry->Flink)
    {
        Request = CONTAINING_RECORD(Entry, ATA_DEVICE_REQUEST, PortEntry);
        ASSERT_REQUEST(Request);

        if (Request->Device != (PATA_IO_CONTEXT_COMMON)Device)
            continue;

        RemoveEntryList(&Request->PortEntry);

        Result = Request;
        break;
    }

    KeReleaseSpinLockFromDpcLevel(&PortData->QueueLock);

    ASSERT(Result != NULL);
    return Result;
}

_Requires_lock_held_(Device->QueueLock)
static
VOID
AtaReqDeviceQueueRemoveEntry(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PREQUEST_QUEUE_ENTRY QueueEntry)
{
    RemoveEntryList(&QueueEntry->ListEntry);
}

static
VOID
NTAPI
AtaReqDeviceQueueCancelIo(
    _Inout_ PDEVICE_OBJECT DeviceObject,
    _Inout_ _IRQL_uses_cancel_ PIRP Irp)
{
    PATAPORT_IO_CONTEXT Device;
    KIRQL OldLevel;
    PSCSI_REQUEST_BLOCK Srb;
    PREQUEST_QUEUE_ENTRY QueueEntry;

    UNREFERENCED_PARAMETER(DeviceObject);

    IoReleaseCancelSpinLock(Irp->CancelIrql);

    QueueEntry = QUEUE_ENTRY_FROM_IRP(Irp);
    Device = QueueEntry->Context;

    KeAcquireSpinLock(&Device->QueueLock, &OldLevel);
    AtaReqDeviceQueueRemoveEntry(Device, QueueEntry);
    KeReleaseSpinLock(&Device->QueueLock, OldLevel);

    Srb = IoGetCurrentIrpStackLocation(Irp)->Parameters.Scsi.Srb;
    ASSERT(Srb);
    ASSERT(Srb->OriginalRequest == Irp);

    Srb->SrbStatus = SRB_STATUS_ABORTED;
    Srb->InternalStatus = STATUS_CANCELLED;

    Irp->IoStatus.Status = STATUS_CANCELLED;
    IoCompleteRequest(Irp, IO_NO_INCREMENT);
}

static
VOID
AtaReqDeviceQueueInsertSrb(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PSCSI_REQUEST_BLOCK Srb,
    _In_ PIRP Irp)
{
    PLIST_ENTRY Entry;
    PREQUEST_QUEUE_ENTRY QueueEntry;
    ULONG SortKey;

    SortKey = Srb->QueueSortKey;

    /* Use the C-LOOK hard disk scheduling algorithm to service the PDO requests */
    for (Entry = Device->DeviceQueueList.Flink;
         Entry != &Device->DeviceQueueList;
         Entry = Entry->Flink)
    {
        QueueEntry = CONTAINING_RECORD(Entry, REQUEST_QUEUE_ENTRY, ListEntry);
        ASSERT(QueueEntry->Context == Device);

        if (QueueEntry->SortKey > SortKey)
            break;
    }

    QueueEntry = QUEUE_ENTRY_FROM_IRP(Irp);
    QueueEntry->Context = Device;
    QueueEntry->SortKey = SortKey;

    InsertTailList(Entry, &QueueEntry->ListEntry);
}

static
NTSTATUS
AtaReqDeviceQueueAddSrb(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PSCSI_REQUEST_BLOCK Srb)
{
    PIRP Irp = Srb->OriginalRequest;

    if (Device->QueueFlags & QUEUE_FLAG_FROZEN_REMOVED)
    {
        Srb->SrbStatus = SRB_STATUS_NO_DEVICE;
        return STATUS_NO_SUCH_DEVICE;
    }

    AtaReqDeviceQueueInsertSrb(Device, Srb, Irp);

    AtaDeviceCheckPowerState(Device);

    /*
     * If the device queue is full or frozen,
     * the requests may take a long period of time to process,
     * and therefore we have to do the cancellation ourselves.
     */
    (VOID)IoSetCancelRoutine(Irp, AtaReqDeviceQueueCancelIo);

    /* This IRP has already been cancelled */
    if (Irp->Cancel && IoSetCancelRoutine(Irp, NULL))
    {
        /* Remove the IRP from the queue */
        AtaReqDeviceQueueRemoveEntry(Device, QUEUE_ENTRY_FROM_IRP(Irp));

        Srb->SrbStatus = SRB_STATUS_ABORTED;
        Srb->InternalStatus = STATUS_CANCELLED;
        return STATUS_CANCELLED;
    }

    return STATUS_PENDING;
}

static
inline
UCHAR
AtaReqGetTagForRequest(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PSCSI_REQUEST_BLOCK Srb,
    _Out_ PULONG Tag)
{
    /* Check if the queue is actually frozen and the request is not a bypass */
    if (Device->QueueFlags & (QUEUE_FLAGS_FROZEN &
                              ~(QUEUE_FLAG_FROZEN_QUEUE_LOCK | QUEUE_FLAG_FROZEN_QUEUE_FREEZE)))
    {
        return FALSE;
    }
    if ((Device->QueueFlags & QUEUE_FLAG_FROZEN_QUEUE_FREEZE) &&
        !(Srb->SrbFlags & SRB_FLAGS_BYPASS_FROZEN_QUEUE))
    {
        return FALSE;
    }
    if ((Device->QueueFlags & QUEUE_FLAG_FROZEN_QUEUE_LOCK) &&
        !(Srb->SrbFlags & SRB_FLAGS_BYPASS_LOCKED_QUEUE))
    {
        return FALSE;
    }

    /*
     * Stack-based request allocation.
     * It will return us the last request structure in the CPU cache.
     */
    return _BitScanForward(Tag, Device->FreeRequestsBitmap);
}

static
inline
PATA_DEVICE_REQUEST
AtaReqAllocateRequestFromTag(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ ULONG Tag)
{
    PATA_DEVICE_REQUEST Request;

    ASSERT(Device->FreeRequestsBitmap & (1 << Tag));

    Device->FreeRequestsBitmap &= ~(1 << Tag);

    Request = &Device->Requests[Tag];
    ASSERT_REQUEST(Request);

    Request->Tag = 1 << Tag;

    return Request;
}

static
BOOLEAN
AtaReqDeviceQueueDispatchNextRequest(
    _In_ PATAPORT_IO_CONTEXT Device)
{
    PREQUEST_QUEUE_ENTRY QueueEntry;
    PIRP Irp;
    PSCSI_REQUEST_BLOCK Srb;
    PATA_DEVICE_REQUEST Request;
    ULONG Tag;

    if (Device->FreeRequestsBitmap == 0)
        return FALSE;

    /* Find a bypass request to dispatch */
    if ((Device->QueueFlags & (QUEUE_FLAG_FROZEN_QUEUE_FREEZE | QUEUE_FLAG_FROZEN_QUEUE_LOCK)) &&
        !(Device->QueueFlags & QUEUE_FLAGS_FROZEN_NOT_BYPASS))
    {
        PLIST_ENTRY Entry;

        for (Entry = Device->DeviceQueueList.Flink;
             Entry != &Device->DeviceQueueList;
             Entry = Entry->Flink)
        {
            QueueEntry = CONTAINING_RECORD(Entry, REQUEST_QUEUE_ENTRY, ListEntry);
            ASSERT(QueueEntry->Context == Device);

            Irp = IRP_FROM_QUEUE_ENTRY(QueueEntry);
            Srb = IoGetCurrentIrpStackLocation(Irp)->Parameters.Scsi.Srb;

            ASSERT(Srb);
            ASSERT(Srb->OriginalRequest == Irp);

            if (!AtaReqGetTagForRequest(Device, Srb, &Tag))
                continue;

            AtaReqDeviceQueueRemoveEntry(Device, QueueEntry);

            /* Clear our cancel routine */
            if (!IoSetCancelRoutine(Irp, NULL))
            {
                /* We're already canceled, reset the list entry to point to itself */
                InitializeListHead(&QueueEntry->ListEntry);
                continue;
            }

            Request = AtaReqAllocateRequestFromTag(Device, Tag);

            AtaReqStartRequest(Device, Request, Srb);
            return TRUE;
        }
    }
    else
    {
        PLIST_ENTRY Entry;

        if (Device->QueueFlags & QUEUE_FLAGS_FROZEN)
            return FALSE;

        NT_VERIFY(_BitScanForward(&Tag, Device->FreeRequestsBitmap));

        for (Entry = Device->DeviceQueueList.Flink;
             Entry != &Device->DeviceQueueList;
             Entry = Entry->Flink)
        {
            QueueEntry = CONTAINING_RECORD(Entry,
                                           REQUEST_QUEUE_ENTRY,
                                           ListEntry);
            ASSERT(QueueEntry->Context == Device);

            Irp = IRP_FROM_QUEUE_ENTRY(QueueEntry);
            Srb = IoGetCurrentIrpStackLocation(Irp)->Parameters.Scsi.Srb;

            ASSERT(Srb);
            ASSERT(Srb->OriginalRequest == Irp);

            AtaReqDeviceQueueRemoveEntry(Device, QueueEntry);

            /* Clear our cancel routine */
            if (!IoSetCancelRoutine(Irp, NULL))
            {
                /* We're already canceled, reset the list entry to point to itself */
                InitializeListHead(&QueueEntry->ListEntry);
                continue;
            }

            Request = AtaReqAllocateRequestFromTag(Device, Tag);

            AtaReqStartRequest(Device, Request, Srb);
            return TRUE;
        }
    }

    return FALSE;
}

static
NTSTATUS
AtaReqStartSrb(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PSCSI_REQUEST_BLOCK Srb)
{
    KIRQL OldIrql;
    NTSTATUS Status;
    PATA_DEVICE_REQUEST Request;
    ULONG Tag;

    KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
    KeAcquireSpinLockAtDpcLevel(&Device->QueueLock);

#if DBG
    ++Device->Statistics.RequestsStarted;
#endif

    if (!AtaReqGetTagForRequest(Device, Srb, &Tag))
    {
        Status = AtaReqDeviceQueueAddSrb(Device, Srb);

        KeReleaseSpinLockFromDpcLevel(&Device->QueueLock);
    }
    else
    {
        Request = AtaReqAllocateRequestFromTag(Device, Tag);
        AtaReqStartRequest(Device, Request, Srb);

        Status = STATUS_PENDING;
    }

    KeLowerIrql(OldIrql);

    return Status;
}

DECLSPEC_NOINLINE_FROM_PAGED
VOID
AtaReqFreezeQueue(
    _In_ PATAPORT_DEVICE_EXTENSION DevExt,
    _In_ ULONG ReasonFlags)
{
    PATAPORT_IO_CONTEXT Device = &DevExt->Device;
    KIRQL OldLevel;

    KeAcquireSpinLock(&Device->QueueLock, &OldLevel);
    _InterlockedOr(&Device->QueueFlags, ReasonFlags);
    KeReleaseSpinLock(&Device->QueueLock, OldLevel);
}

DECLSPEC_NOINLINE_FROM_PAGED
VOID
AtaReqThawQueue(
    _In_ PATAPORT_DEVICE_EXTENSION DevExt,
    _In_ ULONG ReasonFlags)
{
    PATAPORT_IO_CONTEXT Device = &DevExt->Device;
    PATAPORT_PORT_DATA PortData;
    KIRQL OldIrql;

    KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
    KeAcquireSpinLockAtDpcLevel(&Device->QueueLock);

    if (Device->FreeRequestsBitmap != Device->MaxRequestsBitmap)
        AtaDeviceCheckPowerState(Device);

    PortData = Device->PortData;
    KeAcquireSpinLockAtDpcLevel(&PortData->QueueLock);
    if (!AtaReqPortQueueListDispatchNextRequest(PortData))
        KeReleaseSpinLockFromDpcLevel(&PortData->QueueLock);

    _InterlockedAnd(&Device->QueueFlags, ~ReasonFlags);

    if (!AtaReqDeviceQueueDispatchNextRequest(Device))
        KeReleaseSpinLockFromDpcLevel(&Device->QueueLock);

    KeLowerIrql(OldIrql);
}

/* Must not be paged */
DECLSPEC_NOINLINE_FROM_PAGED
VOID
AtaReqWaitForOutstandingIoToComplete(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PSCSI_REQUEST_BLOCK Srb)
{
    KIRQL OldIrql;
    BOOLEAN DoWait;
    PATA_DEVICE_REQUEST Request;

    KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
    KeAcquireSpinLockAtDpcLevel(&Device->QueueLock);

    ASSERT(Device->QueueFlags & QUEUE_FLAGS_FROZEN);

    KeClearEvent(&Device->QueueStoppedEvent);

    if (Srb)
        Device->QuiescenceSrb = Srb;

    /* Wait for all the active IRPs to finish executing */
    if (Device->FreeRequestsBitmap != Device->MaxRequestsBitmap)
    {
        Device->QueueFlags |= QUEUE_FLAG_SIGNAL_STOP;
        DoWait = !Srb;
    }
    else
    {
        AtaDeviceQueueEmptyEvent(Device);

        KeSetEvent(&Device->QueueStoppedEvent, 0, FALSE);
        DoWait = FALSE;
    }

    Request = AtaReqRemovePortRequest(Device);

    KeReleaseSpinLockFromDpcLevel(&Device->QueueLock);
    KeLowerIrql(OldIrql);

    /* Requeue the pending request */
    if (Request)
    {
        Request->SrbStatus = SRB_STATUS_BUSY;
        Request->InternalState = REQUEST_STATE_NOT_STARTED;

        KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
        AtaReqCompleteRequest(Request);
        KeLowerIrql(OldIrql);
    }

    if (!DoWait)
        return;

    KeWaitForSingleObject(&Device->QueueStoppedEvent, Executive, KernelMode, FALSE, NULL);
}

DECLSPEC_NOINLINE_FROM_PAGED
VOID
AtaReqFlushDeviceQueue(
    _In_ PATAPORT_IO_CONTEXT Device)
{
    PATA_DEVICE_REQUEST Request;
    KIRQL OldLevel;
    PLIST_ENTRY Entry;

    KeAcquireSpinLock(&Device->QueueLock, &OldLevel);

    for (Entry = Device->DeviceQueueList.Flink;
         Entry != &Device->DeviceQueueList;
         Entry = Entry->Flink)
    {
        PREQUEST_QUEUE_ENTRY QueueEntry;
        PIRP Irp;
        PSCSI_REQUEST_BLOCK Srb;

        QueueEntry = CONTAINING_RECORD(Entry, REQUEST_QUEUE_ENTRY, ListEntry);
        ASSERT(QueueEntry->Context == Device);

        AtaReqDeviceQueueRemoveEntry(Device, QueueEntry);

        /* Clear our cancel routine */
        Irp = IRP_FROM_QUEUE_ENTRY(QueueEntry);
        if (!IoSetCancelRoutine(Irp, NULL))
        {
            /* We're already canceled, reset the list entry to point to itself */
            InitializeListHead(&QueueEntry->ListEntry);
            continue;
        }

        KeReleaseSpinLock(&Device->QueueLock, OldLevel);

        Srb = IoGetCurrentIrpStackLocation(Irp)->Parameters.Scsi.Srb;
        ASSERT(Srb);
        ASSERT(Srb->OriginalRequest == Irp);

        Srb->SrbStatus = SRB_STATUS_ABORTED;
        Irp->IoStatus.Status = STATUS_CANCELLED;
        Irp->IoStatus.Information = Srb->DataTransferLength;
        IoCompleteRequest(Irp, IO_DISK_INCREMENT);

        KeAcquireSpinLock(&Device->QueueLock, &OldLevel);
    }

    Request = AtaReqRemovePortRequest(Device);

    KeReleaseSpinLock(&Device->QueueLock, OldLevel);

    if (Request)
    {
        KIRQL OldIrql;

        Request->SrbStatus = SRB_STATUS_ABORTED;
        Request->InternalState = REQUEST_STATE_NOT_STARTED;

        KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
        AtaReqCompleteRequest(Request);
        KeLowerIrql(OldIrql);
    }
}

static
DECLSPEC_NOINLINE_FROM_NOT_PAGED
CODE_SEG("PAGE")
NTSTATUS
AtaPdoHandleAttachReleaseDevice(
    _In_ PATAPORT_DEVICE_EXTENSION DevExt,
    _In_ PSCSI_REQUEST_BLOCK Srb)
{
    ULONG Status, SrbStatus;

    PAGED_CODE();

    if (Srb->Function == SRB_FUNCTION_RELEASE_DEVICE)
    {
        DevExt->DeviceClaimed = FALSE;

        SrbStatus = SRB_STATUS_SUCCESS;
        Status = STATUS_SUCCESS;
        goto Exit;
    }

    if (DevExt->DeviceClaimed)
    {
        SrbStatus = SRB_STATUS_BUSY;
        Status = STATUS_DEVICE_BUSY;
        goto Exit;
    }

    DevExt->DeviceClaimed = TRUE;
    Srb->DataBuffer = DevExt->Common.Self;

    SrbStatus = SRB_STATUS_SUCCESS;
    Status = STATUS_SUCCESS;

Exit:
    Srb->SrbStatus = SrbStatus;
    return Status;
}

static
NTSTATUS
AtaPdoHandleQuiesceDevice(
    _In_ PATAPORT_IO_CONTEXT Device,
    _In_ PIRP Irp,
    _In_ PSCSI_REQUEST_BLOCK Srb)
{
    ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
    ASSERT(Device->QueueFlags & QUEUE_FLAG_FROZEN_QUEUE_LOCK);

    IoMarkIrpPending(Irp);

    /* Wait for outstanding I/O requests to finish */
    AtaReqWaitForOutstandingIoToComplete(Device, Srb);

    Srb->SrbStatus = SRB_STATUS_PENDING;
    return STATUS_PENDING;
}

static
DECLSPEC_NOINLINE_FROM_NOT_PAGED
CODE_SEG("PAGE")
BOOLEAN
AtaPdoHandleIoControl(
    _In_ PATAPORT_DEVICE_EXTENSION DevExt,
    _In_ PSCSI_REQUEST_BLOCK Srb,
    _Out_ NTSTATUS* Status)
{
    PSRB_IO_CONTROL SrbControl = (PSRB_IO_CONTROL)Srb->DataBuffer;

    PAGED_CODE();

    switch (SrbControl->ControlCode)
    {
        case IOCTL_SCSI_MINIPORT_SMART_VERSION:
            *Status = AtaPdoHandleMiniportSmartVersion(DevExt, Srb);
            break;

        case IOCTL_SCSI_MINIPORT_IDENTIFY:
            *Status = AtaPdoHandleMiniportIdentify(DevExt, Srb);
            break;

        case IOCTL_SCSI_MINIPORT_DISABLE_SMART:
        case IOCTL_SCSI_MINIPORT_ENABLE_DISABLE_AUTOSAVE:
        case IOCTL_SCSI_MINIPORT_ENABLE_DISABLE_AUTO_OFFLINE:
        case IOCTL_SCSI_MINIPORT_ENABLE_SMART:
        case IOCTL_SCSI_MINIPORT_EXECUTE_OFFLINE_DIAGS:
        case IOCTL_SCSI_MINIPORT_READ_SMART_ATTRIBS:
        case IOCTL_SCSI_MINIPORT_READ_SMART_LOG:
        case IOCTL_SCSI_MINIPORT_READ_SMART_THRESHOLDS:
        case IOCTL_SCSI_MINIPORT_RETURN_STATUS:
        case IOCTL_SCSI_MINIPORT_SAVE_ATTRIBUTE_VALUES:
        case IOCTL_SCSI_MINIPORT_WRITE_SMART_LOG:
        {
            /* Queue the request */
            return FALSE;
        }

        default:
            Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST;
            *Status = STATUS_INVALID_DEVICE_REQUEST;
            break;
    }

    return TRUE;
}

static
NTSTATUS
AtaPdoDispatchScsi(
    _In_ PATAPORT_DEVICE_EXTENSION DevExt,
    _Inout_ PIRP Irp)
{
    NTSTATUS Status;
    PSCSI_REQUEST_BLOCK Srb;

    Srb = IoGetCurrentIrpStackLocation(Irp)->Parameters.Scsi.Srb;
    ASSERT(Srb);
    ASSERT(Srb->OriginalRequest == Irp);

    Status = IoAcquireRemoveLock(&DevExt->Common.RemoveLock, Irp);
    if (!NT_SUCCESS(Status))
    {
        Srb->SrbStatus = SRB_STATUS_NO_DEVICE;
        Status = STATUS_NO_SUCH_DEVICE;

        Irp->IoStatus.Status = Status;
        IoCompleteRequest(Irp, IO_NO_INCREMENT);

        return Status;
    }

    switch (Srb->Function)
    {
        case SRB_FUNCTION_CLAIM_DEVICE:
        case SRB_FUNCTION_RELEASE_DEVICE:
        {
            Status = AtaPdoHandleAttachReleaseDevice(DevExt, Srb);
            break;
        }

        case SRB_FUNCTION_LOCK_QUEUE:
        {
            AtaReqFreezeQueue(DevExt, QUEUE_FLAG_FROZEN_QUEUE_LOCK);

            Srb->SrbStatus = SRB_STATUS_SUCCESS;
            Status = STATUS_SUCCESS;
            break;
        }

        case SRB_FUNCTION_FLUSH_QUEUE:
        {
            ASSERT(DevExt->Device.QueueFlags & QUEUE_FLAGS_FROZEN);
            AtaReqFlushDeviceQueue(&DevExt->Device);
            __fallthrough;
        }
        case SRB_FUNCTION_UNLOCK_QUEUE:
        case SRB_FUNCTION_RELEASE_QUEUE:
        {
            ULONG Reason;

            if (Srb->Function == SRB_FUNCTION_UNLOCK_QUEUE)
                Reason = QUEUE_FLAG_FROZEN_QUEUE_LOCK;
            else
                Reason = QUEUE_FLAG_FROZEN_QUEUE_FREEZE;
            AtaReqThawQueue(DevExt, Reason);

            Srb->SrbStatus = SRB_STATUS_SUCCESS;
            Status = STATUS_SUCCESS;
            break;
        }

        case SRB_FUNCTION_QUIESCE_DEVICE:
        {
            Status = AtaPdoHandleQuiesceDevice(&DevExt->Device, Irp, Srb);
            break;
        }

        case SRB_FUNCTION_IO_CONTROL:
        {
            if (AtaPdoHandleIoControl(DevExt, Srb, &Status))
                break;

            __fallthrough;
        }
        case SRB_FUNCTION_SHUTDOWN:
        case SRB_FUNCTION_FLUSH:
        case SRB_FUNCTION_EXECUTE_SCSI:
        {
            ATA_SCSI_ADDRESS AtaScsiAddress;

            IoMarkIrpPending(Irp);

            /* Set the SCSI address to the correct value */
            AtaScsiAddress = DevExt->Device.AtaScsiAddress;
            Srb->PathId = AtaScsiAddress.PathId;
            Srb->TargetId = AtaScsiAddress.TargetId;
            Srb->Lun = AtaScsiAddress.Lun;

            /* This field is used by the driver to mark internal requests */
            Srb->SrbExtension = NULL;

            /*
             * NOTE: Disk I/O requests need a lot of the kernel stack space.
             * We should avoid nesting several levels deep in the call chain.
             */
            Status = AtaReqStartSrb(&DevExt->Device, Srb);
            break;
        }

        default:
            Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST;
            Status = STATUS_INVALID_DEVICE_REQUEST;
            break;
    }

    if (Status != STATUS_PENDING)
    {
        Irp->IoStatus.Status = Status;
        IoCompleteRequest(Irp, IO_NO_INCREMENT);
    }

    IoReleaseRemoveLock(&DevExt->Common.RemoveLock, Irp);

    return Status;
}

static
DECLSPEC_NOINLINE_FROM_NOT_PAGED
CODE_SEG("PAGE")
NTSTATUS
AtaFdoDispatchScsi(
    _In_ PATAPORT_CHANNEL_EXTENSION ChanExt,
    _Inout_ PIRP Irp)
{
    NTSTATUS Status;
    PSCSI_REQUEST_BLOCK Srb;

    UNREFERENCED_PARAMETER(ChanExt);

    PAGED_CODE();

    /* Drivers should not call the FDO */
    ASSERT(FALSE);

    Srb = IoGetCurrentIrpStackLocation(Irp)->Parameters.Scsi.Srb;
    ASSERT(Srb);
    ASSERT(Srb->OriginalRequest == Irp);

    Srb->SrbStatus = SRB_STATUS_NO_DEVICE;
    Status = STATUS_NO_SUCH_DEVICE;

    Irp->IoStatus.Status = Status;
    IoCompleteRequest(Irp, IO_NO_INCREMENT);
    return Status;
}

NTSTATUS
NTAPI
AtaDispatchScsi(
    _In_ PDEVICE_OBJECT DeviceObject,
    _Inout_ PIRP Irp)
{
    if (IS_FDO(DeviceObject->DeviceExtension))
        return AtaFdoDispatchScsi(DeviceObject->DeviceExtension, Irp);
    else
        return AtaPdoDispatchScsi(DeviceObject->DeviceExtension, Irp);
}

VOID
NTAPI
AtaPortIoTimer(
    _In_ PDEVICE_OBJECT DeviceObject,
    _In_opt_ PVOID Context)
{
    PATAPORT_PORT_DATA PortData = Context;
    ULONG Slot, ActiveTimersBitmap;

    KeAcquireSpinLockAtDpcLevel(&PortData->QueueLock);

    /* Check timeouts */
    ActiveTimersBitmap = PortData->ActiveTimersBitmap;
    while (_BitScanForward(&Slot, ActiveTimersBitmap) != 0)
    {
        ActiveTimersBitmap &= ~(1 << Slot);

        /* Decrease the timeout counter */
        if ((PortData->TimerCount[Slot] > 0) && (--PortData->TimerCount[Slot] == 0))
        {
            KIRQL OldIrql = KeAcquireInterruptSpinLock(PortData->InterruptObject);

            /* Handle timeout of an active command */
            if (PortData->ActiveSlotsBitmap & (1 << Slot))
                AtaPortTimeout(PortData, Slot);

            KeReleaseInterruptSpinLock(PortData->InterruptObject, OldIrql);
        }
    }

    KeReleaseSpinLockFromDpcLevel(&PortData->QueueLock);
}