diff --git a/reactos/include/ddk/kefuncs.h b/reactos/include/ddk/kefuncs.h index 7de1d53cb00..0f7d7013008 100644 --- a/reactos/include/ddk/kefuncs.h +++ b/reactos/include/ddk/kefuncs.h @@ -149,17 +149,17 @@ VOID STDCALL KeInitializeEvent (PKEVENT Event, EVENT_TYPE Type, BOOLEAN State); -NTSTATUS STDCALL KeInitializeInterrupt(PKINTERRUPT InterruptObject, - PKSERVICE_ROUTINE ServiceRoutine, - PVOID ServiceContext, - PKSPIN_LOCK SpinLock, - ULONG Vector, - KIRQL Irql, - KIRQL SynchronizeIrql, - KINTERRUPT_MODE InterruptMode, - BOOLEAN ShareVector, - KAFFINITY ProcessorEnableMask, - BOOLEAN FloatingSave); +VOID STDCALL KeInitializeInterrupt(PKINTERRUPT InterruptObject, + PKSERVICE_ROUTINE ServiceRoutine, + PVOID ServiceContext, + PKSPIN_LOCK SpinLock, + ULONG Vector, + KIRQL Irql, + KIRQL SynchronizeIrql, + KINTERRUPT_MODE InterruptMode, + BOOLEAN ShareVector, + CHAR ProcessorNumber, + BOOLEAN FloatingSave); VOID STDCALL KeInitializeMutant(IN PKMUTANT Mutant, IN BOOLEAN InitialOwner); @@ -367,6 +367,10 @@ KeRosGetStackFrames ( PULONG Frames, ULONG FrameCount ); BOOLEAN STDCALL KeRosPrintAddress(PVOID address); +NTSTATUS STDCALL +KeSetAffinityThread(PKTHREAD Thread, + KAFFINITY Affinity); + LONG STDCALL KeSetBasePriorityThread(struct _KTHREAD* Thread, LONG Increment); diff --git a/reactos/include/ntos/zwtypes.h b/reactos/include/ntos/zwtypes.h index 1d5ca09b6bb..9ebd6510a4f 100755 --- a/reactos/include/ntos/zwtypes.h +++ b/reactos/include/ntos/zwtypes.h @@ -1818,13 +1818,17 @@ typedef struct _KINTERRUPT { ULONG Vector; KAFFINITY ProcessorEnableMask; - PKSPIN_LOCK IrqLock; + KSPIN_LOCK SpinLock; + PKSPIN_LOCK ActualLock; BOOLEAN Shareable; BOOLEAN FloatingSave; + CHAR ProcessorNumber; PKSERVICE_ROUTINE ServiceRoutine; PVOID ServiceContext; LIST_ENTRY Entry; + KIRQL Irql; KIRQL SynchLevel; + KINTERRUPT_MODE InterruptMode; } KINTERRUPT; #ifndef __USE_W32API diff --git a/reactos/ntoskrnl/io/irq.c b/reactos/ntoskrnl/io/irq.c index 44e40901c55..66dee26c58b 100644 --- a/reactos/ntoskrnl/io/irq.c +++ b/reactos/ntoskrnl/io/irq.c @@ -62,43 +62,70 @@ IoConnectInterrupt(PKINTERRUPT* InterruptObject, */ { PKINTERRUPT Interrupt; - NTSTATUS Status = STATUS_SUCCESS; + ULONG i, count; ASSERT_IRQL(PASSIVE_LEVEL); DPRINT("IoConnectInterrupt(Vector %x)\n",Vector); + ProcessorEnableMask &= ((1 << KeNumberProcessors) - 1); + + if (ProcessorEnableMask == 0) + { + return STATUS_INVALID_PARAMETER; + } + + for (i = 0, count = 0; i < KeNumberProcessors; i++) + { + if (ProcessorEnableMask & (1 << i)) + { + count++; + } + } /* * Initialize interrupt object */ - Interrupt=ExAllocatePoolWithTag(NonPagedPool,sizeof(KINTERRUPT), + Interrupt=ExAllocatePoolWithTag(NonPagedPool,count*sizeof(KINTERRUPT), TAG_KINTERRUPT); if (Interrupt==NULL) { return(STATUS_INSUFFICIENT_RESOURCES); } - Status = KeInitializeInterrupt(Interrupt, - ServiceRoutine, - ServiceContext, - SpinLock, - Vector, - Irql, - SynchronizeIrql, - InterruptMode, - ShareVector, - ProcessorEnableMask, - FloatingSave); - if (!NT_SUCCESS(Status)) + if (SpinLock == NULL) { - ExFreePool(Interrupt); - return Status; + SpinLock = &Interrupt[0].SpinLock; + KeInitializeSpinLock(SpinLock); } - if (!KeConnectInterrupt(Interrupt)) + Interrupt[0].ProcessorEnableMask = ProcessorEnableMask; + + for (i = 0, count = 0; i < KeNumberProcessors; i++) { - ExFreePool(Interrupt); - return STATUS_INVALID_PARAMETER; + if (ProcessorEnableMask & (1 << i)) + { + KeInitializeInterrupt(&Interrupt[count], + ServiceRoutine, + ServiceContext, + SpinLock, + Vector, + Irql, + SynchronizeIrql, + InterruptMode, + ShareVector, + i, + FloatingSave); + if (!KeConnectInterrupt(&Interrupt[count])) + { + for (i = 0; i < count; i++) + { + KeDisconnectInterrupt(&Interrupt[i]); + } + ExFreePool(Interrupt); + return STATUS_INVALID_PARAMETER; + } + count++; + } } *InterruptObject = Interrupt; @@ -118,7 +145,16 @@ IoDisconnectInterrupt(PKINTERRUPT InterruptObject) * InterruptObject = isr to release */ { - KeDisconnectInterrupt(InterruptObject); + ULONG i, count; + + for (i = 0, count = 0; i < KeNumberProcessors; i++) + { + if (InterruptObject[0].ProcessorEnableMask & (1 << i)) + { + KeDisconnectInterrupt(&InterruptObject[count]); + count++; + } + } ExFreePool(InterruptObject); } diff --git a/reactos/ntoskrnl/ke/i386/irq.c b/reactos/ntoskrnl/ke/i386/irq.c index 0e1e055cd56..adc1f9e2258 100644 --- a/reactos/ntoskrnl/ke/i386/irq.c +++ b/reactos/ntoskrnl/ke/i386/irq.c @@ -1,6 +1,6 @@ /* * ReactOS kernel - * Copyright (C) 1998, 1999, 2000, 2001, 2002 ReactOS Team + * Copyright (C) 1998 - 2005 ReactOS Team * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,12 +16,13 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -/* $Id: irq.c,v 1.56 2004/11/27 19:24:15 hbirr Exp $ +/* $Id$ * * PROJECT: ReactOS kernel * FILE: ntoskrnl/ke/i386/irq.c * PURPOSE: IRQ handling * PROGRAMMER: David Welch (welch@mcmail.com) + * Hartmut Birr * UPDATE HISTORY: * 29/05/98: Created */ @@ -182,9 +183,19 @@ static unsigned int irq_handler[NR_IRQS]= * be accessed at any irq level. */ -static LIST_ENTRY isr_table[NR_IRQS]; -static KSPIN_LOCK isr_lock[NR_IRQS]; -static KSPIN_LOCK isr_table_lock; +typedef struct +{ + LIST_ENTRY ListHead; + KSPIN_LOCK Lock; + ULONG Count; +} +ISR_TABLE, *PISR_TABLE; + +#ifdef MP +static ISR_TABLE IsrTable[NR_IRQS][MAXIMUM_PROCESSORS]; +#else +static ISR_TABLE IsrTable[NR_IRQS][1]; +#endif #define TAG_ISR_LOCK TAG('I', 'S', 'R', 'L') @@ -196,7 +207,8 @@ static KSPIN_LOCK isr_table_lock; VOID INIT_FUNCTION KeInitInterrupts (VOID) { - int i; + int i, j; + /* * Setup the IDT entries to point to the interrupt handlers @@ -206,10 +218,17 @@ KeInitInterrupts (VOID) KiIdt[IRQ_BASE+i].a=(irq_handler[i]&0xffff)+(KERNEL_CS<<16); KiIdt[IRQ_BASE+i].b=(irq_handler[i]&0xffff0000)+PRESENT+ I486_INTERRUPT_GATE; - InitializeListHead(&isr_table[i]); - KeInitializeSpinLock(&isr_lock[i]); +#ifdef MP + for (j = 0; j < MAXIMUM_PROCESSORS; j++) +#else + j = 0; +#endif + { + InitializeListHead(&IsrTable[i][j].ListHead); + KeInitializeSpinLock(&IsrTable[i][j].Lock); + IsrTable[i][j].Count = 0; + } } - KeInitializeSpinLock(&isr_table_lock); } STATIC VOID @@ -266,21 +285,34 @@ KiInterruptDispatch2 (ULONG vector, KIRQL old_level) { PKINTERRUPT isr; PLIST_ENTRY current; + KIRQL oldlvl; + PISR_TABLE CurrentIsr; DPRINT("I(0x%.08x, 0x%.08x)\n", vector, old_level); /* * Iterate the list until one of the isr tells us its device interrupted */ - current = isr_table[vector - IRQ_BASE].Flink; - isr = CONTAINING_RECORD(current,KINTERRUPT,Entry); + CurrentIsr = &IsrTable[vector - IRQ_BASE][(ULONG)KeGetCurrentProcessorNumber()]; - while (current != &isr_table[vector - IRQ_BASE] && - !isr->ServiceRoutine(isr, isr->ServiceContext)) + KiAcquireSpinLock(&CurrentIsr->Lock); + + CurrentIsr->Count++; + current = CurrentIsr->ListHead.Flink; + + while (current != &CurrentIsr->ListHead) { - current = current->Flink; isr = CONTAINING_RECORD(current,KINTERRUPT,Entry); + oldlvl = KeAcquireInterruptSpinLock(isr); + if (isr->ServiceRoutine(isr, isr->ServiceContext)) + { + KeReleaseInterruptSpinLock(isr, oldlvl); + break; + } + KeReleaseInterruptSpinLock(isr, oldlvl); + current = current->Flink; } + KiReleaseSpinLock(&CurrentIsr->Lock); } VOID @@ -382,19 +414,35 @@ KeDumpIrqList(VOID) { PKINTERRUPT current; PLIST_ENTRY current_entry; - unsigned int i; + ULONG i, j; + KIRQL oldlvl; + BOOLEAN printed; for (i=0;iFlink; - current = CONTAINING_RECORD(current_entry,KINTERRUPT,Entry); + KiAcquireSpinLock(&IsrTable[i][j].Lock); + + current_entry = IsrTable[i][j].ListHead.Flink; + current = CONTAINING_RECORD(current_entry,KINTERRUPT,Entry); + while (current_entry!=&(IsrTable[i][j].ListHead)) + { + if (printed == FALSE) + { + printed = TRUE; + DPRINT("For irq %x:\n",i); + } + DPRINT(" Isr %x\n",current); + current_entry = current_entry->Flink; + current = CONTAINING_RECORD(current_entry,KINTERRUPT,Entry); + } + KiReleaseSpinLock(&IsrTable[i][j].Lock); } + KeLowerIrql(oldlvl); } } @@ -404,10 +452,11 @@ KeDumpIrqList(VOID) BOOLEAN STDCALL KeConnectInterrupt(PKINTERRUPT InterruptObject) { - KIRQL oldlvl; - KIRQL synch_oldlvl; + KIRQL oldlvl,synch_oldlvl; PKINTERRUPT ListHead; ULONG Vector; + PISR_TABLE CurrentIsr; + BOOLEAN Result; DPRINT("KeConnectInterrupt()\n"); @@ -418,45 +467,54 @@ KeConnectInterrupt(PKINTERRUPT InterruptObject) Vector -= IRQ_BASE; - /* - * Acquire the table spinlock - */ - KeAcquireSpinLock(&isr_table_lock,&oldlvl); - + ASSERT (InterruptObject->ProcessorNumber < KeNumberProcessors); + + KeSetSystemAffinityThread(1 << InterruptObject->ProcessorNumber); + + CurrentIsr = &IsrTable[Vector][(ULONG)InterruptObject->ProcessorNumber]; + + KeRaiseIrql(VECTOR2IRQL(Vector + IRQ_BASE),&oldlvl); + KiAcquireSpinLock(&CurrentIsr->Lock); + /* * Check if the vector is already in use that we can share it */ - ListHead = CONTAINING_RECORD(isr_table[Vector].Flink,KINTERRUPT,Entry); - if (!IsListEmpty(&isr_table[Vector]) && - (InterruptObject->Shareable == FALSE || ListHead->Shareable==FALSE)) + if (!IsListEmpty(&CurrentIsr->ListHead)) { - KeReleaseSpinLock(&isr_table_lock,oldlvl); - return FALSE; + ListHead = CONTAINING_RECORD(CurrentIsr->ListHead.Flink,KINTERRUPT,Entry); + if (InterruptObject->Shareable == FALSE || ListHead->Shareable==FALSE) + { + KiReleaseSpinLock(&CurrentIsr->Lock); + KeLowerIrql(oldlvl); + KeRevertToUserAffinityThread(); + return FALSE; + } } - InterruptObject->IrqLock = &isr_lock[Vector]; + synch_oldlvl = KeAcquireInterruptSpinLock(InterruptObject); - KeRaiseIrql(InterruptObject->SynchLevel,&synch_oldlvl); - KiAcquireSpinLock(InterruptObject->IrqLock); - DPRINT("%x %x\n",isr_table[Vector].Flink,isr_table[Vector].Blink); - if (IsListEmpty(&isr_table[Vector])) + DPRINT("%x %x\n",CurrentIsr->ListHead.Flink, CurrentIsr->ListHeads.Blink); + + Result = HalEnableSystemInterrupt(Vector + IRQ_BASE, InterruptObject->Irql, InterruptObject->InterruptMode); + if (Result) { - HalEnableSystemInterrupt(Vector + IRQ_BASE, 0, 0); + InsertTailList(&CurrentIsr->ListHead,&InterruptObject->Entry); + DPRINT("%x %x\n",InterruptObject->Entry.Flink, InterruptObject->Entry.Blink); } - InsertTailList(&isr_table[Vector],&InterruptObject->Entry); - DPRINT("%x %x\n",InterruptObject->Entry.Flink, - InterruptObject->Entry.Blink); - KiReleaseSpinLock(InterruptObject->IrqLock); - KeLowerIrql(synch_oldlvl); - + + KeReleaseInterruptSpinLock(InterruptObject, synch_oldlvl); + /* * Release the table spinlock */ - KeReleaseSpinLock(&isr_table_lock,oldlvl); + KiReleaseSpinLock(&CurrentIsr->Lock); + KeLowerIrql(oldlvl); KeDumpIrqList(); - return TRUE; + KeRevertToUserAffinityThread(); + + return Result; } @@ -471,24 +529,43 @@ KeDisconnectInterrupt(PKINTERRUPT InterruptObject) * InterruptObject = isr to release */ { - KIRQL oldlvl; + KIRQL oldlvl,synch_oldlvl; + PISR_TABLE CurrentIsr; + + DPRINT("KeDisconnectInterrupt\n"); - KeRaiseIrql(InterruptObject->SynchLevel,&oldlvl); - KiAcquireSpinLock(InterruptObject->IrqLock); + ASSERT (InterruptObject->ProcessorNumber < KeNumberProcessors); + + KeSetSystemAffinityThread(1 << InterruptObject->ProcessorNumber); + + CurrentIsr = &IsrTable[InterruptObject->Vector - IRQ_BASE][(ULONG)InterruptObject->ProcessorNumber]; + + KeRaiseIrql(VECTOR2IRQL(InterruptObject->Vector),&oldlvl); + KiAcquireSpinLock(&CurrentIsr->Lock); + + synch_oldlvl = KeAcquireInterruptSpinLock(InterruptObject); + RemoveEntryList(&InterruptObject->Entry); - if (IsListEmpty(&isr_table[InterruptObject->Vector - IRQ_BASE])) + if (IsListEmpty(&CurrentIsr->ListHead)) { HalDisableSystemInterrupt(InterruptObject->Vector, 0); } - KiReleaseSpinLock(InterruptObject->IrqLock); + KeReleaseInterruptSpinLock(InterruptObject, synch_oldlvl); + + /* + * Release the table spinlock + */ + KiReleaseSpinLock(&CurrentIsr->Lock); KeLowerIrql(oldlvl); + + KeRevertToUserAffinityThread(); } /* * @implemented */ -NTSTATUS +VOID STDCALL KeInitializeInterrupt(PKINTERRUPT InterruptObject, PKSERVICE_ROUTINE ServiceRoutine, @@ -499,18 +576,37 @@ KeInitializeInterrupt(PKINTERRUPT InterruptObject, KIRQL SynchronizeIrql, KINTERRUPT_MODE InterruptMode, BOOLEAN ShareVector, - KAFFINITY ProcessorEnableMask, + CHAR ProcessorNumber, BOOLEAN FloatingSave) { - InterruptObject->ServiceContext = ServiceContext; InterruptObject->ServiceRoutine = ServiceRoutine; + InterruptObject->ServiceContext = ServiceContext; + InterruptObject->ActualLock = SpinLock; InterruptObject->Vector = Vector; - InterruptObject->ProcessorEnableMask = ProcessorEnableMask; + InterruptObject->Irql = Irql; InterruptObject->SynchLevel = SynchronizeIrql; + InterruptObject->InterruptMode = InterruptMode; InterruptObject->Shareable = ShareVector; + InterruptObject->ProcessorNumber = ProcessorNumber; InterruptObject->FloatingSave = FloatingSave; - - return STATUS_SUCCESS; } +VOID KePrintInterruptStatistic(VOID) +{ + ULONG i, j; + + for (j = 0; j < KeNumberProcessors; j++) + { + DPRINT1("CPU%d:\n", j); + for (i = 0; i < NR_IRQS; i++) + { + if (IsrTable[i][j].Count) + { + DPRINT1(" Irq %x(%d): %d\n", i, VECTOR2IRQ(i + IRQ_BASE), IsrTable[i][j].Count); + } + } + } +} + + /* EOF */ diff --git a/reactos/ntoskrnl/ke/spinlock.c b/reactos/ntoskrnl/ke/spinlock.c index a123af00605..284d19792ad 100644 --- a/reactos/ntoskrnl/ke/spinlock.c +++ b/reactos/ntoskrnl/ke/spinlock.c @@ -1,4 +1,4 @@ -/* $Id: spinlock.c,v 1.25 2004/12/24 17:06:58 navaraf Exp $ +/* $Id$ * * COPYRIGHT: See COPYING in the top level directory * PROJECT: ReactOS kernel @@ -17,6 +17,7 @@ /* INCLUDES ****************************************************************/ #include +#define NDEBUG #include /* FUNCTIONS ***************************************************************/ @@ -42,19 +43,17 @@ KeSynchronizeExecution (PKINTERRUPT Interrupt, KIRQL oldlvl; BOOLEAN ret; - KeRaiseIrql(Interrupt->SynchLevel, &oldlvl); - KiAcquireSpinLock(Interrupt->IrqLock); + oldlvl = KeAcquireInterruptSpinLock(Interrupt); ret = SynchronizeRoutine(SynchronizeContext); - KiReleaseSpinLock(Interrupt->IrqLock); - KeLowerIrql(oldlvl); + KeReleaseInterruptSpinLock(Interrupt, oldlvl); return(ret); } /* - * @unimplemented + * @implemented */ KIRQL STDCALL @@ -62,8 +61,11 @@ KeAcquireInterruptSpinLock( IN PKINTERRUPT Interrupt ) { - UNIMPLEMENTED; - return 0; + KIRQL oldIrql; + + KeRaiseIrql(Interrupt->SynchLevel, &oldIrql); + KiAcquireSpinLock(Interrupt->ActualLock); + return oldIrql; } /* @@ -195,7 +197,7 @@ KiAcquireSpinLock(PKSPIN_LOCK SpinLock) } /* - * @unimplemented + * @implemented */ VOID STDCALL @@ -204,7 +206,8 @@ KeReleaseInterruptSpinLock( IN KIRQL OldIrql ) { - UNIMPLEMENTED; + KiReleaseSpinLock(Interrupt->ActualLock); + KeLowerIrql(OldIrql); } /*