Hesham/wintun
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use standard volatile semantics

Browse Source 
Make all archs are use the standardized concept of volatile.
This patch will cause the most changes to arm64 codegen.

Signed-off-by: Shawn Hoffman <godisgovernment@gmail.com>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
This commit is contained in:
Shawn Hoffman 2020-04-24 16:47:45 -07:00 committed by Jason A. Donenfeld
parent b19c7abd41
commit 9c39b2b567
4 changed files with 40 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
atomic.h
View File

@ -1,54 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0
*
* Copyright (C) 2018-2019 WireGuard LLC. All Rights Reserved.
*/
#pragma once
#include <wdm.h>
static __forceinline VOID
InterlockedSet(_Inout_ _Interlocked_operand_ LONG volatile *Target, _In_ LONG Value)
{
*Target = Value;
}
static __forceinline VOID
InterlockedSetU(_Inout_ _Interlocked_operand_ ULONG volatile *Target, _In_ ULONG Value)
{
*Target = Value;
}
static __forceinline VOID
InterlockedSetPointer(_Inout_ _Interlocked_operand_ VOID *volatile *Target, _In_opt_ VOID *Value)
{
*Target = Value;
}
static __forceinline LONG
InterlockedGet(_In_ _Interlocked_operand_ LONG volatile *Value)
{
return *Value;
}
static __forceinline ULONG
InterlockedGetU(_In_ _Interlocked_operand_ ULONG volatile *Value)
{
return *Value;
}
static __forceinline PVOID
InterlockedGetPointer(_In_ _Interlocked_operand_ PVOID volatile *Value)
{
return *Value;
}
static __forceinline LONG64
InterlockedGet64(_In_ _Interlocked_operand_ LONG64 volatile *Value)
{
#ifdef _WIN64
return *Value;
#else
return InterlockedCompareExchange64(Value, 0, 0);
#endif
}

76
wintun.c
View File

@ -9,7 +9,6 @@
#include <ndis.h> #include <ndis.h>
#include <ntstrsafe.h> #include <ntstrsafe.h>
#include "undocumented.h" #include "undocumented.h"
#include "atomic.h"
#pragma warning(disable : 4100) /* unreferenced formal parameter */ #pragma warning(disable : 4100) /* unreferenced formal parameter */
#pragma warning(disable : 4200) /* nonstandard: zero-sized array in struct/union */ #pragma warning(disable : 4200) /* nonstandard: zero-sized array in struct/union */
@ -250,7 +249,7 @@ TunSendNetBufferLists(
KIRQL Irql = ExAcquireSpinLockShared(&Ctx->TransitionLock); KIRQL Irql = ExAcquireSpinLockShared(&Ctx->TransitionLock);
NDIS_STATUS Status; NDIS_STATUS Status;
if ((Status = NDIS_STATUS_PAUSED, !InterlockedGet(&Ctx->Running)) || if ((Status = NDIS_STATUS_PAUSED, !ReadAcquire(&Ctx->Running)) ||
(Status = NDIS_STATUS_MEDIA_DISCONNECTED, KeReadStateEvent(&Ctx->Device.Disconnected))) (Status = NDIS_STATUS_MEDIA_DISCONNECTED, KeReadStateEvent(&Ctx->Device.Disconnected)))
goto skipNbl; goto skipNbl;
@ -258,7 +257,7 @@ TunSendNetBufferLists(
ULONG RingCapacity = Ctx->Device.Send.Capacity; ULONG RingCapacity = Ctx->Device.Send.Capacity;
/* Allocate space for packets in the ring. */ /* Allocate space for packets in the ring. */
ULONG RingHead = InterlockedGetU(&Ring->Head); ULONG RingHead = ReadULongAcquire(&Ring->Head);
if (Status = NDIS_STATUS_ADAPTER_NOT_READY, RingHead >= RingCapacity) if (Status = NDIS_STATUS_ADAPTER_NOT_READY, RingHead >= RingCapacity)
goto skipNbl; goto skipNbl;
@ -325,7 +324,7 @@ TunSendNetBufferLists(
{ {
NET_BUFFER_LIST *CompletedNbl = Ctx->Device.Send.ActiveNbls.Head; NET_BUFFER_LIST *CompletedNbl = Ctx->Device.Send.ActiveNbls.Head;
Ctx->Device.Send.ActiveNbls.Head = NET_BUFFER_LIST_NEXT_NBL_EX(CompletedNbl); Ctx->Device.Send.ActiveNbls.Head = NET_BUFFER_LIST_NEXT_NBL_EX(CompletedNbl);
InterlockedSetU(&Ring->Tail, TunNblGetOffset(CompletedNbl)); WriteULongRelease(&Ring->Tail, TunNblGetOffset(CompletedNbl));
KeSetEvent(Ctx->Device.Send.TailMoved, IO_NETWORK_INCREMENT, FALSE); KeSetEvent(Ctx->Device.Send.TailMoved, IO_NETWORK_INCREMENT, FALSE);
NdisMSendNetBufferListsComplete( NdisMSendNetBufferListsComplete(
Ctx->MiniportAdapterHandle, CompletedNbl, NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL); Ctx->MiniportAdapterHandle, CompletedNbl, NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL);
@ -343,11 +342,11 @@ skipNbl:
ExReleaseSpinLockShared(&Ctx->TransitionLock, Irql); ExReleaseSpinLockShared(&Ctx->TransitionLock, Irql);
NdisMSendNetBufferListsComplete(Ctx->MiniportAdapterHandle, NetBufferLists, 0); NdisMSendNetBufferListsComplete(Ctx->MiniportAdapterHandle, NetBufferLists, 0);
updateStatistics: updateStatistics:
InterlockedAdd64((LONG64 *)&Ctx->Statistics.ifHCOutOctets, SentPacketsSize); InterlockedAddNoFence64((LONG64 *)&Ctx->Statistics.ifHCOutOctets, SentPacketsSize);
InterlockedAdd64((LONG64 *)&Ctx->Statistics.ifHCOutUcastOctets, SentPacketsSize); InterlockedAddNoFence64((LONG64 *)&Ctx->Statistics.ifHCOutUcastOctets, SentPacketsSize);
InterlockedAdd64((LONG64 *)&Ctx->Statistics.ifHCOutUcastPkts, SentPacketsCount); InterlockedAddNoFence64((LONG64 *)&Ctx->Statistics.ifHCOutUcastPkts, SentPacketsCount);
InterlockedAdd64((LONG64 *)&Ctx->Statistics.ifOutErrors, ErrorPacketsCount); InterlockedAddNoFence64((LONG64 *)&Ctx->Statistics.ifOutErrors, ErrorPacketsCount);
InterlockedAdd64((LONG64 *)&Ctx->Statistics.ifOutDiscards, DiscardedPacketsCount); InterlockedAddNoFence64((LONG64 *)&Ctx->Statistics.ifOutDiscards, DiscardedPacketsCount);
} }
static MINIPORT_CANCEL_SEND TunCancelSend; static MINIPORT_CANCEL_SEND TunCancelSend;
@ -393,15 +392,15 @@ TunReturnNetBufferLists(NDIS_HANDLE MiniportAdapterContext, PNET_BUFFER_LIST Net
if (!Ctx->Device.Receive.ActiveNbls.Head) if (!Ctx->Device.Receive.ActiveNbls.Head)
KeSetEvent(&Ctx->Device.Receive.ActiveNbls.Empty, IO_NO_INCREMENT, FALSE); KeSetEvent(&Ctx->Device.Receive.ActiveNbls.Empty, IO_NO_INCREMENT, FALSE);
KeReleaseInStackQueuedSpinLock(&LockHandle); KeReleaseInStackQueuedSpinLock(&LockHandle);
InterlockedSetU(&Ring->Head, TunNblGetOffset(CompletedNbl)); WriteULongRelease(&Ring->Head, TunNblGetOffset(CompletedNbl));
NdisFreeNetBufferList(CompletedNbl); NdisFreeNetBufferList(CompletedNbl);
} }
} }
InterlockedAdd64((LONG64 *)&Ctx->Statistics.ifHCInOctets, ReceivedPacketsSize); InterlockedAddNoFence64((LONG64 *)&Ctx->Statistics.ifHCInOctets, ReceivedPacketsSize);
InterlockedAdd64((LONG64 *)&Ctx->Statistics.ifHCInUcastOctets, ReceivedPacketsSize); InterlockedAddNoFence64((LONG64 *)&Ctx->Statistics.ifHCInUcastOctets, ReceivedPacketsSize);
InterlockedAdd64((LONG64 *)&Ctx->Statistics.ifHCInUcastPkts, ReceivedPacketsCount); InterlockedAddNoFence64((LONG64 *)&Ctx->Statistics.ifHCInUcastPkts, ReceivedPacketsCount);
InterlockedAdd64((LONG64 *)&Ctx->Statistics.ifInErrors, ErrorPacketsCount); InterlockedAddNoFence64((LONG64 *)&Ctx->Statistics.ifInErrors, ErrorPacketsCount);
} }
_IRQL_requires_max_(PASSIVE_LEVEL) _IRQL_requires_max_(PASSIVE_LEVEL)
@ -419,20 +418,20 @@ TunProcessReceiveData(_Inout_ TUN_CTX *Ctx)
VOID *Events[] = { &Ctx->Device.Disconnected, Ctx->Device.Receive.TailMoved }; VOID *Events[] = { &Ctx->Device.Disconnected, Ctx->Device.Receive.TailMoved };
ASSERT(RTL_NUMBER_OF(Events) <= THREAD_WAIT_OBJECTS); ASSERT(RTL_NUMBER_OF(Events) <= THREAD_WAIT_OBJECTS);
ULONG RingHead = InterlockedGetU(&Ring->Head); ULONG RingHead = ReadULongAcquire(&Ring->Head);
if (RingHead >= RingCapacity) if (RingHead >= RingCapacity)
goto cleanup; goto cleanup;
while (!KeReadStateEvent(&Ctx->Device.Disconnected)) while (!KeReadStateEvent(&Ctx->Device.Disconnected))
{ {
/* Get next packet from the ring. */ /* Get next packet from the ring. */
ULONG RingTail = InterlockedGetU(&Ring->Tail); ULONG RingTail = ReadULongAcquire(&Ring->Tail);
if (RingHead == RingTail) if (RingHead == RingTail)
{ {
LARGE_INTEGER SpinStart = KeQueryPerformanceCounter(NULL); LARGE_INTEGER SpinStart = KeQueryPerformanceCounter(NULL);
for (;;) for (;;)
{ {
RingTail = InterlockedGetU(&Ring->Tail); RingTail = ReadULongAcquire(&Ring->Tail);
if (RingTail != RingHead) if (RingTail != RingHead)
break; break;
if (KeReadStateEvent(&Ctx->Device.Disconnected)) if (KeReadStateEvent(&Ctx->Device.Disconnected))
@ -444,16 +443,16 @@ TunProcessReceiveData(_Inout_ TUN_CTX *Ctx)
} }
if (RingHead == RingTail) if (RingHead == RingTail)
{ {
InterlockedSet(&Ring->Alertable, TRUE); WriteRelease(&Ring->Alertable, TRUE);
RingTail = InterlockedGetU(&Ring->Tail); RingTail = ReadULongAcquire(&Ring->Tail);
if (RingHead == RingTail) if (RingHead == RingTail)
{ {
KeWaitForMultipleObjects( KeWaitForMultipleObjects(
RTL_NUMBER_OF(Events), Events, WaitAny, Executive, KernelMode, FALSE, NULL, NULL); RTL_NUMBER_OF(Events), Events, WaitAny, Executive, KernelMode, FALSE, NULL, NULL);
InterlockedSet(&Ring->Alertable, FALSE); WriteRelease(&Ring->Alertable, FALSE);
continue; continue;
} }
InterlockedSet(&Ring->Alertable, FALSE); WriteRelease(&Ring->Alertable, FALSE);
KeClearEvent(Ctx->Device.Receive.TailMoved); KeClearEvent(Ctx->Device.Receive.TailMoved);
} }
} }
@ -501,7 +500,7 @@ TunProcessReceiveData(_Inout_ TUN_CTX *Ctx)
TunNblSetOffsetAndMarkActive(Nbl, RingHead); TunNblSetOffsetAndMarkActive(Nbl, RingHead);
KIRQL Irql = ExAcquireSpinLockShared(&Ctx->TransitionLock); KIRQL Irql = ExAcquireSpinLockShared(&Ctx->TransitionLock);
if (!InterlockedGet(&Ctx->Running)) if (!ReadAcquire(&Ctx->Running))
goto cleanupNbl; goto cleanupNbl;
KLOCK_QUEUE_HANDLE LockHandle; KLOCK_QUEUE_HANDLE LockHandle;
@ -530,16 +529,16 @@ TunProcessReceiveData(_Inout_ TUN_CTX *Ctx)
ExReleaseSpinLockShared(&Ctx->TransitionLock, Irql); ExReleaseSpinLockShared(&Ctx->TransitionLock, Irql);
NdisFreeNetBufferList(Nbl); NdisFreeNetBufferList(Nbl);
skipNbl: skipNbl:
InterlockedIncrement64((LONG64 *)&Ctx->Statistics.ifInDiscards); InterlockedIncrementNoFence64((LONG64 *)&Ctx->Statistics.ifInDiscards);
KeWaitForSingleObject(&Ctx->Device.Receive.ActiveNbls.Empty, Executive, KernelMode, FALSE, NULL); KeWaitForSingleObject(&Ctx->Device.Receive.ActiveNbls.Empty, Executive, KernelMode, FALSE, NULL);
InterlockedSetU(&Ring->Head, RingHead); WriteULongRelease(&Ring->Head, RingHead);
} }
/* Wait for all NBLs to return: 1. To prevent race between proceeding and invalidating ring head. 2. To have /* Wait for all NBLs to return: 1. To prevent race between proceeding and invalidating ring head. 2. To have
* TunDispatchUnregisterBuffers() implicitly wait before releasing ring MDL used by NBL(s). */ * TunDispatchUnregisterBuffers() implicitly wait before releasing ring MDL used by NBL(s). */
KeWaitForSingleObject(&Ctx->Device.Receive.ActiveNbls.Empty, Executive, KernelMode, FALSE, NULL); KeWaitForSingleObject(&Ctx->Device.Receive.ActiveNbls.Empty, Executive, KernelMode, FALSE, NULL);
cleanup: cleanup:
InterlockedSetU(&Ring->Head, MAXULONG); WriteULongRelease(&Ring->Head, MAXULONG);
} }
#define IS_POW2(x) ((x) && !((x) & ((x)-1))) #define IS_POW2(x) ((x) && !((x) & ((x)-1)))
@ -595,7 +594,7 @@ TunRegisterBuffers(_Inout_ TUN_CTX *Ctx, _Inout_ IRP *Irp)
if (Status = STATUS_INSUFFICIENT_RESOURCES, !Ctx->Device.Send.Ring) if (Status = STATUS_INSUFFICIENT_RESOURCES, !Ctx->Device.Send.Ring)
goto cleanupSendUnlockPages; goto cleanupSendUnlockPages;
Ctx->Device.Send.RingTail = InterlockedGetU(&Ctx->Device.Send.Ring->Tail); Ctx->Device.Send.RingTail = ReadULongAcquire(&Ctx->Device.Send.Ring->Tail);
if (Status = STATUS_INVALID_PARAMETER, Ctx->Device.Send.RingTail >= Ctx->Device.Send.Capacity) if (Status = STATUS_INVALID_PARAMETER, Ctx->Device.Send.RingTail >= Ctx->Device.Send.Capacity)
goto cleanupSendUnlockPages; goto cleanupSendUnlockPages;
@ -709,7 +708,7 @@ TunUnregisterBuffers(_Inout_ TUN_CTX *Ctx, _In_ FILE_OBJECT *Owner)
} }
ZwClose(Ctx->Device.Receive.Thread); ZwClose(Ctx->Device.Receive.Thread);
InterlockedSetU(&Ctx->Device.Send.Ring->Tail, MAXULONG); WriteULongRelease(&Ctx->Device.Send.Ring->Tail, MAXULONG);
KeSetEvent(Ctx->Device.Send.TailMoved, IO_NO_INCREMENT, FALSE); KeSetEvent(Ctx->Device.Send.TailMoved, IO_NO_INCREMENT, FALSE);
MmUnlockPages(Ctx->Device.Receive.Mdl); MmUnlockPages(Ctx->Device.Receive.Mdl);
@ -925,7 +924,7 @@ static NDIS_STATUS
TunRestart(NDIS_HANDLE MiniportAdapterContext, PNDIS_MINIPORT_RESTART_PARAMETERS MiniportRestartParameters) TunRestart(NDIS_HANDLE MiniportAdapterContext, PNDIS_MINIPORT_RESTART_PARAMETERS MiniportRestartParameters)
{ {
TUN_CTX *Ctx = (TUN_CTX *)MiniportAdapterContext; TUN_CTX *Ctx = (TUN_CTX *)MiniportAdapterContext;
InterlockedSet(&Ctx->Running, TRUE); WriteRelease(&Ctx->Running, TRUE);
return NDIS_STATUS_SUCCESS; return NDIS_STATUS_SUCCESS;
} }
@ -936,7 +935,7 @@ TunPause(NDIS_HANDLE MiniportAdapterContext, PNDIS_MINIPORT_PAUSE_PARAMETERS Min
{ {
TUN_CTX *Ctx = (TUN_CTX *)MiniportAdapterContext; TUN_CTX *Ctx = (TUN_CTX *)MiniportAdapterContext;
InterlockedSet(&Ctx->Running, FALSE); WriteRelease(&Ctx->Running, FALSE);
ExReleaseSpinLockExclusive( ExReleaseSpinLockExclusive(
&Ctx->TransitionLock, &Ctx->TransitionLock,
ExAcquireSpinLockExclusive(&Ctx->TransitionLock)); /* Ensure above change is visible to all readers. */ ExAcquireSpinLockExclusive(&Ctx->TransitionLock)); /* Ensure above change is visible to all readers. */
@ -1130,9 +1129,10 @@ TunHaltEx(NDIS_HANDLE MiniportAdapterContext, NDIS_HALT_ACTION HaltAction)
ExAcquireSpinLockExclusive(&Ctx->TransitionLock)); /* Ensure above change is visible to all readers. */ ExAcquireSpinLockExclusive(&Ctx->TransitionLock)); /* Ensure above change is visible to all readers. */
NdisFreeNetBufferListPool(Ctx->NblPool); NdisFreeNetBufferListPool(Ctx->NblPool);
InterlockedSetPointer(&Ctx->MiniportAdapterHandle, NULL); #pragma warning(suppress : 6387)
#pragma warning(suppress : 28175) WritePointerNoFence(&Ctx->MiniportAdapterHandle, NULL);
InterlockedSetPointer(&Ctx->FunctionalDeviceObject->Reserved, NULL); #pragma warning(suppress : 6387 28175)
WritePointerNoFence(&Ctx->FunctionalDeviceObject->Reserved, NULL);
KeEnterCriticalRegion(); KeEnterCriticalRegion();
ExAcquireResourceExclusiveLite(&TunDispatchCtxGuard, TRUE); /* Ensure above change is visible to all readers. */ ExAcquireResourceExclusiveLite(&TunDispatchCtxGuard, TRUE); /* Ensure above change is visible to all readers. */
ExReleaseResourceLite(&TunDispatchCtxGuard); ExReleaseResourceLite(&TunDispatchCtxGuard);
@ -1241,16 +1241,16 @@ TunOidQuery(_Inout_ TUN_CTX *Ctx, _Inout_ NDIS_OID_REQUEST *OidRequest)
case OID_GEN_XMIT_OK: case OID_GEN_XMIT_OK:
return TunOidQueryWrite32or64( return TunOidQueryWrite32or64(
OidRequest, OidRequest,
InterlockedGet64((LONG64 *)&Ctx->Statistics.ifHCOutUcastPkts) + ReadNoFence64((LONG64 *)&Ctx->Statistics.ifHCOutUcastPkts) +
InterlockedGet64((LONG64 *)&Ctx->Statistics.ifHCOutMulticastPkts) + ReadNoFence64((LONG64 *)&Ctx->Statistics.ifHCOutMulticastPkts) +
InterlockedGet64((LONG64 *)&Ctx->Statistics.ifHCOutBroadcastPkts)); ReadNoFence64((LONG64 *)&Ctx->Statistics.ifHCOutBroadcastPkts));
case OID_GEN_RCV_OK: case OID_GEN_RCV_OK:
return TunOidQueryWrite32or64( return TunOidQueryWrite32or64(
OidRequest, OidRequest,
InterlockedGet64((LONG64 *)&Ctx->Statistics.ifHCInUcastPkts) + ReadNoFence64((LONG64 *)&Ctx->Statistics.ifHCInUcastPkts) +
InterlockedGet64((LONG64 *)&Ctx->Statistics.ifHCInMulticastPkts) + ReadNoFence64((LONG64 *)&Ctx->Statistics.ifHCInMulticastPkts) +
InterlockedGet64((LONG64 *)&Ctx->Statistics.ifHCInBroadcastPkts)); ReadNoFence64((LONG64 *)&Ctx->Statistics.ifHCInBroadcastPkts));
case OID_GEN_STATISTICS: case OID_GEN_STATISTICS:
return TunOidQueryWriteBuf(OidRequest, &Ctx->Statistics, (ULONG)sizeof(Ctx->Statistics)); return TunOidQueryWriteBuf(OidRequest, &Ctx->Statistics, (ULONG)sizeof(Ctx->Statistics));

5
wintun.vcxproj
View File

@ -125,7 +125,7 @@
<ItemDefinitionGroup> <ItemDefinitionGroup>
<ClCompile> <ClCompile>
<PreprocessorDefinitions>NDIS_MINIPORT_DRIVER=1;NDIS620_MINIPORT=1;NDIS683_MINIPORT=1;NDIS_WDM=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>NDIS_MINIPORT_DRIVER=1;NDIS620_MINIPORT=1;NDIS683_MINIPORT=1;NDIS_WDM=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalOptions>/volatile:ms %(AdditionalOptions)</AdditionalOptions> <AdditionalOptions>/volatile:iso %(AdditionalOptions)</AdditionalOptions>
<EnablePREfast>true</EnablePREfast> <EnablePREfast>true</EnablePREfast>
</ClCompile> </ClCompile>
<ResourceCompile> <ResourceCompile>
@ -171,9 +171,8 @@
<FilesToPackage Include="$(TargetPath)" Condition="'$(ConfigurationType)'=='Driver' or '$(ConfigurationType)'=='DynamicLibrary'" /> <FilesToPackage Include="$(TargetPath)" Condition="'$(ConfigurationType)'=='Driver' or '$(ConfigurationType)'=='DynamicLibrary'" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ClInclude Include="atomic.h" />
<ClInclude Include="undocumented.h" /> <ClInclude Include="undocumented.h" />
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets" /> <ImportGroup Label="ExtensionTargets" />
</Project> </Project>

3
wintun.vcxproj.filters
View File

@ -33,8 +33,5 @@
<ClInclude Include="undocumented.h"> <ClInclude Include="undocumented.h">
<Filter>Header Files</Filter> <Filter>Header Files</Filter>
</ClInclude> </ClInclude>
<ClInclude Include="atomic.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup> </ItemGroup>
</Project> </Project>