Windows驱动编程之NDIS(VPN)

for (i = 0; i < 8; i++){    readBytes = 0;    memset(&ol, 0, sizeof(ol));    ol.hEvent = g_ioEvent;    if (!ReadFile(hDevice, &rr, sizeof(rr), NULL, &ol))    {        if (GetLastError() != ERROR_IO_PENDING)        {            OutputDebugString(L"ReadFile Error!");            goto finish;        }    }    for (;;)    {        dwRes = WaitForMultipleObjects(            sizeof(events) / sizeof(events[0]),            events,            FALSE,            waitTimeout);    ......    }    ......}

IoCsqInsertIrp(&adapter->PendingReadIrpQueue.CsqQueue, Irp, NULL);tapProcessSendPacketQueue(adapter);ntStatus = STATUS_PENDING;

IoMarkIrpPending(irp);InsertTailList(&g_pendedIoRequests, &irp->Tail.Overlay.ListEntry);status = STATUS_PENDING;KeSetEvent(&ThreadEvent, IO_NO_INCREMENT, FALSE);

```if(adapter->TapFileObject == NULL){    //    // Complete all NBLs and return if adapter not ready.    //    tapSendNetBufferListsComplete(        adapter,        NetBufferLists,        NDIS_STATUS_SUCCESS,        DispatchLevel        );     return;}if(!Adapter->LogicalMediaState){    status = NDIS_STATUS_MEDIA_DISCONNECTED;}else if(Adapter->CurrentPowerState != NdisDeviceStateD0){    status = NDIS_STATUS_LOW_POWER_STATE;}else if(Adapter->ResetInProgress){    status = NDIS_STATUS_RESET_IN_PROGRESS;}else{    switch(Adapter->Locked.AdapterState)    {    case MiniportPausingState:    case MiniportPausedState:        status = NDIS_STATUS_PAUSED;        break;     case MiniportHaltedState:        status = NDIS_STATUS_INVALID_STATE;        break;     default:        status = NDIS_STATUS_SUCCESS;        break;    }}```

// Minimum packet size is size of Ethernet plus IPv4 headers.ASSERT(packetLength >= (ETHERNET_HEADER_SIZE + IP_HEADER_SIZE)); if(packetLength < (ETHERNET_HEADER_SIZE + IP_HEADER_SIZE)){    return FALSE;} // Maximum size should be Ethernet header size plus MTU plus modest pad for// VLAN tag.ASSERT( packetLength <= (ETHERNET_HEADER_SIZE + VLAN_TAG_SIZE + Adapter->MtuSize)); if(packetLength > (ETHERNET_HEADER_SIZE + VLAN_TAG_SIZE + Adapter->MtuSize)){    return FALSE;}

#define MACADDR_SIZE    6typedef unsigned char MACADDR[MACADDR_SIZE];typedef struct{    MACADDR dest;               /* destination eth addr    */    MACADDR src;                /* source ether addr    */    USHORT proto;               /* packet type ID field    */} ETH_HEADER, *PETH_HEADER;

#define NDIS_ETH_TYPE_IPV4              0x0800  // IPV4#define NDIS_ETH_TYPE_ARP               0x0806  // ARP#define NDIS_ETH_TYPE_IPV6              0x86dd  // IPV6

case TAP_WIN_IOCTL_CONFIG_DHCP_MASQ:    {        if(inBufLength >= sizeof(IPADDR)*4)        {            adapter->m_dhcp_enabled = FALSE;            adapter->m_dhcp_server_arp = FALSE;            adapter->m_dhcp_user_supplied_options_buffer_len = 0;             // Adapter IP addr / netmask            adapter->m_dhcp_addr =                ((IPADDR*) (Irp->AssociatedIrp.SystemBuffer))[0];            adapter->m_dhcp_netmask =                ((IPADDR*) (Irp->AssociatedIrp.SystemBuffer))[1];             // IP addr of DHCP masq server            adapter->m_dhcp_server_ip =                ((IPADDR*) (Irp->AssociatedIrp.SystemBuffer))[2];             // Lease time in seconds            adapter->m_dhcp_lease_time =                ((IPADDR*) (Irp->AssociatedIrp.SystemBuffer))[3];             GenerateRelatedMAC(                adapter->m_dhcp_server_mac,                adapter->CurrentAddress,                2                );             adapter->m_dhcp_enabled = TRUE;            adapter->m_dhcp_server_arp = TRUE;             CheckIfDhcpAndTunMode (adapter);             Irp->IoStatus.Information = 1; // Simple boolean value             DEBUGP (("[Boom] Configured DHCP MASQ.n"));        }        else        {            NOTE_ERROR();            Irp->IoStatus.Status = ntStatus = STATUS_INVALID_PARAMETER;        }    }    break;

```// Allocate TAP packet memorytapPacket = (PTAP_PACKET )NdisAllocateMemoryWithTagPriority(                Adapter->MiniportAdapterHandle,                TAP_PACKET_SIZE (packetLength+addHeaderSize),                TAP_PACKET_TAG,                NormalPoolPriority                );// 提取Buf数据packetData = NdisGetDataBuffer(NetBuffer,packetLength,tapPacket->m_Data+addHeaderSize,1,0); // 拷贝数据到tapPacket+addHeaderSize后的位置，预留出来addHeaderSizeif(packetData != (tapPacket->m_Data+addHeaderSize)){    // Packet data was contiguous and not yet copied to m_Data.    NdisMoveMemory(tapPacket->m_Data+addHeaderSize,packetData,packetLength);} // 填充addHeaderSize大小数据if(addHeaderSize > 0){    // Add an 802.1Q header between the ethernet header and the payload    NdisMoveMemory(tapPacket->m_Data,tapPacket->m_Data+addHeaderSize,ETHERNET_HEADER_SIZE-2);    PETH_HEADER header = (PETH_HEADER)tapPacket->m_Data;    PETH_8021Q_HEADER tag = (PETH_8021Q_HEADER)(header+1);    header->proto = htons(0x8100);    USHORT tagValue = 0;    tagValue |= packetPriority.TagHeader.UserPriority<<13;    tagValue |= packetPriority.TagHeader.VlanId & 0xFFF;    tag->Tag = tagValue;     packetLength += addHeaderSize;} // DHCP的处理从数据链路层(ETH_HEADER) 到 网络层(IP_HDR) 到  UDPHDR(传输层) DHCP......const ETH_HEADER *eth = (ETH_HEADER *) tapPacket->m_Data;const IPHDR *ip = (IPHDR *) (tapPacket->m_Data + sizeof (ETH_HEADER));const UDPHDR *udp = (UDPHDR *) (tapPacket->m_Data + sizeof (ETH_HEADER) + sizeof (IPHDR));......else if (packetLength >= sizeof (ETH_HEADER) + sizeof (IPHDR) + sizeof (UDPHDR) + sizeof (DHCP)&& eth->proto == htons (NDIS_ETH_TYPE_IPV4)&& ip->version_len == 0x45 // IPv4, 20 byte header&& ip->protocol == IPPROTO_UDP&& udp->dest == htons (BOOTPS_PORT))...... // 首先要从Ethernet->proto确认协议ETH_HEADER *e;e = (ETH_HEADER *) tapPacket->m_Data;switch (ntohs (e->proto)) // ARP处理if (packetLength != sizeof (ARP_PACKET)){    goto no_queue;} ProcessARP (    Adapter,    (PARP_PACKET) tapPacket->m_Data,    Adapter->m_localIP,    Adapter->m_remoteNetwork,    Adapter->m_remoteNetmask,    Adapter->m_TapToUser.dest    ); // ipv4/ipv6处理case NDIS_ETH_TYPE_IPV4:     // Make sure that packet is large enough to be IPv4.    if (packetLength < (ETHERNET_HEADER_SIZE + IP_HEADER_SIZE))    {        goto no_queue;    }     // Only accept directed packets, not broadcasts.    if (memcmp (e, &Adapter->m_TapToUser, ETHERNET_HEADER_SIZE))    {        goto no_queue;    }     // Packet looks like IPv4, queue it. :-)    tapPacket->m_SizeFlags |= TP_TUN;    break; case NDIS_ETH_TYPE_IPV6:    // Make sure that packet is large enough to be IPv6.    if (packetLength < (ETHERNET_HEADER_SIZE + IPV6_HEADER_SIZE))    {        goto no_queue;    }     // Broadcasts and multicasts are handled specially    // (to be implemented)     // Neighbor discovery packets to fe80::8 are special    // OpenVPN sets this next-hop to signal "handled by tapdrv"    if ( HandleIPv6NeighborDiscovery(Adapter,tapPacket->m_Data,                                     packetLength) )    {        goto no_queue;    }     // Packet looks like IPv6, queue it. :-)    tapPacket->m_SizeFlags |= TP_TUN;}```

static IPV6ADDR IPV6_NS_TARGET_MCAST =    { 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,          0x00, 0x00, 0x00, 0x01, 0xff, 0x00, 0x00, 0x08 };static IPV6ADDR IPV6_NS_TARGET_UNICAST =    { 0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08 };if ( memcmp( ipv6->daddr, IPV6_NS_TARGET_MCAST,    sizeof(IPV6ADDR) ) != 0 &&    memcmp( ipv6->daddr, IPV6_NS_TARGET_UNICAST,    sizeof(IPV6ADDR) ) != 0 ){    return FALSE;                // wrong target address} // ICMPv6 type+code must be 135/0 for NSif ( icmpv6_ns->type != ICMPV6_TYPE_NS ||    icmpv6_ns->code != ICMPV6_CODE_0 ){    return FALSE;                // wrong ICMPv6 type} // 需要计算和填充checksumicmpv6_csum = icmpv6_checksum (                (UCHAR*) &(na->icmpv6),                icmpv6_len,                na->ipv6.saddr,                na->ipv6.daddr                );

............// 拿到应用层传递过来的数据包unsigned char* packetBuffer = (unsigned char *) Irp->AssociatedIrp.SystemBuffer;ULONG packetLength = irpSp->Parameters.Write.Length;PVOID packetPriority = 0; DUMP_PACKET ("IRP_MJ_WRITE ETH",    packetBuffer,    packetLength); // 8021Q处理packetPriority = TapStrip8021Q(&packetBuffer, &packetLength); // 发送包到虚拟网卡ntStatus = TapSharedSendPacket(adapter,Irp,packetBuffer,packetLength,packetPriority,NULL,0);

// 小于60byte拷贝副本在MDL映射到NetBufferList// Copy packet data to flat buffer.......if(PrefixLength > 0){    NdisMoveMemory(allocBuffer, PrefixData, PrefixLength);}NdisMoveMemory (allocBuffer + PrefixLength, PacketBuffer, PacketLength);NdisZeroMemory(allocBuffer + fullLength, paddedLength - fullLength);......// 标记注入包TAP_RX_NBL_FLAGS_CLEAR_ALL(netBufferList);TAP_RX_NBL_FLAG_SET(netBufferList,TAP_RX_NBL_FLAGS_IS_INJECTED); // 设置pendingIoMarkIrpPending(Irp);   IoSetCancelRoutine(Irp,NULL); // Stash IRP pointer in NBL MiniportReserved[0] field.netBufferList->MiniportReserved[0] = Irp;netBufferList->MiniportReserved[1] = NULL; NET_BUFFER_LIST_INFO(netBufferList, Ieee8021QNetBufferListInfo) = PacketPriority; // Increment in-flight receive NBL count.nblCount = NdisInterlockedIncrement(&Adapter->ReceiveNblInFlightCount);ASSERT(nblCount > 0 ); //// Indicate the packet// -------------------// This NBL contains the complete packet including Ethernet header and payload.//// 完整的数据调用ReceiveNet完成包NdisMIndicateReceiveNetBufferLists(    Adapter->MiniportAdapterHandle,    netBufferList,    NDIS_DEFAULT_PORT_NUMBER,    1,      // NumberOfNetBufferLists    0       // 清除所有标志    ); // 注意返回标志是Pendingreturn STATUS_PENDING;

if (!l_Adapter->m_tun && ((l_IrpSp->Parameters.Write.Length) >= ETHERNET_HEADER_SIZE)){    ......    NdisMEthIndicateReceive    (l_Adapter->m_MiniportAdapterHandle,        (NDIS_HANDLE)l_Adapter,        (unsigned char*)p_IRP->AssociatedIrp.SystemBuffer,        ETHERNET_HEADER_SIZE,        (unsigned char*)p_IRP->AssociatedIrp.SystemBuffer + ETHERNET_HEADER_SIZE,        l_IrpSp->Parameters.Write.Length - ETHERNET_HEADER_SIZE,        l_IrpSp->Parameters.Write.Length - ETHERNET_HEADER_SIZE);    NdisMEthIndicateReceiveComplete();    p_IRP->IoStatus.Status = l_Status = STATUS_SUCCESS;    ......}else if (l_Adapter->m_tun && ((l_IrpSp->Parameters.Write.Length) >= IP_HEADER_SIZE)){    ......    if (IPH_GET_VER(((IPHDR*)p_IRP->AssociatedIrp.SystemBuffer)->version_len) == 6)    {        p_UserToTap = &l_Adapter->m_UserToTap_IPv6;    }    ......    NdisMEthIndicateReceive    (l_Adapter->m_MiniportAdapterHandle,        (NDIS_HANDLE)l_Adapter,        (unsigned char*)p_UserToTap,        sizeof(ETH_HEADER),        (unsigned char*)p_IRP->AssociatedIrp.SystemBuffer,        l_IrpSp->Parameters.Write.Length,        l_IrpSp->Parameters.Write.Length);     NdisMEthIndicateReceiveComplete(l_Adapter->m_MiniportAdapterHandle);     p_IRP->IoStatus.Status = l_Status = STATUS_SUCCESS;    ......}

// WG_IOCTL_INTERFACE_HAS_LISTEN_PORT 初始化WSK// WIREGUARD_INTERFACE_HAS_PRIVATE_KEY 初始化Key// WireGuard首先要先发送keystruct{    WIREGUARD_INTERFACE Interface;    WIREGUARD_PEER DemoServer;    WIREGUARD_ALLOWED_IP AllV4;} Config = { .Interface = { .Flags = WIREGUARD_INTERFACE_HAS_PRIVATE_KEY, .PeersCount = 1 },             .DemoServer = { .Flags = WIREGUARD_PEER_HAS_PUBLIC_KEY | WIREGUARD_PEER_HAS_ENDPOINT,                             .AllowedIPsCount = 1 },             .AllV4 = { .AddressFamily = AF_INET } };// 通过发送Key初始化WIREGUARD_SET_CONFIGURATION_FUNC WireGuardSetConfiguration;_Use_decl_annotations_BOOL WINAPIWireGuardSetConfiguration(WIREGUARD_ADAPTER *Adapter, const WIREGUARD_INTERFACE *Config, DWORD Bytes){    HANDLE ControlFile = AdapterOpenDeviceObject(Adapter);    if (ControlFile == INVALID_HANDLE_VALUE)        return FALSE;    if (!DeviceIoControl(ControlFile, WG_IOCTL_SET, NULL, 0, (VOID *)Config, Bytes, &Bytes, NULL))    {        DWORD LastError = GetLastError();        CloseHandle(ControlFile);        SetLastError(LastError);        return FALSE;    }    CloseHandle(ControlFile);    return TRUE;}// WireGuardSetAdapterState把WIREGUARD_ADAPTER_STATE_UP参数传递到内核Log(WIREGUARD_LOG_INFO, L"Setting configuration and adapter up");if (!WireGuardSetConfiguration(Adapter, &Config.Interface, sizeof(Config)) ||    !WireGuardSetAdapterState(Adapter, WIREGUARD_ADAPTER_STATE_UP)){    LastError = LogError(L"Failed to set configuration and adapter up", GetLastError());    goto cleanupAdapter;}

// WIREGUARD_ADAPTER_STATE_UP内核处理case WG_IOCTL_SET_ADAPTER_STATE:    AdapterState(DeviceObject, Irp);AdapterState{    case WG_IOCTL_ADAPTER_STATE_UP:        Irp->IoStatus.Status = Up(Wg);}WIREGUARD_ADAPTER_STATE_UP --> Up(Wg) --> SocketInit(Wg, Wg->IncomingPort); // 上文说到也可以通过发送WG_IOCTL_INTERFACE_HAS_LISTEN_PORT来实现,他会执行以下代码NTSTATUS Status;if (IoctlInterface.Flags & WG_IOCTL_INTERFACE_HAS_LISTEN_PORT){    Status = SetListenPort(Wg, IoctlInterface.ListenPort);    if (!NT_SUCCESS(Status))        goto cleanupLock;}WG_IOCTL_INTERFACE_HAS_LISTEN_PORT --> SetListenPort() --> SocketInit()

// 初始化固定大小的SendCtxStatus = ExInitializeLookasideListEx(    &SocketSendCtxCache, NULL, NULL, NonPagedPool, 0, sizeof(SOCKET_SEND_CTX), MEMORY_TAG, 0); // 初始化/注册WSK_CLIENT_NPIWSK_CLIENT_NPI WskClientNpi = { .Dispatch = &WskAppDispatchV1 };Status = WskRegister(&WskClientNpi, &WskRegistration); // 注册后要捕获WSK NPI(Network Programming Interface)// WSK_INFINITE_WAIT就是要等待到WSK子系统准备好才可以Status = WskCaptureProviderNPI(&WskRegistration, WSK_INFINITE_WAIT, &WskProviderNpi);if (!NT_SUCCESS(Status))    goto cleanupWskRegister; // 成功连接WSK子系统后，WSK_TRANSPORT_LIST_QUERY 检索可用的传输列表Status = WskProviderNpi.Dispatch->WskControlClient(    WskProviderNpi.Client,    WSK_TRANSPORT_LIST_QUERY,    0,    NULL,    WskTransportsSize,    WskTransports,    &WskTransportsSize,    NULL); for (SIZE_T i = 0, n = WskTransportsSize / sizeof(*WskTransports); i < n; ++i){    if (WskTransports[i].SocketType == SOCK_DGRAM && WskTransports[i].Protocol == IPPROTO_UDP)    {        if (WskTransports[i].AddressFamily == AF_UNSPEC)        {            WskHasIpv4Transport = TRUE;            WskHasIpv6Transport = TRUE;        }        else if (WskTransports[i].AddressFamily == AF_INET)            WskHasIpv4Transport = TRUE;        else if (WskTransports[i].AddressFamily == AF_INET6)            WskHasIpv6Transport = TRUE;    }} // 所有套接字自动启动回调, 使用的参数WSK_EVENT_RECEIVE_FROM参数。WSK_EVENT_CALLBACK_CONTROL WskEventCallbackControl = { .NpiId = &NPI_WSK_INTERFACE_ID,                                                       .EventMask = WSK_EVENT_RECEIVE_FROM };Status = WskProviderNpi.Dispatch->WskControlClient(    WskProviderNpi.Client,    WSK_SET_STATIC_EVENT_CALLBACKS,    sizeof(WskEventCallbackControl),    &WskEventCallbackControl,    0,    NULL,    NULL,    NULL); Status = NotifyRouteChange2(AF_INET, RouteNotification, &RoutingGenerationV4, FALSE, &RouteNotifierV4);if (!NT_SUCCESS(Status))    goto cleanupWskProviderNPI;Status = NotifyRouteChange2(AF_INET6, RouteNotification, &RoutingGenerationV6, FALSE, &RouteNotifierV6);if (!NT_SUCCESS(Status))    goto cleanupRouteNotifierV4; // 更具体的参数See Msdn: https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/wsk/nc-wsk-pfn_wsk_control_client

// 如果List支持v4if (WskHasIpv4Transport){    Status = CreateAndBindSocket(Wg, (SOCKADDR *)&Sa4, &New4);    if (!NT_SUCCESS(Status))        goto out;}// 如果List支持v6if (WskHasIpv6Transport){    Sa6.sin6_port = Sa4.sin_port;    Status = CreateAndBindSocket(Wg, (SOCKADDR *)&Sa6, &New6);    if (!NT_SUCCESS(Status))    {        CloseSocket(New4);        New4 = NULL;        if (Status == STATUS_ADDRESS_ALREADY_EXISTS && !Port && Retries++ < 100)            goto retry;        goto out;    }}

// 默认是0.0.0.0和配置传递过来的Port// 初始异步Event/Irp，Irp完成回调触发KeSetEvent EventKeInitializeEvent(&Done, SynchronizationEvent, FALSE);IoInitializeIrp(&I.Irp, sizeof(I.IrpBuffer), 1);IoSetCompletionRoutine(&I.Irp, RaiseEventOnComplete, &Done, TRUE, TRUE, TRUE);// Irp传递参数传递给了NPIDisPathch，WskReceiveFromEvent这个地方赋值了IPPROTO_UDP Receive回调函数static CONST WSK_CLIENT_DATAGRAM_DISPATCH WskClientDatagramDispatch = { .WskReceiveFromEvent = Receive };// 创建新UDP Socket：SOCK_DGRAM IPPROTO_UDPStatus = WskProviderNpi.Dispatch->WskSocket(    WskProviderNpi.Client,    Sa->sa_family,    SOCK_DGRAM,    IPPROTO_UDP,    WSK_FLAG_DATAGRAM_SOCKET,    Socket,    &WskClientDatagramDispatch,    Wg->SocketOwnerProcess,    NULL,    NULL,    &I.Irp); ULONG True = TRUE;if (Sa->sa_family == AF_INET){    // IP_PKTINFO允许启动/禁用v4  LPFN_WSARECVMSG(WSARecvMsg)返回数据包信息    Status = SetSockOpt(Sock, IPPROTO_IP, IP_PKTINFO, &True, sizeof(True));    if (!NT_SUCCESS(Status))        goto cleanupSocket;}else if (Sa->sa_family == AF_INET6){    Status = SetSockOpt(Sock, IPPROTO_IPV6, IPV6_V6ONLY, &True, sizeof(True));    if (!NT_SUCCESS(Status))        goto cleanupSocket;    Status = SetSockOpt(Sock, IPPROTO_IPV6, IPV6_PKTINFO, &True, sizeof(True));    if (!NT_SUCCESS(Status))        goto cleanupSocket;}Status = ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Sock->Dispatch)->WskBind(Sock, Sa, 0, &I.Irp);Status = ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Sock->Dispatch)->WskGetLocalAddress(Sock, Sa, &I.Irp);

    ......    Irql = RcuReadLock();    Keypair = NoiseKeypairGet(RcuDereference(NOISE_KEYPAIR, Peer->Keypairs.CurrentKeypair));    RcuReadUnlock(Irql);    if (!Keypair)        goto outNokey;    // 有key 数据生产消费    PeerGet(Keypair->Entry.Peer);        _Analysis_assume_(NET_BUFFER_LIST_FIRST_NB(Packets.Head)); /* Checked in SendNetBufferLists(). */    NET_BUFFER_LIST_KEYPAIR(Packets.Head) = Keypair;    PacketCreateData(Peer, Packets.Head); outNokey:    // 无key    ......    ......    }

Ret = QueueEnqueuePerDeviceAndPeer(&Wg->EncryptQueue, &Peer->TxQueue, &Wg->EncryptThreads, First);if (Ret == STATUS_PIPE_BROKEN){// 失败处理    QueueEnqueuePerPeer(&Peer->Device->TxQueue, &Peer->TxSerialEntry, First, PACKET_STATE_DEAD);    MulticoreWorkQueueBump(&Wg->EncryptThreads);}if (NT_SUCCESS(Ret) || Ret == STATUS_PIPE_BROKEN)    return;

if (!QueueInsertPerPeer(PeerQueue, Nbl))    return STATUS_BUFFER_TOO_SMALL;/* Then we queue it up in the device queue, which consumes the * packet as soon as it can. */// 排队消费数据包意思，就是加密处理掉这个数据包if (!QueueEnqueuePerDevice(DeviceQueue, DeviceThreads, Nbl))    return STATUS_PIPE_BROKEN;

......EncryptPacket --> ChaCha20Poly1305EncryptMdl// 加密失败包状态会改变State = PACKET_STATE_DEAD;......QueueEnqueuePerPeer(&Peer->Device->TxQueue, &Peer->TxSerialEntry, First, State);ProcessPerPeerWork(&Wg->TxQueue); ProcessPerPeerWork{    PEER_SERIAL_ENTRY *Entry;    while ((Entry = PeerSerialDequeue(WorkQueue)) != NULL)        PeerSerialMaybeRetire(            WorkQueue,            Entry,            PacketPeerTxWork(CONTAINING_RECORD(Entry, WG_PEER, TxSerialEntry), PEER_XMIT_PACKETS_PER_ROUND));} PacketPeerTxWork{    // PacketCreateDataDone负责发送    if (State == PACKET_STATE_CRYPTED)        PacketCreateDataDone(Peer, First);    else        FreeSendNetBufferList(Peer->Device, First, 0);} PacketCreateDataDone {    if (NT_SUCCESS(SocketSendNblsToPeer(Peer, First, &IsKeepalive)) && !IsKeepalive)        TimersDataSent(Peer);}  if (NT_SUCCESS(SocketSendNblsToPeer(Peer, First, &IsKeepalive)) && !IsKeepalive)    TimersDataSent(Peer);

PFN_WSK_SEND_MESSAGES WskSendMessages = ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Socket->Sock->Dispatch)->WskSendMessages;#if NTDDI_VERSION == NTDDI_WIN7    if (NoWskSendMessages)        WskSendMessages = PolyfilledWskSendMessages;#endif    Status = WskSendMessages(        Socket->Sock,        FirstWskBuf,        0,        (PSOCKADDR)&Peer->Endpoint.Addr,        (ULONG)WSA_CMSGDATA_ALIGN(Peer->Endpoint.Cmsg.cmsg_len) + WSA_CMSG_SPACE(0),        &Peer->Endpoint.Cmsg,        &Ctx->Irp);

............while (NetBufferListQueueLength(&Peer->StagedPacketQueue) > MAX_STAGED_PACKETS){    NET_BUFFER_LIST *NblToDiscard = NetBufferListDequeue(&Peer->StagedPacketQueue);    _Analysis_assume_(NblToDiscard); /* NetBufferListQueueLength() > MAX_STAGED_PACKETS implies                                        NetBufferListDequeue() returns a NBL. */    NET_BUFFER_LIST_STATUS(NblToDiscard) = NDIS_STATUS_FAILURE;    ++Wg->Statistics.ifOutDiscards;    FreeSendNetBufferList(Wg, NblToDiscard, CompleteFlags | NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL);}NetBufferListEnqueue(&Peer->StagedPacketQueue, Nbl);KeReleaseSpinLock(&Peer->StagedPacketQueue.Lock, Irql); PacketSendStagedPackets(Peer);............

for (NET_BUFFER_LIST *Nbl = First, *NextNbl; Nbl; Nbl = NextNbl){    NextNbl = NET_BUFFER_LIST_NEXT_NBL(Nbl);    NET_BUFFER_LIST_NEXT_NBL(Nbl) = NULL;    WSK_DATAGRAM_INDICATION *DatagramIndication = NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl);    SOCKET *Socket = (SOCKET *)DatagramIndication->Next;    DatagramIndication->Next = NULL;    ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Socket->Sock->Dispatch)->WskRelease(Socket->Sock, DatagramIndication);    MemFreeNetBufferList(Nbl);    ExReleaseRundownProtection(&Socket->ItemsInFlight);}

// PacketHandshakeRxWorker线程处理Recv调用NdisMIndicateReceiveNetBufferLists：if (First)    NdisMIndicateReceiveNetBufferLists(First->SourceHandle, First, NDIS_DEFAULT_PORT_NUMBER, NumNbls, 0);

// 加密线程Status = MulticoreWorkQueueInit(&Wg->EncryptThreads, PacketEncryptWorker);if (!NT_SUCCESS(Status))    goto cleanupHandshakeRxQueue; // 解密线程Status = MulticoreWorkQueueInit(&Wg->DecryptThreads, PacketDecryptWorker);if (!NT_SUCCESS(Status))    goto cleanupEncryptThreads; // HANDSHAKE_TX_SEND的时候，调用PacketSendHandshakeInitiationStatus = MulticoreWorkQueueInit(&Wg->HandshakeTxThreads, PacketHandshakeTxWorker);if (!NT_SUCCESS(Status))    goto cleanupDecryptThreads;PacketSendHandshakeInitiation{    if (NoiseHandshakeCreateInitiation(&Packet, &Peer->Handshake))    {        CookieAddMacToPacket(&Packet, sizeof(Packet), Peer);        TimersAnyAuthenticatedPacketTraversal(Peer);        TimersAnyAuthenticatedPacketSent(Peer);        WriteNoFence64(&Peer->LastSentHandshake, KeQueryInterruptTime());        SocketSendBufferToPeer(Peer, &Packet, sizeof(Packet));        TimersHandshakeInitiated(Peer);    }}SocketSendBufferToPeer-->SocketResolvePeerEndpoint; // PacketHandshakeRxWorker处理wsk recvStatus = MulticoreWorkQueueInit(&Wg->HandshakeRxThreads, PacketHandshakeRxWorker);if (!NT_SUCCESS(Status))    goto cleanupHandshakeTxThreads;