Skip to content

perf: Jobify SendBatchedMessages in UnityTransport [MTT-4999] #2304

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
simon-lemay-unity merged 2 commits into from
Nov 10, 2022
Merged

perf: Jobify SendBatchedMessages in UnityTransport [MTT-4999] #2304

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0f76e2e
Make ErrorUtilities usable from Burst
simon-lemay-unity Nov 10, 2022
ec1ffdf
Jobify SendBatchedMessages
simon-lemay-unity Nov 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 83 additions & 64 deletions com.unity.netcode.gameobjects/Runtime/Transports/UTP/UnityTransport.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@
using UnityEngine;
using NetcodeNetworkEvent = Unity.Netcode.NetworkEvent;
using TransportNetworkEvent = Unity.Networking.Transport.NetworkEvent;
using Unity.Burst;
using Unity.Collections.LowLevel.Unsafe;
using Unity.Collections;
using Unity.Jobs;
using Unity.Networking.Transport;
using Unity.Networking.Transport.Relay;
using Unity.Networking.Transport.Utilities;
Expand Down Expand Up @@ -51,50 +53,48 @@ void CreateDriver(
/// </summary>
public static class ErrorUtilities
{
private const string k_NetworkSuccess = "Success";
private const string k_NetworkIdMismatch = "NetworkId is invalid, likely caused by stale connection {0}.";
private const string k_NetworkVersionMismatch = "NetworkVersion is invalid, likely caused by stale connection {0}.";
private const string k_NetworkStateMismatch = "Sending data while connecting on connection {0} is not allowed.";
private const string k_NetworkPacketOverflow = "Unable to allocate packet due to buffer overflow.";
private const string k_NetworkSendQueueFull = "Currently unable to queue packet as there is too many in-flight packets. This could be because the send queue size ('Max Send Queue Size') is too small.";
private const string k_NetworkHeaderInvalid = "Invalid Unity Transport Protocol header.";
private const string k_NetworkDriverParallelForErr = "The parallel network driver needs to process a single unique connection per job, processing a single connection multiple times in a parallel for is not supported.";
private const string k_NetworkSendHandleInvalid = "Invalid NetworkInterface Send Handle. Likely caused by pipeline send data corruption.";
private const string k_NetworkArgumentMismatch = "Invalid NetworkEndpoint Arguments.";
private static readonly FixedString128Bytes k_NetworkSuccess = "Success";
private static readonly FixedString128Bytes k_NetworkIdMismatch = "Invalid connection ID {0}.";
private static readonly FixedString128Bytes k_NetworkVersionMismatch = "Connection ID is invalid. Likely caused by sending on stale connection {0}.";
private static readonly FixedString128Bytes k_NetworkStateMismatch = "Connection state is invalid. Likely caused by sending on connection {0} which is stale or still connecting.";
private static readonly FixedString128Bytes k_NetworkPacketOverflow = "Packet is too large to be allocated by the transport.";
private static readonly FixedString128Bytes k_NetworkSendQueueFull = "Unable to queue packet in the transport. Likely caused by send queue size ('Max Send Queue Size') being too small.";

/// <summary>
/// Convert error code to human readable error message.
/// Convert a UTP error code to human-readable error message.
/// </summary>
/// <param name="error">Status code of the error</param>
/// <param name="connectionId">Subject connection ID of the error</param>
/// <returns>Human readable error message.</returns>
/// <param name="error">UTP error code.</param>
/// <param name="connectionId">ID of the connection on which the error occurred.</param>
/// <returns>Human-readable error message.</returns>
public static string ErrorToString(Networking.Transport.Error.StatusCode error, ulong connectionId)
{
switch (error)
return ErrorToString((int)error, connectionId);
}

internal static string ErrorToString(int error, ulong connectionId)
{
return ErrorToFixedString(error, connectionId).ToString();
}

internal static FixedString128Bytes ErrorToFixedString(int error, ulong connectionId)
{
switch ((Networking.Transport.Error.StatusCode)error)
{
case Networking.Transport.Error.StatusCode.Success:
return k_NetworkSuccess;
case Networking.Transport.Error.StatusCode.NetworkIdMismatch:
return string.Format(k_NetworkIdMismatch, connectionId);
return FixedString.Format(k_NetworkIdMismatch, connectionId);
case Networking.Transport.Error.StatusCode.NetworkVersionMismatch:
return string.Format(k_NetworkVersionMismatch, connectionId);
return FixedString.Format(k_NetworkVersionMismatch, connectionId);
case Networking.Transport.Error.StatusCode.NetworkStateMismatch:
return string.Format(k_NetworkStateMismatch, connectionId);
return FixedString.Format(k_NetworkStateMismatch, connectionId);
case Networking.Transport.Error.StatusCode.NetworkPacketOverflow:
return k_NetworkPacketOverflow;
case Networking.Transport.Error.StatusCode.NetworkSendQueueFull:
return k_NetworkSendQueueFull;
case Networking.Transport.Error.StatusCode.NetworkHeaderInvalid:
return k_NetworkHeaderInvalid;
case Networking.Transport.Error.StatusCode.NetworkDriverParallelForErr:
return k_NetworkDriverParallelForErr;
case Networking.Transport.Error.StatusCode.NetworkSendHandleInvalid:
return k_NetworkSendHandleInvalid;
case Networking.Transport.Error.StatusCode.NetworkArgumentMismatch:
return k_NetworkArgumentMismatch;
default:
return FixedString.Format("Unknown error code {0}.", error);
}

return $"Unknown ErrorCode {Enum.GetName(typeof(Networking.Transport.Error.StatusCode), error)}";
}
}

Expand Down Expand Up @@ -676,55 +676,74 @@ private bool StartRelayServer()
}
}

// Send as many batched messages from the queue as possible.
private void SendBatchedMessages(SendTarget sendTarget, BatchedSendQueue queue)
[BurstCompile]
private struct SendBatchedMessagesJob : IJob
{
var clientId = sendTarget.ClientId;
var connection = ParseClientId(clientId);
var pipeline = sendTarget.NetworkPipeline;
public NetworkDriver.Concurrent Driver;
public SendTarget Target;
public BatchedSendQueue Queue;
public NetworkPipeline ReliablePipeline;

while (!queue.IsEmpty)
public void Execute()
{
var result = m_Driver.BeginSend(pipeline, connection, out var writer);
if (result != (int)Networking.Transport.Error.StatusCode.Success)
var clientId = Target.ClientId;
var connection = ParseClientId(clientId);
var pipeline = Target.NetworkPipeline;

while (!Queue.IsEmpty)
{
Debug.LogError("Error sending the message: " +
ErrorUtilities.ErrorToString((Networking.Transport.Error.StatusCode)result, clientId));
return;
}
var result = Driver.BeginSend(pipeline, connection, out var writer);
if (result != (int)Networking.Transport.Error.StatusCode.Success)
{
Debug.LogError($"Error sending message: {ErrorUtilities.ErrorToFixedString(result, clientId)}");
return;
}

// We don't attempt to send entire payloads over the reliable pipeline. Instead we
// fragment it manually. This is safe and easy to do since the reliable pipeline
// basically implements a stream, so as long as we separate the different messages
// in the stream (the send queue does that automatically) we are sure they'll be
// reassembled properly at the other end. This allows us to lift the limit of ~44KB
// on reliable payloads (because of the reliable window size).
var written = pipeline == m_ReliableSequencedPipeline ? queue.FillWriterWithBytes(ref writer) : queue.FillWriterWithMessages(ref writer);
// We don't attempt to send entire payloads over the reliable pipeline. Instead we
// fragment it manually. This is safe and easy to do since the reliable pipeline
// basically implements a stream, so as long as we separate the different messages
// in the stream (the send queue does that automatically) we are sure they'll be
// reassembled properly at the other end. This allows us to lift the limit of ~44KB
// on reliable payloads (because of the reliable window size).
var written = pipeline == ReliablePipeline ? Queue.FillWriterWithBytes(ref writer) : Queue.FillWriterWithMessages(ref writer);

result = m_Driver.EndSend(writer);
if (result == written)
{
// Batched message was sent successfully. Remove it from the queue.
queue.Consume(written);
}
else
{
// Some error occured. If it's just the UTP queue being full, then don't log
// anything since that's okay (the unsent message(s) are still in the queue
// and we'll retry sending the later). Otherwise log the error and remove the
// message from the queue (we don't want to resend it again since we'll likely
// just get the same error again).
if (result != (int)Networking.Transport.Error.StatusCode.NetworkSendQueueFull)
result = Driver.EndSend(writer);
if (result == written)
{
Debug.LogError("Error sending the message: " + ErrorUtilities.ErrorToString((Networking.Transport.Error.StatusCode)result, clientId));
queue.Consume(written);
// Batched message was sent successfully. Remove it from the queue.
Queue.Consume(written);
}
else
{
// Some error occured. If it's just the UTP queue being full, then don't log
// anything since that's okay (the unsent message(s) are still in the queue
// and we'll retry sending them later). Otherwise log the error and remove the
// message from the queue (we don't want to resend it again since we'll likely
// just get the same error again).
if (result != (int)Networking.Transport.Error.StatusCode.NetworkSendQueueFull)
{
Debug.LogError($"Error sending the message: {ErrorUtilities.ErrorToFixedString(result, clientId)}");
Queue.Consume(written);
}

return;
return;
}
}
}
}

// Send as many batched messages from the queue as possible.
private void SendBatchedMessages(SendTarget sendTarget, BatchedSendQueue queue)
{
new SendBatchedMessagesJob
{
Driver = m_Driver.ToConcurrent(),
Target = sendTarget,
Queue = queue,
ReliablePipeline = m_ReliableSequencedPipeline
}.Run();
}

private bool AcceptConnection()
{
var connection = m_Driver.Accept();
Expand Down