using System;
using System.Runtime.CompilerServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;
namespace Robust.Shared.Maths
{
public static unsafe partial class NumericsHelpers
{
#region Add
///
/// Adds b to a and stores the result in a.
///
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
public static void Add(Span a, ReadOnlySpan b)
{
Add(a, b, a);
}
///
/// Adds b to a and stores the result in s.
///
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
public static void Add(ReadOnlySpan a, ReadOnlySpan b, Span s)
{
if (a.Length != b.Length || a.Length != s.Length)
throw new ArgumentException("Length of arrays must be the same!");
if (Enabled)
{
if (AvxEnabled && LengthValid256Single(a.Length))
{
AddAvx(a, b, s);
return;
}
if (LengthValid128Single(a.Length))
{
if (Sse.IsSupported)
{
AddSse(a, b, s);
return;
}
if (AdvSimd.IsSupported)
{
AddAdvSimd(a, b, s);
return;
}
}
}
AddNaive(a, b, s, 0, a.Length);
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static void AddNaive(ReadOnlySpan a, ReadOnlySpan b, Span s, int start, int end)
{
for (var i = start; i < end; i++)
{
s[i] = a[i] + b[i];
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static void AddSse(ReadOnlySpan a, ReadOnlySpan b, Span s)
{
var remainder = a.Length & 3;
var length = a.Length - remainder;
fixed (float* ptr = a)
{
fixed (float* ptrB = b)
{
fixed (float* ptrS = s)
{
for (var i = 0; i < length; i += 4)
{
var j = Sse.LoadVector128(ptr + i);
var k = Sse.LoadVector128(ptrB + i);
Sse.Store(ptrS + i, Sse.Add(j, k));
}
}
}
}
if(remainder != 0)
{
AddNaive(a, b, s, length, a.Length);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static void AddAdvSimd(ReadOnlySpan a, ReadOnlySpan b, Span s)
{
var remainder = a.Length & 3;
var length = a.Length - remainder;
fixed (float* ptr = a)
{
fixed (float* ptrB = b)
{
fixed (float* ptrS = s)
{
for (var i = 0; i < length; i += 4)
{
var j = AdvSimd.LoadVector128(ptr + i);
var k = AdvSimd.LoadVector128(ptrB + i);
AdvSimd.Store(ptrS + i, AdvSimd.Add(j, k));
}
}
}
}
if(remainder != 0)
{
AddNaive(a, b, s, length, a.Length);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static void AddAvx(ReadOnlySpan a, ReadOnlySpan b, Span s)
{
var remainder = a.Length & 7;
var length = a.Length - remainder;
fixed (float* ptr = a)
{
fixed (float* ptrB = b)
{
fixed (float* ptrS = s)
{
for (var i = 0; i < length; i += 8)
{
var j = Avx.LoadVector256(ptr + i);
var k = Avx.LoadVector256(ptrB + i);
Avx.Store(ptrS + i, Avx.Add(j, k));
}
}
}
}
if(remainder != 0)
{
AddNaive(a, b, s, length, a.Length);
}
}
#endregion
#region AddByScalar
///
/// Adds scalar b to a and stores the result in a.
///
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
public static void Add(Span a, float b)
{
Add(a, b, a);
}
///
/// Adds scalar b to a and stores the result in s.
///
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
public static void Add(ReadOnlySpan a, float b, Span s)
{
if (a.Length != s.Length)
throw new ArgumentException("Length of arrays must be the same!");
if (Enabled)
{
if (AvxEnabled && LengthValid256Single(a.Length))
{
AddAvx(a, b, s);
return;
}
if (LengthValid128Single(a.Length))
{
if (Sse.IsSupported)
{
AddSse(a, b, s);
return;
}
if (AdvSimd.IsSupported)
{
AddAdvSimd(a, b, s);
return;
}
}
}
AddNaive(a, b, s, 0, a.Length);
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static void AddNaive(ReadOnlySpan a, float b, Span s, int start, int end)
{
for (var i = start; i < end; i++)
{
s[i] = a[i] + b;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static void AddSse(ReadOnlySpan a, float b, Span s)
{
var remainder = a.Length & 3;
var length = a.Length - remainder;
var scalar = Vector128.Create(b);
fixed (float* ptr = a)
{
fixed (float* ptrS = s)
{
for (var i = 0; i < length; i += 4)
{
var j = Sse.LoadVector128(ptr + i);
Sse.Store(ptrS + i, Sse.Add(j, scalar));
}
}
}
if(remainder != 0)
{
AddNaive(a, b, s, length, a.Length);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static void AddAdvSimd(ReadOnlySpan a, float b, Span s)
{
var remainder = a.Length & 3;
var length = a.Length - remainder;
var scalar = Vector128.Create(b);
fixed (float* ptr = a)
{
fixed (float* ptrS = s)
{
for (var i = 0; i < length; i += 4)
{
var j = AdvSimd.LoadVector128(ptr + i);
AdvSimd.Store(ptrS + i, AdvSimd.Add(j, scalar));
}
}
}
if(remainder != 0)
{
AddNaive(a, b, s, length, a.Length);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static void AddAvx(ReadOnlySpan a, float b, Span s)
{
var remainder = a.Length & 7;
var length = a.Length - remainder;
var scalar = Vector256.Create(b);
fixed (float* ptr = a)
{
fixed (float* ptrS = s)
{
for (var i = 0; i < length; i += 8)
{
var j = Avx.LoadVector256(ptr + i);
Avx.Store(ptrS + i, Avx.Add(j, scalar));
}
}
}
if(remainder != 0)
{
AddNaive(a, b, s, length, a.Length);
}
}
#endregion
#region HorizontalAdd
///
/// Adds all elements of a and returns the value.
///
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
public static float HorizontalAdd(ReadOnlySpan a)
{
if (Enabled)
{
if (AvxEnabled && Sse3.IsSupported && LengthValid256Single(a.Length))
{
return HorizontalAddAvx(a);
}
if (LengthValid128Single(a.Length))
{
if (Sse3.IsSupported)
{
return HorizontalAddSse(a);
}
if (AdvSimd.Arm64.IsSupported)
{
return HorizontalAddAdvSimd(a);
}
}
}
return HorizontalAddNaive(a, 0, a.Length);
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static float HorizontalAddNaive(ReadOnlySpan a, int start, int end)
{
var sum = 0f;
for (var i = start; i < end; i++)
{
sum += a[i];
}
return sum;
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static float HorizontalAddSse(ReadOnlySpan a)
{
var remainder = a.Length & 3;
var length = a.Length - remainder;
var accumulator = Vector128.Create(0f);
fixed (float* ptr = a)
{
for (var i = 0; i < length; i += 4)
{
var j = Sse.LoadVector128(ptr + i);
accumulator = Sse3.HorizontalAdd(accumulator, j);
}
}
var sum = 0f;
accumulator = Sse3.HorizontalAdd(Sse3.HorizontalAdd(accumulator, accumulator), accumulator);
Sse.StoreScalar(&sum, accumulator);
if(remainder != 0)
{
sum += HorizontalAddNaive(a, length, a.Length);
}
return sum;
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static float HorizontalAddAdvSimd(ReadOnlySpan a)
{
var remainder = a.Length & 3;
var length = a.Length - remainder;
var accumulator = Vector128.Create(0f);
fixed (float* ptr = a)
{
for (var i = 0; i < length; i += 4)
{
var j = AdvSimd.LoadVector128(ptr + i);
accumulator = AdvSimd.Arm64.AddPairwise(accumulator, j);
}
}
var sum = 0f;
accumulator = AdvSimd.Arm64.AddPairwise(AdvSimd.Arm64.AddPairwise(accumulator, accumulator), accumulator);
AdvSimd.StoreSelectedScalar(&sum, accumulator, 0);
if(remainder != 0)
{
sum += HorizontalAddNaive(a, length, a.Length);
}
return sum;
}
[MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)]
private static float HorizontalAddAvx(ReadOnlySpan a)
{
var remainder = a.Length & 7;
var length = a.Length - remainder;
var accumulator = Vector128.Create(0f);
fixed (float* ptr = a)
{
for (var i = 0; i < length; i += 8)
{
var j = Avx.LoadVector256(ptr + i);
var x128 = Sse.Add(Avx.ExtractVector128(j, 0), Avx.ExtractVector128(j, 1));
accumulator = Sse3.HorizontalAdd(x128, accumulator);
}
}
var sum = 0f;
accumulator = Sse3.HorizontalAdd(Sse3.HorizontalAdd(accumulator, accumulator), accumulator);
Sse.StoreScalar(&sum, accumulator);
if(remainder != 0)
{
sum += HorizontalAddNaive(a, length, a.Length);
}
return sum;
}
#endregion
}
}