using System; using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; namespace Robust.Shared.Maths { public static unsafe partial class NumericsHelpers { #region Multiply /// /// Multiplies a by b and stores the result in a. /// [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] public static void Multiply(Span a, ReadOnlySpan b) { Multiply(a, b, a); } /// /// Multiplies a by b and stores the result in s. /// [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] public static void Multiply(ReadOnlySpan a, ReadOnlySpan b, Span s) { if (a.Length != b.Length || a.Length != s.Length) throw new ArgumentException("Length of arrays must be the same!"); if (Enabled) { if (AvxEnabled && LengthValid256Single(a.Length)) { MultiplyAvx(a, b, s); return; } if (LengthValid128Single(a.Length)) { if (Sse.IsSupported) { MultiplySse(a, b, s); return; } if (AdvSimd.IsSupported) { MultiplyAdvSimd(a, b, s); return; } } } MultiplyNaive(a, b, s, 0, a.Length); } [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] private static void MultiplyNaive(ReadOnlySpan a, ReadOnlySpan b, Span s, int start, int end) { for (var i = start; i < end; i++) { s[i] = a[i] * b[i]; } } [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] private static void MultiplySse(ReadOnlySpan a, ReadOnlySpan b, Span s) { var remainder = a.Length & 3; var length = a.Length - remainder; fixed (float* ptr = a) { fixed (float* ptrB = b) { fixed (float* ptrS = s) { for (var i = 0; i < length; i += 4) { var j = Sse.LoadVector128(ptr + i); var k = Sse.LoadVector128(ptrB + i); Sse.Store(ptrS + i, Sse.Multiply(j, k)); } } } } if(remainder != 0) { MultiplyNaive(a, b, s, length, a.Length); } } [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] private static void MultiplyAdvSimd(ReadOnlySpan a, ReadOnlySpan b, Span s) { var remainder = a.Length & 3; var length = a.Length - remainder; fixed (float* ptr = a) { fixed (float* ptrB = b) { fixed (float* ptrS = s) { for (var i = 0; i < length; i += 4) { var j = AdvSimd.LoadVector128(ptr + i); var k = AdvSimd.LoadVector128(ptrB + i); AdvSimd.Store(ptrS + i, AdvSimd.Multiply(j, k)); } } } } if(remainder != 0) { MultiplyNaive(a, b, s, length, a.Length); } } [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] private static void MultiplyAvx(ReadOnlySpan a, ReadOnlySpan b, Span s) { var remainder = a.Length & 7; var length = a.Length - remainder; fixed (float* ptr = a) { fixed (float* ptrB = b) { fixed (float* ptrS = s) { for (var i = 0; i < length; i += 8) { var j = Avx.LoadVector256(ptr + i); var k = Avx.LoadVector256(ptrB + i); Avx.Store(ptrS + i, Avx.Multiply(j, k)); } } } } if(remainder != 0) { MultiplyNaive(a, b, s, length, a.Length); } } #endregion #region MultiplyByScalar /// /// Multiply a by scalar b and stores the result in a. /// [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] public static void Multiply(Span a, float b) { Multiply(a, b, a); } /// /// Multiply a by scalar b and stores the result in s. /// [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] public static void Multiply(ReadOnlySpan a, float b, Span s) { if (a.Length != s.Length) throw new ArgumentException("Length of arrays must be the same!"); if (Enabled) { if (AvxEnabled && LengthValid256Single(a.Length)) { MultiplyAvx(a, b, s); return; } if (LengthValid128Single(a.Length)) { if (Sse.IsSupported) { MultiplySse(a, b, s); return; } if (AdvSimd.IsSupported) { MultiplyAdvSimd(a, b, s); return; } } } MultiplyNaive(a, b, s, 0, a.Length); } [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] private static void MultiplyNaive(ReadOnlySpan a, float b, Span s, int start, int end) { for (var i = start; i < end; i++) { s[i] = a[i] * b; } } [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] private static void MultiplySse(ReadOnlySpan a, float b, Span s) { var remainder = a.Length & 3; var length = a.Length - remainder; var scalar = Vector128.Create(b); fixed (float* ptr = a) { fixed (float* ptrS = s) { for (var i = 0; i < length; i += 4) { var j = Sse.LoadVector128(ptr + i); Sse.Store(ptrS + i, Sse.Multiply(j, scalar)); } } } if(remainder != 0) { MultiplyNaive(a, b, s, length, a.Length); } } [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] private static void MultiplyAdvSimd(ReadOnlySpan a, float b, Span s) { var remainder = a.Length & 3; var length = a.Length - remainder; var scalar = Vector128.Create(b); fixed (float* ptr = a) { fixed (float* ptrS = s) { for (var i = 0; i < length; i += 4) { var j = AdvSimd.LoadVector128(ptr + i); AdvSimd.Store(ptrS + i, AdvSimd.Multiply(j, scalar)); } } } if(remainder != 0) { MultiplyNaive(a, b, s, length, a.Length); } } [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] private static void MultiplyAvx(ReadOnlySpan a, float b, Span s) { var remainder = a.Length & 7; var length = a.Length - remainder; var scalar = Vector256.Create(b); fixed (float* ptr = a) { fixed (float* ptrS = s) { for (var i = 0; i < length; i += 8) { var j = Avx.LoadVector256(ptr + i); Avx.Store(ptrS + i, Avx.Multiply(j, scalar)); } } } if(remainder != 0) { MultiplyNaive(a, b, s, length, a.Length); } } #endregion } }