intial commit

Library/PackageCache/com.unity.render-pipelines.core@1c27dfa04d8a/Runtime/GPUDriven/Utilities/MemoryUtilities.cs

@@ -0,0 +1,21 @@
+using Unity.Collections;
+using Unity.Collections.LowLevel.Unsafe;
+
+namespace UnityEngine.Rendering
+{
+    internal static class MemoryUtilities
+    {
+        public unsafe static T* Malloc<T>(int count, Allocator allocator) where T : unmanaged
+        {
+            return (T*)UnsafeUtility.Malloc(
+                UnsafeUtility.SizeOf<T>() * count,
+                UnsafeUtility.AlignOf<T>(),
+                allocator);
+        }
+
+        public unsafe static void Free<T>(T* p, Allocator allocator) where T : unmanaged
+        {
+            UnsafeUtility.Free(p, allocator);
+        }
+    }
+}

Library/PackageCache/com.unity.render-pipelines.core@1c27dfa04d8a/Runtime/GPUDriven/Utilities/MemoryUtilities.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: ce2eb19ed182ce44292cdf33f8191ec8
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Library/PackageCache/com.unity.render-pipelines.core@1c27dfa04d8a/Runtime/GPUDriven/Utilities/ParallelBitArray.cs

@@ -0,0 +1,174 @@
+using System;
+using System.Diagnostics;
+using System.Threading;
+using Unity.Collections;
+using Unity.Collections.LowLevel.Unsafe;
+using Unity.Jobs;
+
+namespace UnityEngine.Rendering
+{
+    internal struct ParallelBitArray
+    {
+        private Allocator m_Allocator;
+        private NativeArray<long> m_Bits;
+        private int m_Length;
+
+        public int Length => m_Length;
+
+        public bool IsCreated
+        {
+            get { return m_Bits.IsCreated; }
+        }
+
+        public ParallelBitArray(int length, Allocator allocator, NativeArrayOptions options = NativeArrayOptions.ClearMemory)
+        {
+            m_Allocator = allocator;
+            m_Bits = new NativeArray<long>((length + 63) / 64, allocator, options);
+            m_Length = length;
+        }
+
+        public void Dispose()
+        {
+            m_Bits.Dispose();
+            m_Length = 0;
+        }
+
+        public void Dispose(JobHandle inputDeps)
+        {
+            m_Bits.Dispose(inputDeps);
+            m_Length = 0;
+        }
+
+        public void Resize(int newLength)
+        {
+            int oldLength = m_Length;
+            if (newLength == oldLength)
+                return;
+
+            int oldBitsLength = m_Bits.Length;
+            int newBitsLength = (newLength + 63) / 64;
+            if (newBitsLength != oldBitsLength)
+            {
+                var newBits = new NativeArray<long>(newBitsLength, m_Allocator, NativeArrayOptions.UninitializedMemory);
+                if (m_Bits.IsCreated)
+                {
+                    NativeArray<long>.Copy(m_Bits, newBits, m_Bits.Length);
+                    m_Bits.Dispose();
+                }
+                m_Bits = newBits;
+            }
+
+            // mask off bits past the length
+            int validLength = Math.Min(oldLength, newLength);
+            int validBitsLength = Math.Min(oldBitsLength, newBitsLength);
+            for (int chunkIndex = validBitsLength; chunkIndex < m_Bits.Length; ++chunkIndex)
+            {
+                int validBitCount = Math.Max(validLength - 64 * chunkIndex, 0);
+                if (validBitCount < 64)
+                {
+                    ulong validMask = (1ul << validBitCount) - 1;
+                    m_Bits[chunkIndex] &= (long)validMask;
+                }
+            }
+            m_Length = newLength;
+        }
+
+        public void Set(int index, bool value)
+        {
+            unsafe
+            {
+                Debug.Assert(0 <= index && index < m_Length);
+
+                int entry_index = index >> 6;
+                long* entries = (long*)m_Bits.GetUnsafePtr();
+
+                ulong bit = 1ul << (index & 0x3f);
+                long and_mask = (long)(~bit);
+                long or_mask = value ? (long)bit : 0;
+
+                long old_entry, new_entry;
+                do
+                {
+                    old_entry = Interlocked.Read(ref entries[entry_index]);
+                    new_entry = (old_entry & and_mask) | or_mask;
+                } while (Interlocked.CompareExchange(ref entries[entry_index], new_entry, old_entry) != old_entry);
+            }
+        }
+
+        public bool Get(int index)
+        {
+            unsafe
+            {
+                Debug.Assert(0 <= index && index < m_Length);
+
+                int entry_index = index >> 6;
+                long* entries = (long*)m_Bits.GetUnsafeReadOnlyPtr();
+
+                ulong bit = 1ul << (index & 0x3f);
+                long check_mask = (long)bit;
+                return (entries[entry_index] & check_mask) != 0;
+            }
+        }
+
+        public ulong GetChunk(int chunk_index)
+        {
+            return (ulong)m_Bits[chunk_index];
+        }
+
+        public void SetChunk(int chunk_index, ulong chunk_bits)
+        {
+            m_Bits[chunk_index] = (long)chunk_bits;
+        }
+
+        public unsafe ulong InterlockedReadChunk(int chunk_index)
+        {
+            long* entries = (long*)m_Bits.GetUnsafeReadOnlyPtr();
+            return (ulong)Interlocked.Read(ref entries[chunk_index]);
+        }
+
+        public unsafe void InterlockedOrChunk(int chunk_index, ulong chunk_bits)
+        {
+            long* entries = (long*)m_Bits.GetUnsafePtr();
+
+            long old_entry, new_entry;
+            do
+            {
+                old_entry = Interlocked.Read(ref entries[chunk_index]);
+                new_entry = old_entry | (long)chunk_bits;
+            } while (Interlocked.CompareExchange(ref entries[chunk_index], new_entry, old_entry) != old_entry);
+        }
+
+        public int ChunkCount()
+        {
+            return m_Bits.Length;
+        }
+
+        public ParallelBitArray GetSubArray(int length)
+        {
+            ParallelBitArray array = new ParallelBitArray();
+            array.m_Bits = m_Bits.GetSubArray(0, (length + 63) / 64);
+            array.m_Length = length;
+            return array;
+        }
+
+        public NativeArray<long> GetBitsArray()
+        {
+            return m_Bits;
+        }
+
+        public void FillZeroes(int length)
+        {
+            length = Math.Min(length, m_Length);
+            int chunkIndex = length / 64;
+            int remainder = length & 63;
+
+            m_Bits.FillArray(0, 0, chunkIndex);
+
+            if(remainder > 0)
+            {
+                long lastChunkMask = (1L << remainder) - 1;
+                m_Bits[chunkIndex] &= ~lastChunkMask;
+            }
+        }
+    }
+}

Library/PackageCache/com.unity.render-pipelines.core@1c27dfa04d8a/Runtime/GPUDriven/Utilities/ParallelBitArray.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: c0bd11b715efbd24e95cb49f31c3cbeb
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Library/PackageCache/com.unity.render-pipelines.core@1c27dfa04d8a/Runtime/GPUDriven/Utilities/ParallelSortExtensions.cs

@@ -0,0 +1,269 @@
+using System.Threading;
+using UnityEngine.Assertions;
+using Unity.Mathematics;
+using Unity.Collections;
+using Unity.Jobs;
+using Unity.Jobs.LowLevel.Unsafe;
+using Unity.Collections.LowLevel.Unsafe;
+using Unity.Burst;
+using UnityEngine.Profiling;
+
+namespace UnityEngine.Rendering
+{
+    internal static class ParallelSortExtensions
+    {
+        const int kMinRadixSortArraySize = 2048;
+        const int kMinRadixSortBatchSize = 256;
+
+        internal static JobHandle ParallelSort(this NativeArray<int> array)
+        {
+            if (array.Length <= 1)
+                return new JobHandle();
+
+            var jobHandle = new JobHandle();
+
+            if (array.Length >= kMinRadixSortArraySize)
+            {
+                int workersCount = Mathf.Max(JobsUtility.JobWorkerCount + 1, 1);
+                int batchSize = Mathf.Max(kMinRadixSortBatchSize, Mathf.CeilToInt((float)array.Length / workersCount));
+                int jobsCount = Mathf.CeilToInt((float)array.Length / batchSize);
+
+                Assert.IsTrue(jobsCount * batchSize >= array.Length);
+
+                var supportArray = new NativeArray<int>(array.Length, Allocator.TempJob, NativeArrayOptions.UninitializedMemory);
+                var counter = new NativeArray<int>(1, Allocator.TempJob);
+                var buckets = new NativeArray<int>(jobsCount * 256, Allocator.TempJob);
+                var indices = new NativeArray<int>(jobsCount * 256, Allocator.TempJob, NativeArrayOptions.UninitializedMemory);
+                var indicesSum = new NativeArray<int>(16, Allocator.TempJob, NativeArrayOptions.UninitializedMemory);
+
+                var arraySource = array;
+                var arrayDest = supportArray;
+
+                for (int radix = 0; radix < 4; ++radix)
+                {
+                    var bucketCountJobData = new RadixSortBucketCountJob
+                    {
+                        radix = radix,
+                        jobsCount = jobsCount,
+                        batchSize = batchSize,
+                        buckets = buckets,
+                        array = arraySource
+                    };
+
+                    var batchPrefixSumJobData = new RadixSortBatchPrefixSumJob
+                    {
+                        radix = radix,
+                        jobsCount = jobsCount,
+                        array = arraySource,
+                        counter = counter,
+                        buckets = buckets,
+                        indices = indices,
+                        indicesSum = indicesSum
+                    };
+
+                    var prefixSumJobData = new RadixSortPrefixSumJob
+                    {
+                        jobsCount = jobsCount,
+                        indices = indices,
+                        indicesSum = indicesSum
+                    };
+
+                    var bucketSortJobData = new RadixSortBucketSortJob
+                    {
+                        radix = radix,
+                        batchSize = batchSize,
+                        indices = indices,
+                        array = arraySource,
+                        arraySorted = arrayDest
+                    };
+
+                    jobHandle = bucketCountJobData.ScheduleParallel(jobsCount, 1, jobHandle);
+                    jobHandle = batchPrefixSumJobData.ScheduleParallel(16, 1, jobHandle);
+                    jobHandle = prefixSumJobData.ScheduleParallel(16, 1, jobHandle);
+                    jobHandle = bucketSortJobData.ScheduleParallel(jobsCount, 1, jobHandle);
+
+                    JobHandle.ScheduleBatchedJobs();
+
+                    static void Swap(ref NativeArray<int> a, ref NativeArray<int> b)
+                    {
+                        NativeArray<int> temp = a;
+                        a = b;
+                        b = temp;
+                    }
+
+                    Swap(ref arraySource, ref arrayDest);
+                }
+
+                supportArray.Dispose(jobHandle);
+                counter.Dispose(jobHandle);
+                buckets.Dispose(jobHandle);
+                indices.Dispose(jobHandle);
+                indicesSum.Dispose(jobHandle);
+            }
+            else
+            {
+                jobHandle = NativeSortExtension.SortJob(array).Schedule();
+            }
+
+            return jobHandle;
+        }
+
+        [BurstCompile(DisableSafetyChecks = true, OptimizeFor = OptimizeFor.Performance)]
+        internal struct RadixSortBucketCountJob : IJobFor
+        {
+            [ReadOnly] public int radix;
+            [ReadOnly] public int jobsCount;
+            [ReadOnly] public int batchSize;
+            [ReadOnly] [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> array;
+
+            [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> buckets;
+
+            public void Execute(int index)
+            {
+                int start = index * batchSize;
+                int end = math.min(start + batchSize, array.Length);
+
+                int jobBuckets = index * 256;
+
+                for (int i = start; i < end; ++i)
+                {
+                    int value = array[i];
+                    int bucket = (value >> radix * 8) & 0xFF;
+                    buckets[jobBuckets + bucket] += 1;
+                }
+            }
+        }
+
+        [BurstCompile(DisableSafetyChecks = true, OptimizeFor = OptimizeFor.Performance)]
+        internal struct RadixSortBatchPrefixSumJob : IJobFor
+        {
+            [ReadOnly] public int radix;
+            [ReadOnly] public int jobsCount;
+            [ReadOnly] [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> array;
+
+            [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> counter;
+            [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> indicesSum;
+            [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> buckets;
+            [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> indices;
+
+            private unsafe static int AtomicIncrement(NativeArray<int> counter)
+            {
+                return Interlocked.Increment(ref UnsafeUtility.AsRef<int>((int*)counter.GetUnsafePtr()));
+            }
+
+            private int JobIndexPrefixSum(int sum, int i)
+            {
+                for (int j = 0; j < jobsCount; ++j)
+                {
+                    int k = i + j * 256;
+
+                    indices[k] = sum;
+                    sum += buckets[k];
+                    buckets[k] = 0;
+                }
+
+                return sum;
+            }
+
+            public void Execute(int index)
+            {
+                int start = index * 16;
+                int end = start + 16;
+
+                int jobSum = 0;
+
+                for (int i = start; i < end; ++i)
+                    jobSum = JobIndexPrefixSum(jobSum, i);
+
+                indicesSum[index] = jobSum;
+
+                if (AtomicIncrement(counter) == 16)
+                {
+                    int sum = 0;
+
+                    if(radix < 3)
+                    {
+                        for (int i = 0; i < 16; ++i)
+                        {
+                            int indexSum = indicesSum[i];
+                            indicesSum[i] = sum;
+                            sum += indexSum;
+                        }
+                    }
+                    else // Negative
+                    {
+                        for (int i = 8; i < 16; ++i)
+                        {
+                            int indexSum = indicesSum[i];
+                            indicesSum[i] = sum;
+                            sum += indexSum;
+                        }
+                        for (int i = 0; i < 8; ++i)
+                        {
+                            int indexSum = indicesSum[i];
+                            indicesSum[i] = sum;
+                            sum += indexSum;
+                        }
+                    }
+
+                    Assert.AreEqual(sum, array.Length);
+
+                    counter[0] = 0;
+                }
+            }
+        }
+
+        [BurstCompile(DisableSafetyChecks = true, OptimizeFor = OptimizeFor.Performance)]
+        internal struct RadixSortPrefixSumJob : IJobFor
+        {
+            [ReadOnly] public int jobsCount;
+
+            [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> indicesSum;
+            [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> indices;
+
+            public void Execute(int index)
+            {
+                int start = index * 16;
+                int end = start + 16;
+
+                int jobSum = indicesSum[index];
+
+                for (int j = 0; j < jobsCount; ++j)
+                {
+                    for (int i = start; i < end; ++i)
+                    {
+                        int k = j * 256 + i;
+                        indices[k] += jobSum;
+                    }
+                }
+            }
+        }
+
+        [BurstCompile(DisableSafetyChecks = true, OptimizeFor = OptimizeFor.Performance)]
+        internal struct RadixSortBucketSortJob : IJobFor
+        {
+            [ReadOnly] public int radix;
+            [ReadOnly] public int batchSize;
+            [ReadOnly] [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> array;
+
+            [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> indices;
+            [NativeDisableContainerSafetyRestriction, NoAlias] public NativeArray<int> arraySorted;
+
+            public void Execute(int index)
+            {
+                int start = index * batchSize;
+                int end = math.min(start + batchSize, array.Length);
+
+                int jobIndices = index * 256;
+
+                for (int i = start; i < end; ++i)
+                {
+                    int value = array[i];
+                    int bucket = (value >> radix * 8) & 0xFF;
+                    int sortedIndex = indices[jobIndices + bucket]++;
+                    arraySorted[sortedIndex] = value;
+                }
+            }
+        }
+    }
+}

Library/PackageCache/com.unity.render-pipelines.core@1c27dfa04d8a/Runtime/GPUDriven/Utilities/ParallelSortExtensions.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 08852b5d707b78442bda3ca091091ce8
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: