LZ4 Encode Fast

AXiX-official · Jun 21, 2024 · 9227568 · 9227568
1 parent 713f2c2
commit 9227568
Show file tree

Hide file tree

Showing 2 changed files with 117 additions and 16 deletions.
diff --git a/UnityAsset.NET/Compression/Compression.cs b/UnityAsset.NET/Compression/Compression.cs
@@ -42,8 +42,8 @@ public static List<byte> CompressStream(MemoryStream uncompressedStream, string
             case "none":
                 return uncompressedData.ToList();
             case "lz4":
-                byte[] compressedData = new byte[LZ4Codec.MaximumOutputSize(uncompressedData.Length)];
-                int compressedSize = LZ4Codec.Encode(uncompressedData, compressedData);
+                byte[] compressedData = new byte[LZ4.MaximumOutputSize(uncompressedData.Length)];
+                int compressedSize = LZ4.EncodeFast(uncompressedData, compressedData);
                 return compressedData.Take(compressedSize).ToList();
             case "lz4hc":
                 byte[] compressedDataHC = new byte[LZ4Codec.MaximumOutputSize(uncompressedData.Length)];

diff --git a/UnityAsset.NET/Compression/LZ4.cs b/UnityAsset.NET/Compression/LZ4.cs
@@ -1,18 +1,20 @@
 namespace UnityAsset.NET;
 
-public static class LZ4
+public static unsafe class LZ4
 {
-    public static unsafe int Decode(ReadOnlySpan<byte> source, Span<byte> target)
+    // Only about half the speed of K4os.Compression.LZ4
+
+    public static int Decode(ReadOnlySpan<byte> source, Span<byte> target)
     {
-        int length1 = source.Length;
-        if (length1 <= 0)
+        int length = source.Length;
+        if (length < 5)
             return 0;
         fixed (byte* sourcePtr = &source.GetPinnableReference())
         fixed (byte* targetPtr = &target.GetPinnableReference())
         {
             byte* s = sourcePtr;
             byte* t = targetPtr;
-            byte* sourceEnd = sourcePtr + source.Length;
+            byte* sourceEnd = sourcePtr + length;
             while (s < sourceEnd)
             {
                 byte token = *s++;
@@ -27,13 +29,11 @@ public static unsafe int Decode(ReadOnlySpan<byte> source, Span<byte> target)
                         literalLength += b;
                     } while (b == 0xFF && s < sourceEnd);
                 }
-
                 Buffer.MemoryCopy(s, t, literalLength, literalLength);
                 s += literalLength;
                 t += literalLength;
-
-                if (s >= sourceEnd) break;
-
+                if (s == sourceEnd && matchLength == 0) break;
+                if (s >= sourceEnd) return -1;
                 int offset = *s++ | (*s++ << 8);
                 if (matchLength == 0xF)
                 {
@@ -44,25 +44,126 @@ public static unsafe int Decode(ReadOnlySpan<byte> source, Span<byte> target)
                         matchLength += b;
                     } while (b == 0xFF && s < sourceEnd);
                 }
-
                 matchLength += 4;
-
                 while (matchLength > offset)
                 {
                     Buffer.MemoryCopy(t - offset, t, offset, offset);
                     t += offset;
                     matchLength -= offset;
                 }
-
                 if (matchLength > 0)
                 {
                     Buffer.MemoryCopy(t - offset, t, matchLength, matchLength);
                 }
-
                 t += matchLength;
             }
-
             return (int)(t - targetPtr);
         }
     }
+
+    public static int EncodeFast(ReadOnlySpan<byte> source, Span<byte> target)
+    {
+        int sourceLength = source.Length - 5;
+        if (sourceLength < 0)
+        {
+            return 0;
+        }
+        fixed (byte* sourcePtr = &source.GetPinnableReference())
+        fixed (byte* targetPtr = &target.GetPinnableReference())
+        {
+            byte* anchor = sourcePtr;
+            IntPtr[] hashTable = new IntPtr[1 << 16];
+
+            byte* s = sourcePtr;
+            byte* t = targetPtr;
+            byte* sourceEnd = sourcePtr + sourceLength;
+            while (s < sourceEnd - 4)
+            {
+                uint data =  *(uint*) s;
+                ushort hash =(ushort) ((data * 2654435761) >> 16);
+                if (hashTable[hash] == 0)
+                {
+                    hashTable[hash] = (IntPtr)s;
+                    s++;
+                }
+                else
+                {
+                    byte* sourceOffset = (byte*)hashTable[hash];
+                    uint oldData =  *(uint*) sourceOffset;
+                    if (oldData != data)
+                    {
+                        hashTable[hash] = (IntPtr)s;
+                        s++;
+                    }
+                    else
+                    {
+                        hashTable[hash] = (IntPtr)s;
+                        int matchDec = (int) (s - sourceOffset);
+                        if (matchDec > 0xFFFF)
+                        {
+                            s++;
+                            continue;
+                        }
+                        int literalLen = (int) (s - anchor);
+                        int matchLen = 4;
+                        while (s + matchLen < sourceEnd && *(s + matchLen) == *(sourceOffset + matchLen))
+                        {
+                            matchLen++;
+                        }
+                        int token = Math.Min(literalLen, 15) << 4 | Math.Min(matchLen - 4, 15);
+                        *t++ = (byte)token;
+                        if (literalLen >= 15)
+                        {
+                            int l = literalLen - 15;
+                            while (l >= 0xFF)
+                            {
+                                *t++ = 0xFF;
+                                l -= 0xFF;
+                            }
+                            *t++ = (byte)l;
+                        }
+                        Buffer.MemoryCopy(anchor, t, literalLen, literalLen);
+                        t += literalLen;
+                        byte matchDecLow = (byte)matchDec;
+                        byte matchDecHigh = (byte)(matchDec >> 8);
+                        *t++ = matchDecLow;
+                        *t++ = matchDecHigh;
+                        if (matchLen >= 19)
+                        {
+                            int l = matchLen - 19;
+                            while (l >= 0xFF)
+                            {
+                                *t++ = 0xFF;
+                                l -= 0xFF;
+                            }
+                            *t++ = (byte)l;
+                        }
+                        s += matchLen;
+                        anchor = s;
+                    }
+                }
+            }
+            int literalLenFinal = (int) (sourceEnd - anchor + 5);
+            int tokenFinal = Math.Min(literalLenFinal, 15) << 4;
+            *t++ = (byte)tokenFinal;
+            if (literalLenFinal >= 15)
+            {
+                int l = literalLenFinal - 15;
+                while (l >= 255)
+                {
+                    *t++ = 255;
+                    l -= 255;
+                }
+                *t++ = (byte)l;
+            }
+            Buffer.MemoryCopy(anchor, t, literalLenFinal, literalLenFinal);
+            t += literalLenFinal;
+            return (int)(t - targetPtr);
+        }
+    }
+
+    public static int MaximumOutputSize(int inputSize)
+    {
+        return inputSize + inputSize / 255 + 16;
+    }
 }