fix #875 - string encode was failing to encode correctly (also make more...

fix #875 - string encode was failing to encode correctly (also make more consistent use of GetReference)

StackExchange.Redis/StackExchange/Redis/Format.cs

@@ -175,12 +175,12 @@ internal static bool TryParseDouble(ReadOnlySpan<byte> s, out double value)
             var ss = DecodeUtf8(s);
             return double.TryParse(ss, NumberStyles.Any, NumberFormatInfo.InvariantInfo, out value);
         }
-        internal static unsafe string DecodeUtf8(ReadOnlySpan<byte> s)
+        internal static unsafe string DecodeUtf8(ReadOnlySpan<byte> span)
         {
-            if (s.IsEmpty) return "";
-            fixed(byte* ptr = &MemoryMarshal.GetReference(s))
+            if (span.IsEmpty) return "";
+            fixed(byte* ptr = &MemoryMarshal.GetReference(span))
             {
-                return Encoding.UTF8.GetString(ptr, s.Length);
+                return Encoding.UTF8.GetString(ptr, span.Length);
             }
         }
         private static bool CaseInsensitiveASCIIEqual(string xLowerCase, ReadOnlySpan<byte> y)

StackExchange.Redis/StackExchange/Redis/PhysicalConnection.cs

@@ -10,6 +10,7 @@
 using System.Net.Security;
 using System.Net.Sockets;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 using System.Security.Authentication;
 using System.Security.Cryptography.X509Certificates;
 using System.Text;
@@ -749,22 +750,22 @@ private void WriteUnified(PipeWriter writer, byte[] prefix, string value)
             }
         }
 
-        private unsafe void WriteRaw(PipeWriter writer, string value, int encodedLength)
+        private unsafe void WriteRaw(PipeWriter writer, string value, int expectedLength)
         {
             const int MaxQuickEncodeSize = 512;
 
             fixed (char* cPtr = value)
             {
                 int totalBytes;
-                if (encodedLength <= MaxQuickEncodeSize)
+                if (expectedLength <= MaxQuickEncodeSize)
                 {
                     // encode directly in one hit
-                    var span = writer.GetSpan(encodedLength);
-                    fixed (byte* bPtr = &span[0])
+                    var span = writer.GetSpan(expectedLength);
+                    fixed (byte* bPtr = &MemoryMarshal.GetReference(span))
                     {
-                        totalBytes = Encoding.UTF8.GetBytes(cPtr, value.Length, bPtr, encodedLength);
+                        totalBytes = Encoding.UTF8.GetBytes(cPtr, value.Length, bPtr, expectedLength);
                     }
-                    writer.Advance(encodedLength);
+                    writer.Advance(expectedLength);
                 }
                 else
                 {
@@ -772,22 +773,33 @@ private unsafe void WriteRaw(PipeWriter writer, string value, int encodedLength)
                     outEncoder.Reset();
                     int charsRemaining = value.Length, charOffset = 0;
                     totalBytes = 0;
-                    while (charsRemaining != 0)
+
+                    bool final = false;
+                    while (true)
                     {
-                        // note: at most 4 bytes per UTF8 character, despite what UTF8.GetMaxByteCount says
-                        var span = writer.GetSpan(4); // get *some* memory - at least enough for 1 character (but hopefully lots more)
-                        int bytesWritten, charsToWrite = span.Length >> 2; // assume worst case, because the API sucks
-                        fixed (byte* bPtr = &span[0])
+                        var span = writer.GetSpan(5); // get *some* memory - at least enough for 1 character (but hopefully lots more)
+
+                        int charsUsed, bytesUsed;
+                        bool completed;
+                        fixed (byte* bPtr = &MemoryMarshal.GetReference(span))
+                        {
+                            outEncoder.Convert(cPtr + charOffset, span.Length, bPtr, span.Length, final, out charsUsed, out bytesUsed, out completed);
+                        }
+                        writer.Advance(bytesUsed);
+                        totalBytes += bytesUsed;
+                        charOffset += charsUsed;
+                        charsRemaining -= charsUsed;
+
+                        if(charsRemaining <= 0)
                         {
-                            bytesWritten = outEncoder.GetBytes(cPtr + charOffset, charsToWrite, bPtr, span.Length, false);
+                            if (charsRemaining < 0) throw new InvalidOperationException("String encode went negative");
+                            if (completed) break; // fine
+                            if (final) throw new InvalidOperationException("String encode failed to complete");
+                            final = true; // flush the encoder to one more span, then exit
                         }
-                        writer.Advance(bytesWritten);
-                        totalBytes += bytesWritten;
-                        charOffset += charsToWrite;
-                        charsRemaining -= charsRemaining;
                     }
                 }
-                Debug.Assert(totalBytes == encodedLength);
+                if (totalBytes != expectedLength) throw new InvalidOperationException("String encode length check failure");
             }
         }
 

StackExchange.Redis/StackExchange/Redis/RawResult.cs

 using System;
 using System.Buffers;
+using System.Runtime.InteropServices;
 using System.Text;
 
 namespace StackExchange.Redis
@@ -322,7 +323,7 @@ internal unsafe string GetString()
             if (_payload.IsSingleSegment)
             {
                 var span = _payload.First.Span;
-                fixed (byte* ptr = &span[0])
+                fixed (byte* ptr = &MemoryMarshal.GetReference(span))
                 {
                     return Encoding.UTF8.GetString(ptr, span.Length);
                 }
@@ -334,7 +335,7 @@ internal unsafe string GetString()
                 var span = segment.Span;
                 if (span.IsEmpty) continue;
 
-                fixed(byte* bPtr = &span[0])
+                fixed(byte* bPtr = &MemoryMarshal.GetReference(span))
                 {
                     charCount += decoder.GetCharCount(bPtr, span.Length, false);
                 }
@@ -351,7 +352,7 @@ internal unsafe string GetString()
                     var span = segment.Span;
                     if (span.IsEmpty) continue;
 
-                    fixed (byte* bPtr = &span[0])
+                    fixed (byte* bPtr = &MemoryMarshal.GetReference(span))
                     {
                         var written = decoder.GetChars(bPtr, span.Length, cPtr, charCount, false);
                         cPtr += written;