Made GenericFont and TokenizedString support UTF-32 characters like emoji

CHANGELOG.md

@@ -23,6 +23,7 @@ Improvements
 - Multi-target net452, making MLEM compatible with MonoGame for consoles
 - Allow retrieving the cost of a calculated path when using AStar
 - **Drastically improved StaticSpriteBatch batching performance**
+- **Made GenericFont and TokenizedString support UTF-32 characters like emoji**
 
 Fixes
 - Fixed TokenizedString handling trailing spaces incorrectly in the last line of non-left aligned text
@@ -68,6 +69,7 @@ Fixes
 ## MLEM.Extended
 Improvements
 - Multi-target net452, making MLEM compatible with MonoGame for consoles
+- **Made GenericBitmapFont and GenericStashFont support UTF-32 characters like emoji**
 
 ## MLEM.Startup
 Improvements

MLEM.Data/Json/JsonTypeSafeGenericDataHolder.cs

@@ -1,5 +1,4 @@
-using System;
+using System.Collections.Generic;
-using System.Collections.Generic;
 using System.Runtime.Serialization;
 using MLEM.Misc;
 using Newtonsoft.Json;

MLEM.Extended/Font/GenericBitmapFont.cs

@@ -32,14 +32,14 @@ namespace MLEM.Extended.Font {
         }
 
         /// <inheritdoc />
-        protected override float MeasureChar(char c) {
+        protected override float MeasureCharacter(int codePoint) {
-            var region = this.Font.GetCharacterRegion(c);
+            var region = this.Font.GetCharacterRegion(codePoint);
             return region != null ? new Vector2(region.XAdvance, region.Height).X : 0;
         }
 
         /// <inheritdoc />
-        protected override void DrawChar(SpriteBatch batch, string cString, Vector2 position, Color color, float rotation, Vector2 scale, SpriteEffects effects, float layerDepth) {
+        protected override void DrawCharacter(SpriteBatch batch, int codePoint, string character, Vector2 position, Color color, float rotation, Vector2 scale, SpriteEffects effects, float layerDepth) {
-            batch.DrawString(this.Font, cString, position, color, rotation, Vector2.Zero, scale, effects, layerDepth);
+            batch.DrawString(this.Font, character, position, color, rotation, Vector2.Zero, scale, effects, layerDepth);
         }
 
     }

MLEM.Extended/Font/GenericStashFont.cs

@@ -1,7 +1,6 @@
 using FontStashSharp;
 using Microsoft.Xna.Framework;
 using Microsoft.Xna.Framework.Graphics;
-using MLEM.Extensions;
 using MLEM.Font;
 
 namespace MLEM.Extended.Font {
@@ -33,13 +32,13 @@ namespace MLEM.Extended.Font {
         }
 
         /// <inheritdoc />
-        protected override float MeasureChar(char c) {
+        protected override float MeasureCharacter(int codePoint) {
-            return this.Font.MeasureString(c.ToCachedString()).X;
+            return this.Font.MeasureString(char.ConvertFromUtf32(codePoint)).X;
         }
 
         /// <inheritdoc />
-        protected override void DrawChar(SpriteBatch batch, string cString, Vector2 position, Color color, float rotation, Vector2 scale, SpriteEffects effects, float layerDepth) {
+        protected override void DrawCharacter(SpriteBatch batch, int codePoint, string character, Vector2 position, Color color, float rotation, Vector2 scale, SpriteEffects effects, float layerDepth) {
-            this.Font.DrawText(batch, cString, position, color, scale, rotation, Vector2.Zero, layerDepth);
+            this.Font.DrawText(batch, character, position, color, scale, rotation, Vector2.Zero, layerDepth);
         }
 
     }

MLEM/Extensions/CharExtensions.cs

@@ -1,9 +1,11 @@
+using System;
 using System.Collections.Generic;
 
 namespace MLEM.Extensions {
     /// <summary>
     /// A set of extensions for dealing with <see cref="char"/>
     /// </summary>
+    [Obsolete("ToCachedString is deprecated. Consider using a more robust, custom implementation for text caching.")]
     public static class CharExtensions {
 
         private static readonly Dictionary<char, string> Cache = new Dictionary<char, string>();
@@ -14,6 +16,7 @@ namespace MLEM.Extensions {
         /// </summary>
         /// <param name="c">The character to turn into a string</param>
         /// <returns>A string representing the character</returns>
+        [Obsolete("ToCachedString is deprecated. Consider using a more robust, custom implementation for text caching.")]
         public static string ToCachedString(this char c) {
             if (!CharExtensions.Cache.TryGetValue(c, out var ret)) {
                 ret = c.ToString();

MLEM/Font/CodePointSource.cs

@@ -0,0 +1,84 @@
+using System.Collections;
+using System.Collections.Generic;
+using System.Text;
+
+namespace MLEM.Font {
+    /// <summary>
+    /// A code point source is a wrapper around a <see cref="string"/> or <see cref="StringBuilder"/> that allows retrieving UTF-32 code points at a given index using <see cref="GetCodePoint"/>. Additionally, it allows enumerating every code point in the underlying <see cref="string"/> or <see cref="StringBuilder"/>.
+    /// </summary>
+    public readonly struct CodePointSource : IEnumerable<int> {
+
+        private readonly string strg;
+        private readonly StringBuilder builder;
+        private char this[int index] => this.strg?[index] ?? this.builder[index];
+
+        /// <summary>
+        /// The length of this code point, in characters.
+        /// Note that this is not representative of the amount of code points in this source.
+        /// </summary>
+        public int Length => this.strg?.Length ?? this.builder.Length;
+
+        /// <summary>
+        /// Creates a new code point source from the given <see cref="string"/>.
+        /// </summary>
+        /// <param name="strg">The <see cref="string"/> whose code points to inspect.</param>
+        public CodePointSource(string strg) {
+            this.strg = strg;
+            this.builder = null;
+        }
+
+        /// <summary>
+        /// Creates a new code point source from the given <see cref="StringBuilder"/>.
+        /// </summary>
+        /// <param name="builder">The <see cref="StringBuilder"/> whose code points to inspect.</param>
+        public CodePointSource(StringBuilder builder) {
+            this.strg = null;
+            this.builder = builder;
+        }
+
+        /// <summary>
+        /// Returns the code point at the given <paramref name="index"/> in this code point source's underlying string, where the index is measured in characters and not code points.
+        /// The resulting code point will either be a single <see cref="char"/> cast to an <see cref="int"/>, at which point the returned length will be 1, or a UTF-32 <see cref="int"/> character made up of two <see cref="char"/> values, at which point the returned length will be 2.
+        /// </summary>
+        /// <param name="index">The index at which to return the code point, which is measured in characters.</param>
+        /// <param name="indexLowSurrogate">Whether the <paramref name="index"/> represents a low surrogate. If this is <see langword="false"/>, the <paramref name="index"/> represents a high surrogate and the low surrogate will be looked for in the following character. If this is <see langword="true"/>, the <paramref name="index"/> represents a low surrogate and the high surrogate will be looked for in the previous character.</param>
+        /// <returns></returns>
+        public (int CodePoint, int Length) GetCodePoint(int index, bool indexLowSurrogate = false) {
+            var curr = this[index];
+            if (indexLowSurrogate) {
+                if (index > 0) {
+                    var high = this[index - 1];
+                    if (char.IsSurrogatePair(high, curr))
+                        return (char.ConvertToUtf32(high, curr), 2);
+                }
+            } else {
+                if (index < this.Length - 1) {
+                    var low = this[index + 1];
+                    if (char.IsSurrogatePair(curr, low))
+                        return (char.ConvertToUtf32(curr, low), 2);
+                }
+            }
+            return (curr, 1);
+        }
+
+        /// <summary>Returns an enumerator that iterates through the collection.</summary>
+        /// <returns>A <see cref="T:System.Collections.Generic.IEnumerator`1" /> that can be used to iterate through the collection.</returns>
+        /// <filterpriority>1</filterpriority>
+        public IEnumerator<int> GetEnumerator() {
+            var index = 0;
+            while (index < this.Length) {
+                var (codePoint, length) = this.GetCodePoint(index);
+                yield return codePoint;
+                index += length;
+            }
+        }
+
+        /// <summary>Returns an enumerator that iterates through a collection.</summary>
+        /// <returns>An <see cref="T:System.Collections.IEnumerator" /> object that can be used to iterate through the collection.</returns>
+        /// <filterpriority>2</filterpriority>
+        IEnumerator IEnumerable.GetEnumerator() {
+            return this.GetEnumerator();
+        }
+
+    }
+}

MLEM/Font/GenericFont.cs

@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.Text;
 using Microsoft.Xna.Framework;
 using Microsoft.Xna.Framework.Graphics;
-using MLEM.Extensions;
 using MLEM.Misc;
 
 namespace MLEM.Font {
@@ -50,35 +49,36 @@ namespace MLEM.Font {
         public abstract float LineHeight { get; }
 
         /// <summary>
-        /// Measures the width of the given character with the default scale for use in <see cref="MeasureString(string,bool)"/>.
+        /// Measures the width of the given code point with the default scale for use in <see cref="MeasureString(string,bool)"/>.
         /// Note that this method does not support <see cref="Nbsp"/>, <see cref="Zwsp"/> and <see cref="Emsp"/> for most generic fonts, which is why <see cref="MeasureString(string,bool)"/> should be used even for single characters.
         /// </summary>
-        /// <param name="c">The character whose width to calculate</param>
+        /// <param name="codePoint">The code point whose width to calculate</param>
         /// <returns>The width of the given character with the default scale</returns>
-        protected abstract float MeasureChar(char c);
+        protected abstract float MeasureCharacter(int codePoint);
 
         /// <summary>
-        /// Draws the given character with the given data for use in <see cref="DrawString(Microsoft.Xna.Framework.Graphics.SpriteBatch,System.Text.StringBuilder,Microsoft.Xna.Framework.Vector2,Microsoft.Xna.Framework.Color,float,Microsoft.Xna.Framework.Vector2,Microsoft.Xna.Framework.Vector2,Microsoft.Xna.Framework.Graphics.SpriteEffects,float)"/>.
+        /// Draws the given code point with the given data for use in <see cref="DrawString(Microsoft.Xna.Framework.Graphics.SpriteBatch,System.Text.StringBuilder,Microsoft.Xna.Framework.Vector2,Microsoft.Xna.Framework.Color,float,Microsoft.Xna.Framework.Vector2,Microsoft.Xna.Framework.Vector2,Microsoft.Xna.Framework.Graphics.SpriteEffects,float)"/>.
         /// Note that this method is only called internally.
         /// </summary>
         /// <param name="batch">The sprite batch to draw with.</param>
-        /// <param name="cString">A string representation of the character which will be drawn.</param>
+        /// <param name="codePoint">The code point which will be drawn.</param>
+        /// <param name="character">A string representation of the character which will be drawn.</param>
         /// <param name="position">The drawing location on screen.</param>
         /// <param name="color">A color mask.</param>
         /// <param name="rotation">A rotation of this character.</param>
         /// <param name="scale">A scaling of this character.</param>
         /// <param name="effects">Modificators for drawing. Can be combined.</param>
         /// <param name="layerDepth">A depth of the layer of this character.</param>
-        protected abstract void DrawChar(SpriteBatch batch, string cString, Vector2 position, Color color, float rotation, Vector2 scale, SpriteEffects effects, float layerDepth);
+        protected abstract void DrawCharacter(SpriteBatch batch, int codePoint, string character, Vector2 position, Color color, float rotation, Vector2 scale, SpriteEffects effects, float layerDepth);
 
         ///<inheritdoc cref="SpriteBatch.DrawString(SpriteFont,string,Vector2,Color,float,Vector2,float,SpriteEffects,float)"/>
         public void DrawString(SpriteBatch batch, string text, Vector2 position, Color color, float rotation, Vector2 origin, Vector2 scale, SpriteEffects effects, float layerDepth) {
-            this.DrawString(batch, new CharSource(text), position, color, rotation, origin, scale, effects, layerDepth);
+            this.DrawString(batch, new CodePointSource(text), position, color, rotation, origin, scale, effects, layerDepth);
         }
 
         ///<inheritdoc cref="SpriteBatch.DrawString(SpriteFont,string,Vector2,Color,float,Vector2,float,SpriteEffects,float)"/>
         public void DrawString(SpriteBatch batch, StringBuilder text, Vector2 position, Color color, float rotation, Vector2 origin, Vector2 scale, SpriteEffects effects, float layerDepth) {
-            this.DrawString(batch, new CharSource(text), position, color, rotation, origin, scale, effects, layerDepth);
+            this.DrawString(batch, new CodePointSource(text), position, color, rotation, origin, scale, effects, layerDepth);
         }
 
         ///<inheritdoc cref="SpriteBatch.DrawString(SpriteFont,string,Vector2,Color,float,Vector2,float,SpriteEffects,float)"/>
@@ -103,19 +103,19 @@ namespace MLEM.Font {
 
         /// <summary>
         /// Measures the width of the given string when drawn with this font's underlying font.
-        /// This method uses <see cref="MeasureChar"/> internally to calculate the size of known characters and calculates additional characters like <see cref="Nbsp"/>, <see cref="Zwsp"/> and <see cref="Emsp"/>.
+        /// This method uses <see cref="MeasureCharacter"/> internally to calculate the size of known characters and calculates additional characters like <see cref="Nbsp"/>, <see cref="Zwsp"/> and <see cref="Emsp"/>.
         /// If the text contains newline characters (\n), the size returned will represent a rectangle that encompasses the width of the longest line and the string's full height.
         /// </summary>
         /// <param name="text">The text whose size to calculate</param>
         /// <param name="ignoreTrailingSpaces">Whether trailing whitespace should be ignored in the returned size, causing the end of each line to be effectively trimmed</param>
         /// <returns>The size of the string when drawn with this font</returns>
         public Vector2 MeasureString(string text, bool ignoreTrailingSpaces = false) {
-            return this.MeasureString(new CharSource(text), ignoreTrailingSpaces, null);
+            return this.MeasureString(new CodePointSource(text), ignoreTrailingSpaces, null);
         }
 
         /// <inheritdoc cref="MeasureString(string,bool)"/>
         public Vector2 MeasureString(StringBuilder text, bool ignoreTrailingSpaces = false) {
-            return this.MeasureString(new CharSource(text), ignoreTrailingSpaces, null);
+            return this.MeasureString(new CodePointSource(text), ignoreTrailingSpaces, null);
         }
 
         /// <summary>
@@ -129,12 +129,12 @@ namespace MLEM.Font {
         /// <param name="ellipsis">The characters to add to the end of the string if it is too long</param>
         /// <returns>The truncated string, or the same string if it is shorter than the maximum width</returns>
         public string TruncateString(string text, float width, float scale, bool fromBack = false, string ellipsis = "") {
-            return this.TruncateString(new CharSource(text), width, scale, fromBack, ellipsis, null).ToString();
+            return this.TruncateString(new CodePointSource(text), width, scale, fromBack, ellipsis, null).ToString();
         }
 
         /// <inheritdoc cref="TruncateString(string,float,float,bool,string)"/>
         public StringBuilder TruncateString(StringBuilder text, float width, float scale, bool fromBack = false, string ellipsis = "") {
-            return this.TruncateString(new CharSource(text), width, scale, fromBack, ellipsis, null);
+            return this.TruncateString(new CodePointSource(text), width, scale, fromBack, ellipsis, null);
         }
 
         /// <summary>
@@ -165,22 +165,24 @@ namespace MLEM.Font {
         /// <param name="scale">The scale to use for width measurements</param>
         /// <returns>The split string as an enumerable of split sections</returns>
         public IEnumerable<string> SplitStringSeparate(string text, float width, float scale) {
-            return this.SplitStringSeparate(new CharSource(text), width, scale, null);
+            return this.SplitStringSeparate(new CodePointSource(text), width, scale, null);
         }
 
         /// <inheritdoc cref="SplitStringSeparate(string,float,float)"/>
         public IEnumerable<string> SplitStringSeparate(StringBuilder text, float width, float scale) {
-            return this.SplitStringSeparate(new CharSource(text), width, scale, null);
+            return this.SplitStringSeparate(new CodePointSource(text), width, scale, null);
         }
 
-        internal Vector2 MeasureString(CharSource text, bool ignoreTrailingSpaces, Func<int, GenericFont> fontFunction) {
+        internal Vector2 MeasureString(CodePointSource text, bool ignoreTrailingSpaces, Func<int, GenericFont> fontFunction) {
             var size = Vector2.Zero;
             if (text.Length <= 0)
                 return size;
             var xOffset = 0F;
-            for (var i = 0; i < text.Length; i++) {
+            var index = 0;
-                var font = fontFunction?.Invoke(i) ?? this;
+            while (index < text.Length) {
-                switch (text[i]) {
+                var font = fontFunction?.Invoke(index) ?? this;
+                var (codePoint, length) = text.GetCodePoint(index);
+                switch (codePoint) {
                     case '\n':
                         xOffset = 0;
                         size.Y += this.LineHeight;
@@ -189,74 +191,80 @@ namespace MLEM.Font {
                         xOffset += this.LineHeight;
                         break;
                     case GenericFont.Nbsp:
-                        xOffset += font.MeasureChar(' ');
+                        xOffset += font.MeasureCharacter(' ');
                         break;
                     case GenericFont.Zwsp:
                         // don't add width for a zero-width space
                         break;
                     case ' ':
-                        if (ignoreTrailingSpaces && GenericFont.IsTrailingSpace(text, i)) {
+                        if (ignoreTrailingSpaces && GenericFont.IsTrailingSpace(text, index)) {
                             // if this is a trailing space, we can skip remaining spaces too
-                            i = text.Length - 1;
+                            index = text.Length - 1;
                             break;
                         }
-                        xOffset += font.MeasureChar(' ');
+                        xOffset += font.MeasureCharacter(' ');
                         break;
                     default:
-                        xOffset += font.MeasureChar(text[i]);
+                        xOffset += font.MeasureCharacter(codePoint);
                         break;
                 }
                 // increase x size if this line is the longest
                 if (xOffset > size.X)
                     size.X = xOffset;
+                index += length;
             }
             // include the last line's height too!
             size.Y += this.LineHeight;
             return size;
         }
 
-        internal StringBuilder TruncateString(CharSource text, float width, float scale, bool fromBack, string ellipsis, Func<int, GenericFont> fontFunction) {
+        internal StringBuilder TruncateString(CodePointSource text, float width, float scale, bool fromBack, string ellipsis, Func<int, GenericFont> fontFunction) {
             var total = new StringBuilder();
-            for (var i = 0; i < text.Length; i++) {
+            var index = 0;
+            while (index < text.Length) {
+                var innerIndex = fromBack ? text.Length - 1 - index : index;
+                var (codePoint, length) = text.GetCodePoint(innerIndex, fromBack);
                 if (fromBack) {
-                    total.Insert(0, text[text.Length - 1 - i]);
+                    total.Insert(0, char.ConvertFromUtf32(codePoint));
                 } else {
-                    total.Append(text[i]);
+                    total.Append(char.ConvertFromUtf32(codePoint));
                 }
 
-                var font = fontFunction?.Invoke(i) ?? this;
+                if (this.MeasureString(new CodePointSource(total + ellipsis), false, fontFunction).X * scale >= width) {
-                if (font.MeasureString(total + ellipsis).X * scale >= width) {
                     if (fromBack) {
-                        return total.Remove(0, 1).Insert(0, ellipsis);
+                        return total.Remove(0, length).Insert(0, ellipsis);
                     } else {
-                        return total.Remove(total.Length - 1, 1).Append(ellipsis);
+                        return total.Remove(total.Length - length, length).Append(ellipsis);
                     }
                 }
+                index += length;
             }
             return total;
         }
 
-        internal IEnumerable<string> SplitStringSeparate(CharSource text, float width, float scale, Func<int, GenericFont> fontFunction) {
+        internal IEnumerable<string> SplitStringSeparate(CodePointSource text, float width, float scale, Func<int, GenericFont> fontFunction) {
             var currWidth = 0F;
             var lastSpaceIndex = -1;
             var widthSinceLastSpace = 0F;
             var curr = new StringBuilder();
-            for (var i = 0; i < text.Length; i++) {
+            var index = 0;
-                var c = text[i];
+            while (index < text.Length) {
-                if (c == '\n') {
+                var (codePoint, length) = text.GetCodePoint(index);
+                if (codePoint == '\n') {
                     // fake split at pre-defined new lines
-                    curr.Append(c);
+                    curr.Append('\n');
                     lastSpaceIndex = -1;
                     widthSinceLastSpace = 0;
                     currWidth = 0;
                 } else {
-                    var font = fontFunction?.Invoke(i) ?? this;
+                    var font = fontFunction?.Invoke(index) ?? this;
-                    var cWidth = font.MeasureString(c.ToCachedString()).X * scale;
+                    var character = char.ConvertFromUtf32(codePoint);
-                    if (c == ' ' || c == GenericFont.Emsp || c == GenericFont.Zwsp) {
+                    var charWidth = font.MeasureString(character).X * scale;
+                    if (codePoint == ' ' || codePoint == GenericFont.Emsp || codePoint == GenericFont.Zwsp) {
                         // remember the location of this (breaking!) space
                         lastSpaceIndex = curr.Length;
                         widthSinceLastSpace = 0;
-                    } else if (currWidth + cWidth >= width) {
+                    } else if (currWidth + charWidth >= width) {
                         // check if this line contains a space
                         if (lastSpaceIndex < 0) {
                             // if there is no last space, the word is longer than a line so we split here
@@ -275,16 +283,17 @@ namespace MLEM.Font {
                     }
 
                     // add current character
-                    currWidth += cWidth;
+                    currWidth += charWidth;
-                    widthSinceLastSpace += cWidth;
+                    widthSinceLastSpace += charWidth;
-                    curr.Append(c);
+                    curr.Append(character);
                 }
+                index += length;
             }
             if (curr.Length > 0)
                 yield return curr.ToString();
         }
 
-        private void DrawString(SpriteBatch batch, CharSource text, Vector2 position, Color color, float rotation, Vector2 origin, Vector2 scale, SpriteEffects effects, float layerDepth) {
+        private void DrawString(SpriteBatch batch, CodePointSource text, Vector2 position, Color color, float rotation, Vector2 origin, Vector2 scale, SpriteEffects effects, float layerDepth) {
             var (flipX, flipY) = (0F, 0F);
             var flippedV = (effects & SpriteEffects.FlipVertically) != 0;
             var flippedH = (effects & SpriteEffects.FlipHorizontally) != 0;
@@ -318,56 +327,39 @@ namespace MLEM.Font {
             }
 
             var offset = Vector2.Zero;
-            for (var i = 0; i < text.Length; i++) {
+            var index = 0;
-                var c = text[i];
+            while (index < text.Length) {
-                if (c == '\n') {
+                var (codePoint, length) = text.GetCodePoint(index);
+                if (codePoint == '\n') {
                     offset.X = 0;
                     offset.Y += this.LineHeight;
-                    continue;
+                } else {
+                    var character = char.ConvertFromUtf32(codePoint);
+                    var charSize = this.MeasureString(character);
+
+                    var charPos = offset;
+                    if (flippedH)
+                        charPos.X += charSize.X;
+                    if (flippedV)
+                        charPos.Y += charSize.Y - this.LineHeight;
+                    Vector2.Transform(ref charPos, ref trans, out charPos);
+
+                    this.DrawCharacter(batch, codePoint, character, charPos, color, rotation, scale, effects, layerDepth);
+                    offset.X += charSize.X;
                 }
-
+                index += length;
-                var cString = c.ToCachedString();
-                var cSize = this.MeasureString(cString);
-
-                var charPos = offset;
-                if (flippedH)
-                    charPos.X += cSize.X;
-                if (flippedV)
-                    charPos.Y += cSize.Y - this.LineHeight;
-                Vector2.Transform(ref charPos, ref trans, out charPos);
-
-                this.DrawChar(batch, cString, charPos, color, rotation, scale, effects, layerDepth);
-                offset.X += cSize.X;
             }
         }
 
-        private static bool IsTrailingSpace(CharSource s, int index) {
+        private static bool IsTrailingSpace(CodePointSource s, int index) {
-            for (var i = index + 1; i < s.Length; i++) {
+            while (index < s.Length) {
-                if (s[i] != ' ')
+                var (codePoint, length) = s.GetCodePoint(index);
+                if (codePoint != ' ')
                     return false;
+                index += length;
             }
             return true;
         }
 
-        internal readonly struct CharSource {
-
-            private readonly string strg;
-            private readonly StringBuilder builder;
-
-            public int Length => this.strg?.Length ?? this.builder.Length;
-            public char this[int index] => this.strg?[index] ?? this.builder[index];
-
-            public CharSource(string strg) {
-                this.strg = strg;
-                this.builder = null;
-            }
-
-            public CharSource(StringBuilder builder) {
-                this.strg = null;
-                this.builder = builder;
-            }
-
-        }
-
     }
 }

MLEM/Font/GenericSpriteFont.cs

@@ -1,6 +1,5 @@
 using Microsoft.Xna.Framework;
 using Microsoft.Xna.Framework.Graphics;
-using MLEM.Extensions;
 
 #if !FNA
 using System.Linq;
@@ -35,13 +34,13 @@ namespace MLEM.Font {
         }
 
         /// <inheritdoc />
-        protected override float MeasureChar(char c) {
+        protected override float MeasureCharacter(int codePoint) {
-            return this.Font.MeasureString(c.ToCachedString()).X;
+            return this.Font.MeasureString(char.ConvertFromUtf32(codePoint)).X;
         }
 
         /// <inheritdoc />
-        protected override void DrawChar(SpriteBatch batch, string cString, Vector2 position, Color color, float rotation, Vector2 scale, SpriteEffects effects, float layerDepth) {
+        protected override void DrawCharacter(SpriteBatch batch, int codePoint, string character, Vector2 position, Color color, float rotation, Vector2 scale, SpriteEffects effects, float layerDepth) {
-            batch.DrawString(this.Font, cString, position, color, rotation, Vector2.Zero, scale, effects, layerDepth);
+            batch.DrawString(this.Font, character, position, color, rotation, Vector2.Zero, scale, effects, layerDepth);
         }
 
         private static SpriteFont SetDefaults(SpriteFont font) {

MLEM/Formatting/Codes/Code.cs

@@ -85,7 +85,7 @@ namespace MLEM.Formatting.Codes {
         }
 
         /// <inheritdoc cref="Formatting.Token.DrawCharacter"/>
-        public virtual bool DrawCharacter(GameTime time, SpriteBatch batch, char c, string cString, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
+        public virtual bool DrawCharacter(GameTime time, SpriteBatch batch, int codePoint, string character, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
             return false;
         }
 

MLEM/Formatting/Codes/ImageCode.cs

@@ -27,7 +27,7 @@ namespace MLEM.Formatting.Codes {
 
         /// <inheritdoc />
         public override string GetReplacementString(GenericFont font) {
-            return GenericFont.Emsp.ToCachedString();
+            return GenericFont.Emsp.ToString();
         }
 
         /// <inheritdoc />
@@ -42,7 +42,7 @@ namespace MLEM.Formatting.Codes {
         }
 
         /// <inheritdoc />
-        public override bool DrawCharacter(GameTime time, SpriteBatch batch, char c, string cString, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
+        public override bool DrawCharacter(GameTime time, SpriteBatch batch, int codePoint, string character, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
             // we don't want to draw the first (space) character (in case it is set to a missing character in FNA)
             return indexInToken == 0;
         }

MLEM/Formatting/Codes/LinkCode.cs

@@ -35,9 +35,9 @@ namespace MLEM.Formatting.Codes {
         }
 
         /// <inheritdoc />
-        public override bool DrawCharacter(GameTime time, SpriteBatch batch, char c, string cString, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
+        public override bool DrawCharacter(GameTime time, SpriteBatch batch, int codePoint, string character, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
             // since we inherit from UnderlineCode, we can just call base if selected
-            return this.IsSelected() && base.DrawCharacter(time, batch, c, cString, token, indexInToken, ref pos, font, ref color, ref scale, depth);
+            return this.IsSelected() && base.DrawCharacter(time, batch, codePoint, character, token, indexInToken, ref pos, font, ref color, ref scale, depth);
         }
 
     }

MLEM/Formatting/Codes/ShadowCode.cs

@@ -18,8 +18,8 @@ namespace MLEM.Formatting.Codes {
         }
 
         /// <inheritdoc />
-        public override bool DrawCharacter(GameTime time, SpriteBatch batch, char c, string cString, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
+        public override bool DrawCharacter(GameTime time, SpriteBatch batch, int codePoint, string character, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
-            font.DrawString(batch, cString, pos + this.offset * scale, this.color.CopyAlpha(color), 0, Vector2.Zero, scale, SpriteEffects.None, depth);
+            font.DrawString(batch, character, pos + this.offset * scale, this.color.CopyAlpha(color), 0, Vector2.Zero, scale, SpriteEffects.None, depth);
             // we return false since we still want regular drawing to occur
             return false;
         }

MLEM/Formatting/Codes/UnderlineCode.cs

@@ -19,11 +19,11 @@ namespace MLEM.Formatting.Codes {
         }
 
         /// <inheritdoc />
-        public override bool DrawCharacter(GameTime time, SpriteBatch batch, char c, string cString, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
+        public override bool DrawCharacter(GameTime time, SpriteBatch batch, int codePoint, string character, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
             // don't underline spaces at the end of lines
-            if (c == ' ' && token.DisplayString.Length > indexInToken + 1 && token.DisplayString[indexInToken + 1] == '\n')
+            if (codePoint == ' ' && token.DisplayString.Length > indexInToken + 1 && token.DisplayString[indexInToken + 1] == '\n')
                 return false;
-            var size = font.MeasureString(cString) * scale;
+            var size = font.MeasureString(character) * scale;
             var t = size.Y * this.thickness;
             batch.Draw(batch.GetBlankTexture(), new RectangleF(pos.X, pos.Y + this.yOffset * size.Y - t, size.X, t), color);
             return false;

MLEM/Formatting/Codes/WobblyCode.cs

@@ -28,7 +28,7 @@ namespace MLEM.Formatting.Codes {
         }
 
         /// <inheritdoc />
-        public override bool DrawCharacter(GameTime time, SpriteBatch batch, char c, string cString, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
+        public override bool DrawCharacter(GameTime time, SpriteBatch batch, int codePoint, string character, Token token, int indexInToken, ref Vector2 pos, GenericFont font, ref Color color, ref float scale, float depth) {
             var offset = new Vector2(0, (float) Math.Sin(token.Index + indexInToken + this.TimeIntoAnimation.TotalSeconds * this.modifier) * font.LineHeight * this.heightModifier * scale);
             pos += offset;
             // we return false since we still want regular drawing to occur, we just changed the position

MLEM/Formatting/TextFormatter.cs

@@ -59,8 +59,8 @@ namespace MLEM.Formatting {
             this.Codes.Add(new Regex(@"</(\w+)>"), (f, m, r) => new SimpleEndCode(m, r, m.Groups[1].Value));
 
             // macros
-            this.Macros.Add(new Regex("~"), (f, m, r) => GenericFont.Nbsp.ToCachedString());
+            this.Macros.Add(new Regex("~"), (f, m, r) => GenericFont.Nbsp.ToString());
-            this.Macros.Add(new Regex("<n>"), (f, m, r) => '\n'.ToCachedString());
+            this.Macros.Add(new Regex("<n>"), (f, m, r) => '\n'.ToString());
         }
 
         /// <summary>

MLEM/Formatting/Token.cs

@@ -97,26 +97,26 @@ namespace MLEM.Formatting {
         }
 
         /// <summary>
-        /// Draws a given character using this token's formatting options.
+        /// Draws a given code point using this token's formatting options.
         /// </summary>
         /// <param name="time">The time</param>
         /// <param name="batch">The sprite batch to use</param>
-        /// <param name="c">The character to draw</param>
+        /// <param name="codePoint">The code point of the character to draw</param>
-        /// <param name="cString">A single-character string that contains the character to draw</param>
+        /// <param name="character">The string representation of the character to draw</param>
         /// <param name="indexInToken">The index within this token that the character is at</param>
         /// <param name="pos">The position to draw the token at</param>
         /// <param name="font">The font to use to draw</param>
         /// <param name="color">The color to draw with</param>
         /// <param name="scale">The scale to draw at</param>
         /// <param name="depth">The depth to draw at</param>
-        public void DrawCharacter(GameTime time, SpriteBatch batch, char c, string cString, int indexInToken, Vector2 pos, GenericFont font, Color color, float scale, float depth) {
+        public void DrawCharacter(GameTime time, SpriteBatch batch, int codePoint, string character, int indexInToken, Vector2 pos, GenericFont font, Color color, float scale, float depth) {
             foreach (var code in this.AppliedCodes) {
-                if (code.DrawCharacter(time, batch, c, cString, this, indexInToken, ref pos, font, ref color, ref scale, depth))
+                if (code.DrawCharacter(time, batch, codePoint, character, this, indexInToken, ref pos, font, ref color, ref scale, depth))
                     return;
             }
 
             // if no code drew, we have to do it ourselves
-            font.DrawString(batch, cString, pos, color, 0, Vector2.Zero, scale, SpriteEffects.None, depth);
+            font.DrawString(batch, character, pos, color, 0, Vector2.Zero, scale, SpriteEffects.None, depth);
         }
 
         /// <summary>

MLEM/Formatting/TokenizedString.cs

@@ -4,11 +4,9 @@ using System.Linq;
 using System.Text;
 using Microsoft.Xna.Framework;
 using Microsoft.Xna.Framework.Graphics;
-using MLEM.Extensions;
 using MLEM.Font;
 using MLEM.Formatting.Codes;
 using MLEM.Misc;
-using static MLEM.Font.GenericFont;
 
 namespace MLEM.Formatting {
     /// <summary>
@@ -65,7 +63,7 @@ namespace MLEM.Formatting {
         /// <param name="alignment">The text alignment that should be used for width calculations</param>
         public void Split(GenericFont font, float width, float scale, TextAlignment alignment = TextAlignment.Left) {
             // a split string has the same character count as the input string but with newline characters added
-            this.modifiedString = string.Join("\n", font.SplitStringSeparate(new CharSource(this.String), width, scale, i => this.GetFontForIndex(font, i)));
+            this.modifiedString = string.Join("\n", font.SplitStringSeparate(new CodePointSource(this.String), width, scale, i => this.GetFontForIndex(font, i)));
             this.StoreModifiedSubstrings(font, alignment);
         }
 
@@ -80,7 +78,7 @@ namespace MLEM.Formatting {
         /// <param name="ellipsis">The characters to add to the end of the string if it is too long</param>
         /// <param name="alignment">The text alignment that should be used for width calculations</param>
         public void Truncate(GenericFont font, float width, float scale, string ellipsis = "", TextAlignment alignment = TextAlignment.Left) {
-            this.modifiedString = font.TruncateString(new CharSource(this.String), width, scale, false, ellipsis, i => this.GetFontForIndex(font, i)).ToString();
+            this.modifiedString = font.TruncateString(new CodePointSource(this.String), width, scale, false, ellipsis, i => this.GetFontForIndex(font, i)).ToString();
             this.StoreModifiedSubstrings(font, alignment);
         }
 
@@ -122,7 +120,7 @@ namespace MLEM.Formatting {
 
         /// <inheritdoc cref="GenericFont.MeasureString(string,bool)"/>
         public Vector2 Measure(GenericFont font) {
-            return font.MeasureString(new CharSource(this.DisplayString), false, i => this.GetFontForIndex(font, i));
+            return font.MeasureString(new CodePointSource(this.DisplayString), false, i => this.GetFontForIndex(font, i));
         }
 
         /// <summary>
@@ -162,14 +160,18 @@ namespace MLEM.Formatting {
 
                 var indexInToken = 0;
                 for (var l = 0; l < token.SplitDisplayString.Length; l++) {
-                    foreach (var c in token.SplitDisplayString[l]) {
+                    var charIndex = 0;
-                        var cString = c.ToCachedString();
+                    var line = new CodePointSource(token.SplitDisplayString[l]);
+                    while (charIndex < line.Length) {
+                        var (codePoint, length) = line.GetCodePoint(charIndex);
+                        var character = char.ConvertFromUtf32(codePoint);
 
                         if (indexInToken == 0)
                             token.DrawSelf(time, batch, pos + innerOffset, drawFont, color, scale, depth);
-                        token.DrawCharacter(time, batch, c, cString, indexInToken, pos + innerOffset, drawFont, drawColor, scale, depth);
+                        token.DrawCharacter(time, batch, codePoint, character, indexInToken, pos + innerOffset, drawFont, drawColor, scale, depth);
 
-                        innerOffset.X += drawFont.MeasureString(cString).X * scale;
+                        innerOffset.X += drawFont.MeasureString(character).X * scale;
+                        charIndex += length;
                         indexInToken++;
                     }
 

MLEM/Input/TextInput.cs

@@ -417,9 +417,10 @@ namespace MLEM.Input {
         private bool FilterText(ref string text, bool removeMismatching) {
             if (removeMismatching) {
                 var result = new StringBuilder();
-                foreach (var c in text) {
+                foreach (var codePoint in new CodePointSource(text)) {
-                    if (this.InputRule(this, c.ToCachedString()))
+                    var character = char.ConvertFromUtf32(codePoint);
-                        result.Append(c);
+                    if (this.InputRule(this, character))
+                        result.Append(character);
                 }
                 text = result.ToString();
             } else if (!this.InputRule(this, text))

MLEM/Misc/GenericDataHolder.cs

@@ -1,4 +1,3 @@
-using System;
 using System.Collections.Generic;
 using System.Runtime.Serialization;
 

Sandbox/Content/Content.mgcb

@@ -17,6 +17,9 @@
 #begin Fonts/Cadman_Roman.otf
 /copy:Fonts/Cadman_Roman.otf
 
+#begin Fonts/Symbola-Emoji.ttf
+/copy:Fonts/Symbola-Emoji.ttf
+
 #begin Fonts/Regular.fnt
 /importer:BitmapFontImporter
 /processor:BitmapFontProcessor

Sandbox/GameImpl.cs

@@ -11,11 +11,9 @@ using MLEM.Data;
 using MLEM.Data.Content;
 using MLEM.Extended.Font;
 using MLEM.Extensions;
-using MLEM.Font;
 using MLEM.Formatting;
 using MLEM.Formatting.Codes;
 using MLEM.Graphics;
-using MLEM.Input;
 using MLEM.Misc;
 using MLEM.Startup;
 using MLEM.Textures;
@@ -67,10 +65,13 @@ public class GameImpl : MlemGame {
 
         var system = new FontSystem();
         system.AddFont(File.ReadAllBytes("Content/Fonts/Cadman_Roman.otf"));
+        system.AddFont(File.ReadAllBytes("Content/Fonts/Symbola-Emoji.ttf"));
         //var font = new GenericSpriteFont(LoadContent<SpriteFont>("Fonts/TestFont"));
         //var font = new GenericBitmapFont(LoadContent<BitmapFont>("Fonts/Regular"));
         var font = new GenericStashFont(system.GetFont(32));
+        /*
         var spriteFont = new GenericSpriteFont(MlemGame.LoadContent<SpriteFont>("Fonts/TestFont"));
+        */
         this.UiSystem.Style = new UntexturedStyle(this.SpriteBatch) {
             Font = font,
             TextScale = 0.5F,
@@ -147,7 +148,7 @@ public class GameImpl : MlemGame {
         formatter.Macros.Add(new Regex("<testmacro>"), (_, _, _) => "<test1>");
         formatter.Macros.Add(new Regex("<test1>"), (_, _, _) => "<test2> blue");
         formatter.Macros.Add(new Regex("<test2>"), (_, _, _) => "<c Blue>");
-        const string strg = "This \nstring    \nis\n     split     \n    weirdly    \n  .     ";
+        const string strg = "This is a string containing 💡 emoji";
         //var strg = "Lorem Ipsum <i Test> is simply dummy text of the <i Test> printing and typesetting <i Test> industry. Lorem Ipsum has been the industry's standard dummy text <i Test> ever since the <i Test> 1500s, when <i Test><i Test><i Test><i Test><i Test><i Test><i Test> an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.";
         //var strg = "Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.";
         //var strg = "This is <u>a test of the underlined formatting code</u>!";
@@ -256,43 +257,43 @@ public class GameImpl : MlemGame {
         }).SetData("Ref", "Main");
         this.UiSystem.Add("SpillTest", spillPanel);*/
 
-        var regularFont = spriteFont.Font;
+        /* var regularFont = spriteFont.Font;
-        var genericFont = spriteFont;
+         var genericFont = spriteFont;
 
-        var index = 0;
+         var index = 0;
-        var pos = new Vector2(100, 20);
+         var pos = new Vector2(100, 20);
-        var scale = 1F;
+         var scale = 1F;
-        var origin = Vector2.Zero;
+         var origin = Vector2.Zero;
-        var rotation = 0F;
+         var rotation = 0F;
-        var effects = SpriteEffects.None;
+         var effects = SpriteEffects.None;
 
-        this.OnDraw += (_, _) => {
+         this.OnDraw += (_, _) => {
-            const string testString = "This is a\ntest string\n\twith long lines.\nLet's write some more stuff. Let's\r\nsplit lines weirdly.";
+             const string testString = "This is a\ntest string\n\twith long lines.\nLet's write some more stuff. Let's\r\nsplit lines weirdly.";
-            if (MlemGame.Input.IsKeyPressed(Keys.I)) {
+             if (MlemGame.Input.IsKeyPressed(Keys.I)) {
-                index++;
+                 index++;
-                if (index == 1) {
+                 if (index == 1) {
-                    scale = 2;
+                     scale = 2;
-                } else if (index == 2) {
+                 } else if (index == 2) {
-                    origin = new Vector2(15, 15);
+                     origin = new Vector2(15, 15);
-                } else if (index == 3) {
+                 } else if (index == 3) {
-                    rotation = 0.25F;
+                     rotation = 0.25F;
-                } else if (index == 4) {
+                 } else if (index == 4) {
-                    effects = SpriteEffects.FlipHorizontally;
+                     effects = SpriteEffects.FlipHorizontally;
-                } else if (index == 5) {
+                 } else if (index == 5) {
-                    effects = SpriteEffects.FlipVertically;
+                     effects = SpriteEffects.FlipVertically;
-                } else if (index == 6) {
+                 } else if (index == 6) {
-                    effects = SpriteEffects.FlipHorizontally | SpriteEffects.FlipVertically;
+                     effects = SpriteEffects.FlipHorizontally | SpriteEffects.FlipVertically;
-                }
+                 }
-            }
+             }
 
-            /*this.SpriteBatch.Begin();
+             this.SpriteBatch.Begin();
-            if (MlemGame.Input.IsKeyDown(Keys.LeftShift)) {
+             if (MlemGame.Input.IsKeyDown(Keys.LeftShift)) {
-                this.SpriteBatch.DrawString(regularFont, testString, pos, Color.Red, rotation, origin, scale, effects, 0);
+                 this.SpriteBatch.DrawString(regularFont, testString, pos, Color.Red, rotation, origin, scale, effects, 0);
-            } else {
+             } else {
-                genericFont.DrawString(this.SpriteBatch, testString, pos, Color.Green, rotation, origin, scale, effects, 0);
+                 genericFont.DrawString(this.SpriteBatch, testString, pos, Color.Green, rotation, origin, scale, effects, 0);
-            }
+             }
-            this.SpriteBatch.End();*/
+             this.SpriteBatch.End();
-        };
+         };*/
 
         /*var viewport = new BoxingViewportAdapter(this.Window, this.GraphicsDevice, 1280, 720);
         var newPanel = new Panel(Anchor.TopLeft, new Vector2(200, 100), new Vector2(10, 10));

Sandbox/Sandbox.csproj

@@ -23,6 +23,10 @@
         <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     </ItemGroup>
 
+    <ItemGroup>
+      <Content Include="Content\Fonts\Symbola-Emoji.ttf" />
+    </ItemGroup>
+
     <Target Name="RestoreDotnetTools" BeforeTargets="Restore">
         <Message Text="Restoring dotnet tools" Importance="High" />
         <Exec Command="dotnet tool restore" />

Tests/FontTests.cs

@@ -2,7 +2,6 @@ using System;
 using System.Text.RegularExpressions;
 using Microsoft.Xna.Framework;
 using Microsoft.Xna.Framework.Graphics;
-using MLEM.Extensions;
 using MLEM.Font;
 using MLEM.Formatting;
 using MLEM.Formatting.Codes;
@@ -150,8 +149,8 @@ namespace Tests {
             CompareSizes($"This is a very simple{GenericFont.Emsp}test string");
             CompareSizes($"This is a very simple{GenericFont.Zwsp}test string");
 
-            Assert.AreEqual(new Vector2(this.font.LineHeight, this.font.LineHeight), this.font.MeasureString(GenericFont.Emsp.ToCachedString()));
+            Assert.AreEqual(new Vector2(this.font.LineHeight, this.font.LineHeight), this.font.MeasureString(GenericFont.Emsp.ToString()));
-            Assert.AreEqual(new Vector2(0, this.font.LineHeight), this.font.MeasureString(GenericFont.Zwsp.ToCachedString()));
+            Assert.AreEqual(new Vector2(0, this.font.LineHeight), this.font.MeasureString(GenericFont.Zwsp.ToString()));
         }
 
     }