Cleaned up GenericFont and TokenizedString by improving the splitting and truncating algorithms

CHANGELOG.md

@@ -19,6 +19,7 @@ Additions
 - Added the ability for UniformTextureAtlases to have padding for each region
 - Added UniformTextureAtlas methods ToList and ToDictionary
 - Added SingleRandom and SeedSource
+- Added TokenizedString.GetArea
 - **Added the ability to find paths to one of multiple goals using AStar**
 
 Improvements
@@ -46,6 +47,7 @@ Removals
 - Removed DataContract attribute from GenericDataHolder
 - Marked EnumHelper as obsolete due to its reimplementation in [DynamicEnums](https://www.nuget.org/packages/DynamicEnums)
 - Marked Code.GetReplacementString as obsolete
+- Marked TokenizedString.Measure as obsolete in favor of GetArea
 
 ### MLEM.Ui
 Additions

Demos/TextFormattingDemo.cs

@@ -6,6 +6,7 @@ using MLEM.Extensions;
 using MLEM.Font;
 using MLEM.Formatting;
 using MLEM.Formatting.Codes;
+using MLEM.Misc;
 using MLEM.Startup;
 using MLEM.Textures;
 
@@ -57,14 +58,16 @@ namespace Demos {
 
             // we draw the tokenized text in the center of the screen
             // since the text is already center-aligned, we only need to align it on the y axis here
-            var size = this.tokenizedText.Measure(this.font) * TextFormattingDemo.Scale;
+            var size = this.tokenizedText.GetArea(Vector2.Zero, TextFormattingDemo.Scale).Size;
             var pos = new Vector2(this.GraphicsDevice.Viewport.Width / 2, (this.GraphicsDevice.Viewport.Height - size.Y) / 2);
 
             // draw bounds, which can be toggled with B in this demo
             if (this.drawBounds) {
+                var blank = this.SpriteBatch.GetBlankTexture();
+                this.SpriteBatch.Draw(blank, new RectangleF(pos - new Vector2(size.X / 2, 0), size), Color.Red * 0.25F);
                 foreach (var token in this.tokenizedText.Tokens) {
                     foreach (var area in token.GetArea(pos, TextFormattingDemo.Scale))
-                        this.SpriteBatch.Draw(this.SpriteBatch.GetBlankTexture(), area, Color.Black * 0.25F);
+                        this.SpriteBatch.Draw(blank, area, Color.Black * 0.25F);
                 }
             }
 

Demos/UiDemo.cs

@@ -93,11 +93,18 @@ namespace Demos {
             // a paragraph with formatting codes. To see them all or to add more, check the TextFormatting class
             this.root.AddChild(new Paragraph(Anchor.AutoLeft, 1, "Paragraphs can also contain <c Blue>formatting codes</c>, including colors and <i>text styles</i>. For more info, check out the <b>text formatting demo</b>!"));
 
+            this.root.AddChild(new VerticalSpace(3));
+            this.root.AddChild(new Paragraph(Anchor.AutoCenter, 1, "Text input:", true));
+            this.root.AddChild(new TextField(Anchor.AutoLeft, new Vector2(1, 10)) {
+                PositionOffset = new Vector2(0, 1),
+                PlaceholderText = "Click here to input text"
+            });
+
             this.root.AddChild(new VerticalSpace(3));
             this.root.AddChild(new Paragraph(Anchor.AutoCenter, 1, "Multiline text input:", true));
             this.root.AddChild(new TextField(Anchor.AutoLeft, new Vector2(1, 50), multiline: true) {
                 PositionOffset = new Vector2(0, 1),
-                PlaceholderText = "Click here to input text"
+                PlaceholderText = "Click here to input a lot"
             });
 
             this.root.AddChild(new VerticalSpace(3));
@@ -204,12 +211,24 @@ namespace Demos {
             dropdown.AddElement(new Button(Anchor.AutoLeft, new Vector2(1, 10), "Buttons"));
 
             this.root.AddChild(new VerticalSpace(3));
-            this.root.AddChild(new Button(Anchor.AutoLeft, new Vector2(1, 10), "Disabled button", "This button can't be clicked or moved to using automatic navigation") {IsDisabled = true}).PositionOffset = new Vector2(0, 1);
-            this.root.AddChild(new Checkbox(Anchor.AutoLeft, new Vector2(1, 10), "Disabled checkbox") {IsDisabled = true}).PositionOffset = new Vector2(0, 1);
+            this.root.AddChild(new Button(Anchor.AutoLeft, new Vector2(1, 10), "Disabled button", "This button can't be clicked or moved to using automatic navigation") {
+                IsDisabled = true,
+                PositionOffset = new Vector2(0, 1)
+            });
+            this.root.AddChild(new Checkbox(Anchor.AutoLeft, new Vector2(1, 10), "Disabled checkbox") {
+                IsDisabled = true,
+                PositionOffset = new Vector2(0, 1)
+            });
             this.root.AddChild(new Button(Anchor.AutoLeft, new Vector2(1, 10), "Disabled tooltip button", "This button can't be clicked, but can be moved to using automatic navigation, and will display its tooltip even when done so.") {
                 CanSelectDisabled = true,
                 IsDisabled = true,
-                Tooltip = {DisplayInAutoNavMode = true},
+                PositionOffset = new Vector2(0, 1),
+                Tooltip = {
+                    DisplayInAutoNavMode = true
+                }
+            });
+            this.root.AddChild(new Button(Anchor.AutoLeft, new Vector2(1, 10), "Button with <b>far</b> too much text which will automatically be cut off, hi!") {
+                TruncateTextIfLong = true,
                 PositionOffset = new Vector2(0, 1)
             });
 

MLEM.Ui/Elements/Paragraph.cs

@@ -135,7 +135,7 @@ namespace MLEM.Ui.Elements {
         protected override Vector2 CalcActualSize(RectangleF parentArea) {
             var size = base.CalcActualSize(parentArea);
             this.ParseText(size);
-            var textSize = this.TokenizedText.Measure(this.RegularFont) * this.TextScale * this.TextScaleMultiplier * this.Scale;
+            var textSize = this.TokenizedText.GetArea(Vector2.Zero, this.TextScale * this.TextScaleMultiplier * this.Scale).Size;
             return new Vector2(this.AutoAdjustWidth ? textSize.X + this.ScaledPadding.Width : size.X, textSize.Y + this.ScaledPadding.Height);
         }
 

MLEM/Font/GenericFont.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 using Microsoft.Xna.Framework;
 using Microsoft.Xna.Framework.Graphics;
@@ -110,12 +111,12 @@ namespace MLEM.Font {
         /// <param name="ignoreTrailingSpaces">Whether trailing whitespace should be ignored in the returned size, causing the end of each line to be effectively trimmed</param>
         /// <returns>The size of the string when drawn with this font</returns>
         public Vector2 MeasureString(string text, bool ignoreTrailingSpaces = false) {
-            return this.MeasureString(new CodePointSource(text), ignoreTrailingSpaces, null, null);
+            return this.MeasureString(new CodePointSource(text), ignoreTrailingSpaces);
         }
 
         /// <inheritdoc cref="MeasureString(string,bool)"/>
         public Vector2 MeasureString(StringBuilder text, bool ignoreTrailingSpaces = false) {
-            return this.MeasureString(new CodePointSource(text), ignoreTrailingSpaces, null, null);
+            return this.MeasureString(new CodePointSource(text), ignoreTrailingSpaces);
         }
 
         /// <summary>
@@ -129,12 +130,12 @@ namespace MLEM.Font {
         /// <param name="ellipsis">The characters to add to the end of the string if it is too long</param>
         /// <returns>The truncated string, or the same string if it is shorter than the maximum width</returns>
         public string TruncateString(string text, float width, float scale, bool fromBack = false, string ellipsis = "") {
-            return this.TruncateString(new CodePointSource(text), width, scale, fromBack, ellipsis, null, null).ToString();
+            return GenericFont.TruncateString(Enumerable.Repeat(new DecoratedCodePointSource(new CodePointSource(text), this, 0), 1), width, scale, fromBack, ellipsis).First().ToString();
         }
 
         /// <inheritdoc cref="TruncateString(string,float,float,bool,string)"/>
         public StringBuilder TruncateString(StringBuilder text, float width, float scale, bool fromBack = false, string ellipsis = "") {
-            return this.TruncateString(new CodePointSource(text), width, scale, fromBack, ellipsis, null, null);
+            return GenericFont.TruncateString(Enumerable.Repeat(new DecoratedCodePointSource(new CodePointSource(text), this, 0), 1), width, scale, fromBack, ellipsis).First();
         }
 
         /// <summary>
@@ -165,23 +166,21 @@ namespace MLEM.Font {
         /// <param name="scale">The scale to use for width measurements</param>
         /// <returns>The split string as an enumerable of split sections</returns>
         public IEnumerable<string> SplitStringSeparate(string text, float width, float scale) {
-            return this.SplitStringSeparate(new CodePointSource(text), width, scale, null, null);
+            return GenericFont.SplitStringSeparate(Enumerable.Repeat(new DecoratedCodePointSource(new CodePointSource(text), this, 0), 1), width, scale).First();
         }
 
         /// <inheritdoc cref="SplitStringSeparate(string,float,float)"/>
         public IEnumerable<string> SplitStringSeparate(StringBuilder text, float width, float scale) {
-            return this.SplitStringSeparate(new CodePointSource(text), width, scale, null, null);
+            return GenericFont.SplitStringSeparate(Enumerable.Repeat(new DecoratedCodePointSource(new CodePointSource(text), this, 0), 1), width, scale).First();
         }
 
-        internal Vector2 MeasureString(CodePointSource text, bool ignoreTrailingSpaces, Func<int, GenericFont> fontFunction, Func<int, float> extraWidthFunction) {
+        private Vector2 MeasureString(CodePointSource text, bool ignoreTrailingSpaces) {
             var size = Vector2.Zero;
             if (text.Length <= 0)
                 return size;
             var xOffset = 0F;
             var index = 0;
             while (index < text.Length) {
-                xOffset += extraWidthFunction?.Invoke(index) ?? 0;
-                var font = fontFunction?.Invoke(index) ?? this;
                 var (codePoint, length) = text.GetCodePoint(index);
                 switch (codePoint) {
                     case '\n':
@@ -192,7 +191,7 @@ namespace MLEM.Font {
                         xOffset += this.LineHeight;
                         break;
                     case GenericFont.Nbsp:
-                        xOffset += font.MeasureCharacter(' ');
+                        xOffset += this.MeasureCharacter(' ');
                         break;
                     case GenericFont.Zwsp:
                         // don't add width for a zero-width space
@@ -203,10 +202,10 @@ namespace MLEM.Font {
                             index = text.Length - 1;
                             break;
                         }
-                        xOffset += font.MeasureCharacter(' ');
+                        xOffset += this.MeasureCharacter(' ');
                         break;
                     default:
-                        xOffset += font.MeasureCharacter(codePoint);
+                        xOffset += this.MeasureCharacter(codePoint);
                         break;
                 }
                 // increase x size if this line is the longest
@@ -219,87 +218,12 @@ namespace MLEM.Font {
             return size;
         }
 
-        internal StringBuilder TruncateString(CodePointSource text, float width, float scale, bool fromBack, string ellipsis, Func<int, GenericFont> fontFunction, Func<int, float> extraWidthFunction) {
-            var total = new StringBuilder();
-            var index = 0;
-            while (index < text.Length) {
-                var innerIndex = fromBack ? text.Length - 1 - index : index;
-                var (codePoint, length) = text.GetCodePoint(innerIndex, fromBack);
-                if (fromBack) {
-                    total.Insert(0, CodePointSource.ToString(codePoint));
-                } else {
-                    total.Append(CodePointSource.ToString(codePoint));
-                }
-
-                if (this.MeasureString(new CodePointSource(total + ellipsis), false, fontFunction, extraWidthFunction).X * scale >= width) {
-                    if (fromBack) {
-                        return total.Remove(0, length).Insert(0, ellipsis);
-                    } else {
-                        return total.Remove(total.Length - length, length).Append(ellipsis);
-                    }
-                }
-                index += length;
-            }
-            return total;
-        }
-
-        internal IEnumerable<string> SplitStringSeparate(CodePointSource text, float width, float scale, Func<int, GenericFont> fontFunction, Func<int, float> extraWidthFunction) {
-            var currWidth = 0F;
-            var lastSpaceIndex = -1;
-            var widthSinceLastSpace = 0F;
-            var curr = new StringBuilder();
-            var index = 0;
-            while (index < text.Length) {
-                var (codePoint, length) = text.GetCodePoint(index);
-                if (codePoint == '\n') {
-                    // fake split at pre-defined new lines
-                    curr.Append('\n');
-                    lastSpaceIndex = -1;
-                    widthSinceLastSpace = 0;
-                    currWidth = 0;
-                } else {
-                    var font = fontFunction?.Invoke(index) ?? this;
-                    var character = CodePointSource.ToString(codePoint);
-                    var charWidth = (font.MeasureString(character).X + (extraWidthFunction?.Invoke(index) ?? 0)) * scale;
-                    if (codePoint == ' ' || codePoint == GenericFont.Emsp || codePoint == GenericFont.Zwsp) {
-                        // remember the location of this (breaking!) space
-                        lastSpaceIndex = curr.Length;
-                        widthSinceLastSpace = 0;
-                    } else if (currWidth + charWidth >= width) {
-                        // check if this line contains a space
-                        if (lastSpaceIndex < 0) {
-                            // if there is no last space, the word is longer than a line so we split here
-                            yield return curr.ToString();
-                            currWidth = 0;
-                            curr.Clear();
-                        } else {
-                            // split after the last space
-                            yield return curr.ToString().Substring(0, lastSpaceIndex + 1);
-                            curr.Remove(0, lastSpaceIndex + 1);
-                            // we need to restore the width accumulated since the last space for the new line
-                            currWidth = widthSinceLastSpace;
-                        }
-                        widthSinceLastSpace = 0;
-                        lastSpaceIndex = -1;
-                    }
-
-                    // add current character
-                    currWidth += charWidth;
-                    widthSinceLastSpace += charWidth;
-                    curr.Append(character);
-                }
-                index += length;
-            }
-            if (curr.Length > 0)
-                yield return curr.ToString();
-        }
-
         private void DrawString(SpriteBatch batch, CodePointSource text, Vector2 position, Color color, float rotation, Vector2 origin, Vector2 scale, SpriteEffects effects, float layerDepth) {
             var (flipX, flipY) = (0F, 0F);
             var flippedV = (effects & SpriteEffects.FlipVertically) != 0;
             var flippedH = (effects & SpriteEffects.FlipHorizontally) != 0;
             if (flippedV || flippedH) {
-                var size = this.MeasureString(text, false, null, null);
+                var size = this.MeasureString(text, false);
                 if (flippedH) {
                     origin.X *= -1;
                     flipX = -size.X;
@@ -352,6 +276,126 @@ namespace MLEM.Font {
             }
         }
 
+        internal static IEnumerable<IEnumerable<string>> SplitStringSeparate(IEnumerable<DecoratedCodePointSource> text, float maxWidth, float scale) {
+            var currWidth = 0F;
+            var lastSpacePart = -1;
+            var lastSpaceIndex = -1;
+            var widthSinceLastSpace = 0F;
+            var curr = new StringBuilder();
+            var fullSplit = new List<List<string>>();
+            foreach (var part in text) {
+                var partSplit = new List<string>();
+                AddWidth(partSplit, part.ExtraWidth * scale, true);
+
+                var index = 0;
+                while (index < part.Source.Length) {
+                    var (codePoint, length) = part.Source.GetCodePoint(index);
+                    if (codePoint == '\n') {
+                        // fake split at pre-defined new lines
+                        curr.Append('\n');
+                        lastSpacePart = -1;
+                        lastSpaceIndex = -1;
+                        widthSinceLastSpace = 0;
+                        currWidth = 0;
+                    } else {
+                        var character = CodePointSource.ToString(codePoint);
+                        var charWidth = part.Font.MeasureString(character).X * scale;
+                        if (codePoint == ' ' || codePoint == GenericFont.Emsp || codePoint == GenericFont.Zwsp) {
+                            // remember the location of this (breaking!) space
+                            lastSpacePart = fullSplit.Count;
+                            lastSpaceIndex = curr.Length;
+                            widthSinceLastSpace = 0;
+                            // we never want to insert a line break before a space!
+                            AddWidth(partSplit, charWidth, false);
+                        } else {
+                            AddWidth(partSplit, charWidth, true);
+                        }
+                        curr.Append(character);
+                    }
+                    index += length;
+                }
+
+                if (curr.Length > 0) {
+                    partSplit.Add(curr.ToString());
+                    curr.Clear();
+                }
+                fullSplit.Add(partSplit);
+            }
+            return fullSplit;
+
+            void AddWidth(ICollection<string> partSplit, float width, bool canBreakHere) {
+                if (canBreakHere && currWidth + width >= maxWidth) {
+                    // check if this line contains a space
+                    if (lastSpaceIndex < 0) {
+                        // if there is no last space, the word is longer than a line so we split here
+                        partSplit.Add(curr.ToString());
+                        curr.Clear();
+                        currWidth = 0;
+                    } else {
+                        if (lastSpacePart < fullSplit.Count) {
+                            // the last space exists, but isn't a part of curr, so we have to backtrack and split the previous token
+                            var prevPart = fullSplit[lastSpacePart];
+                            var prevCurr = prevPart[prevPart.Count - 1];
+                            prevPart[prevPart.Count - 1] = prevCurr.Substring(0, lastSpaceIndex + 1);
+                            prevPart.Add(prevCurr.Substring(lastSpaceIndex + 1));
+                        } else {
+                            // split after the last space
+                            partSplit.Add(curr.ToString().Substring(0, lastSpaceIndex + 1));
+                            curr.Remove(0, lastSpaceIndex + 1);
+                        }
+                        // we need to restore the width accumulated since the last space for the new line
+                        currWidth = widthSinceLastSpace;
+                    }
+                    widthSinceLastSpace = 0;
+                    lastSpacePart = -1;
+                    lastSpaceIndex = -1;
+                }
+
+                currWidth += width;
+                widthSinceLastSpace += width;
+            }
+        }
+
+        internal static IEnumerable<StringBuilder> TruncateString(IEnumerable<DecoratedCodePointSource> text, float maxWidth, float scale, bool fromBack, string ellipsis) {
+            var total = new StringBuilder();
+            var extraWidth = 0F;
+            var endReached = false;
+            foreach (var part in (fromBack ? text.Reverse() : text)) {
+                var curr = new StringBuilder();
+                // if we reached the end previously, all the other parts should just be empty
+                if (!endReached) {
+                    extraWidth += part.ExtraWidth * scale;
+                    var index = 0;
+                    while (index < part.Source.Length) {
+                        var innerIndex = fromBack ? part.Source.Length - 1 - index : index;
+                        var (codePoint, length) = part.Source.GetCodePoint(innerIndex, fromBack);
+                        var character = CodePointSource.ToString(codePoint);
+                        if (fromBack) {
+                            curr.Insert(0, character);
+                            total.Insert(0, character);
+                        } else {
+                            curr.Append(character);
+                            total.Append(character);
+                        }
+
+                        if (part.Font.MeasureString(new CodePointSource(total + ellipsis), false).X * scale + extraWidth >= maxWidth) {
+                            if (fromBack) {
+                                curr.Remove(0, length).Insert(0, ellipsis);
+                                total.Remove(0, length).Insert(0, ellipsis);
+                            } else {
+                                curr.Remove(curr.Length - length, length).Append(ellipsis);
+                                total.Remove(total.Length - length, length).Append(ellipsis);
+                            }
+                            endReached = true;
+                            break;
+                        }
+                        index += length;
+                    }
+                }
+                yield return curr;
+            }
+        }
+
         private static bool IsTrailingSpace(CodePointSource s, int index) {
             while (index < s.Length) {
                 var (codePoint, length) = s.GetCodePoint(index);
@@ -362,5 +406,19 @@ namespace MLEM.Font {
             return true;
         }
 
+        internal readonly struct DecoratedCodePointSource {
+
+            public readonly CodePointSource Source;
+            public readonly GenericFont Font;
+            public readonly float ExtraWidth;
+
+            public DecoratedCodePointSource(CodePointSource source, GenericFont font, float extraWidth) {
+                this.Source = source;
+                this.Font = font;
+                this.ExtraWidth = extraWidth;
+            }
+
+        }
+
     }
 }

MLEM/Formatting/Token.cs

@@ -134,7 +134,6 @@ namespace MLEM.Formatting {
 
         /// <summary>
         /// Gets a list of rectangles that encompass this token's area.
-        /// Note that more than one rectangle is only returned if the string has been split.
         /// This can be used to invoke events when the mouse is hovered over the token, for example.
         /// </summary>
         /// <param name="stringPos">The position that the string is drawn at</param>

MLEM/Formatting/TokenizedString.cs

@@ -5,6 +5,7 @@ using System.Linq;
 using System.Text;
 using Microsoft.Xna.Framework;
 using Microsoft.Xna.Framework.Graphics;
+using MLEM.Extensions;
 using MLEM.Font;
 using MLEM.Formatting.Codes;
 using MLEM.Misc;
@@ -39,6 +40,7 @@ namespace MLEM.Formatting {
         public readonly Code[] AllCodes;
         private string modifiedString;
         private float initialInnerOffset;
+        private RectangleF area;
 
         internal TokenizedString(GenericFont font, TextAlignment alignment, string rawString, string strg, Token[] tokens) {
             this.RawString = rawString;
@@ -63,10 +65,16 @@ namespace MLEM.Formatting {
         /// <param name="scale">The scale to use for width measurements</param>
         /// <param name="alignment">The text alignment that should be used for width calculations</param>
         public void Split(GenericFont font, float width, float scale, TextAlignment alignment = TextAlignment.Left) {
-            // a split string has the same character count as the input string but with newline characters added
-            this.modifiedString = string.Join("\n", font.SplitStringSeparate(new CodePointSource(this.String), width, scale,
-                i => this.GetFontForIndex(font, i), i => this.GetSelfWidthForIndex(font, i)));
-            this.StoreModifiedSubstrings(font, alignment);
+            var index = 0;
+            var modified = new StringBuilder();
+            foreach (var part in GenericFont.SplitStringSeparate(this.AsDecoratedSources(font), width, scale)) {
+                var joined = string.Join("\n", part);
+                this.Tokens[index].ModifiedSubstring = joined;
+                modified.Append(joined);
+                index++;
+            }
+            this.modifiedString = modified.ToString();
+            this.Realign(font, alignment);
         }
 
         /// <summary>
@@ -80,9 +88,15 @@ namespace MLEM.Formatting {
         /// <param name="ellipsis">The characters to add to the end of the string if it is too long</param>
         /// <param name="alignment">The text alignment that should be used for width calculations</param>
         public void Truncate(GenericFont font, float width, float scale, string ellipsis = "", TextAlignment alignment = TextAlignment.Left) {
-            this.modifiedString = font.TruncateString(new CodePointSource(this.String), width, scale, false, ellipsis,
-                i => this.GetFontForIndex(font, i), i => this.GetSelfWidthForIndex(font, i)).ToString();
-            this.StoreModifiedSubstrings(font, alignment);
+            var index = 0;
+            var modified = new StringBuilder();
+            foreach (var part in GenericFont.TruncateString(this.AsDecoratedSources(font), width, scale, false, ellipsis)) {
+                this.Tokens[index].ModifiedSubstring = part.ToString();
+                modified.Append(part);
+                index++;
+            }
+            this.modifiedString = modified.ToString();
+            this.Realign(font, alignment);
         }
 
         /// <summary>
@@ -97,36 +111,54 @@ namespace MLEM.Formatting {
                 token.SplitDisplayString = token.DisplayString.Split('\n');
 
             // token areas and inner offsets
+            this.area = RectangleF.Empty;
             this.initialInnerOffset = this.GetInnerOffsetX(font, 0, 0, alignment);
             var innerOffset = new Vector2(this.initialInnerOffset, 0);
             for (var t = 0; t < this.Tokens.Length; t++) {
                 var token = this.Tokens[t];
                 var tokenFont = token.GetFont(font);
                 token.InnerOffsets = new float[token.SplitDisplayString.Length - 1];
-                var area = new List<RectangleF>();
+
+                var tokenArea = new List<RectangleF>();
+                var selfRect = new RectangleF(innerOffset, new Vector2(token.GetSelfWidth(tokenFont), tokenFont.LineHeight));
+                if (!selfRect.IsEmpty) {
+                    tokenArea.Add(selfRect);
+                    this.area = RectangleF.Union(this.area, selfRect);
+                    innerOffset.X += selfRect.Width;
+                }
                 for (var l = 0; l < token.SplitDisplayString.Length; l++) {
                     var size = tokenFont.MeasureString(token.SplitDisplayString[l], !this.EndsLater(t, l));
-                    if (l == 0)
-                        size.X += token.GetSelfWidth(tokenFont);
-
                     var rect = new RectangleF(innerOffset, size);
-                    if (!rect.IsEmpty)
-                        area.Add(rect);
+                    if (!rect.IsEmpty) {
+                        tokenArea.Add(rect);
+                        this.area = RectangleF.Union(this.area, rect);
+                    }
 
                     if (l < token.SplitDisplayString.Length - 1) {
                         innerOffset.X = token.InnerOffsets[l] = this.GetInnerOffsetX(font, t, l + 1, alignment);
-                        innerOffset.Y += font.LineHeight;
+                        innerOffset.Y += tokenFont.LineHeight;
                     } else {
                         innerOffset.X += size.X;
                     }
                 }
-                token.Area = area.ToArray();
+                token.Area = tokenArea.ToArray();
             }
         }
 
         /// <inheritdoc cref="GenericFont.MeasureString(string,bool)"/>
+        [Obsolete("Measure is deprecated. Use GetArea, which returns the string's total size measurement, instead.")]
         public Vector2 Measure(GenericFont font) {
-            return font.MeasureString(new CodePointSource(this.DisplayString), false, i => this.GetFontForIndex(font, i), i => this.GetSelfWidthForIndex(font, i));
+            return this.GetArea(Vector2.Zero, 1).Size;
+        }
+
+        /// <summary>
+        /// Measures the area that this entire tokenized string and all of its <see cref="Tokens"/> take up and returns it as a <see cref="RectangleF"/>.
+        /// </summary>
+        /// <param name="stringPos">The position that this string is being rendered at, which will offset the resulting <see cref="RectangleF"/>.</param>
+        /// <param name="scale">The scale that this string is being rendered with, which will scale the resulting <see cref="RectangleF"/>.</param>
+        /// <returns>The area that this tokenized string takes up.</returns>
+        public RectangleF GetArea(Vector2 stringPos, float scale) {
+            return new RectangleF(stringPos + this.area.Location * scale, this.area.Size * scale);
         }
 
         /// <summary>
@@ -185,48 +217,12 @@ namespace MLEM.Formatting {
                     // only split at a new line, not between tokens!
                     if (l < token.SplitDisplayString.Length - 1) {
                         innerOffset.X = token.InnerOffsets[l] * scale;
-                        innerOffset.Y += font.LineHeight * scale;
+                        innerOffset.Y += drawFont.LineHeight * scale;
                     }
                 }
             }
         }
 
-        private void StoreModifiedSubstrings(GenericFont font, TextAlignment alignment) {
-            if (this.Tokens.Length == 1) {
-                // skip substring logic for unformatted text
-                this.Tokens[0].ModifiedSubstring = this.modifiedString;
-            } else {
-                // this is basically a substring function that ignores added newlines for indexing
-                var index = 0;
-                var currToken = 0;
-                var splitIndex = 0;
-                var ret = new StringBuilder();
-                while (splitIndex < this.modifiedString.Length && currToken < this.Tokens.Length) {
-                    var token = this.Tokens[currToken];
-                    if (token.Substring.Length > 0) {
-                        ret.Append(this.modifiedString[splitIndex]);
-                        // if the current char is not an added newline, we simulate length increase
-                        if (this.modifiedString[splitIndex] != '\n' || this.String[index] == '\n')
-                            index++;
-                        splitIndex++;
-                    }
-                    // move on to the next token if we reached its end
-                    if (index >= token.Index + token.Substring.Length) {
-                        token.ModifiedSubstring = ret.ToString();
-                        ret.Clear();
-                        currToken++;
-                    }
-                }
-                // set additional token contents beyond our string in case we truncated
-                if (ret.Length > 0)
-                    this.Tokens[currToken++].ModifiedSubstring = ret.ToString();
-                while (currToken < this.Tokens.Length)
-                    this.Tokens[currToken++].ModifiedSubstring = string.Empty;
-            }
-
-            this.Realign(font, alignment);
-        }
-
         private float GetInnerOffsetX(GenericFont defaultFont, int tokenIndex, int lineIndex, TextAlignment alignment) {
             if (alignment > TextAlignment.Left) {
                 var token = this.Tokens[tokenIndex];
@@ -252,30 +248,17 @@ namespace MLEM.Formatting {
             return 0;
         }
 
-        private GenericFont GetFontForIndex(GenericFont font, int index) {
-            foreach (var token in this.Tokens) {
-                index -= token.Substring.Length;
-                if (index <= 0)
-                    return token.GetFont(font);
-            }
-            return null;
-        }
-
-        private float GetSelfWidthForIndex(GenericFont font, int index) {
-            foreach (var token in this.Tokens) {
-                if (index == token.Index)
-                    return token.GetSelfWidth(token.GetFont(font));
-                // if we're already beyond this token, any later tokens definitely won't match
-                if (token.Index > index)
-                    break;
-            }
-            return 0;
-        }
-
         private bool EndsLater(int tokenIndex, int lineIndex) {
             // if we're the last line in our line array, then we don't contain a line split, so the line ends in a later token
             return lineIndex >= this.Tokens[tokenIndex].SplitDisplayString.Length - 1 && tokenIndex < this.Tokens.Length - 1;
         }
 
+        private IEnumerable<GenericFont.DecoratedCodePointSource> AsDecoratedSources(GenericFont font) {
+            return this.Tokens.Select(t => {
+                var tokenFont = t.GetFont(font);
+                return new GenericFont.DecoratedCodePointSource(new CodePointSource(t.Substring), tokenFont, t.GetSelfWidth(tokenFont));
+            });
+        }
+
     }
 }