Improved text formatter tokenization performance

This reverts commit 7bfe44de07.

CHANGELOG.md

@@ -23,6 +23,7 @@ Additions
 
 Improvements
 - Stopped the text formatter throwing if a color can't be parsed
+- Improved text formatter tokenization performance
 
 Fixes
 - Fixed TextInput not working correctly when using surrogate pairs
@@ -32,6 +33,10 @@ Improvements
 - Allow scrolling panels to contain other scrolling panels
 - Allow dropdowns to have scrolling panels
 
+Fixes
+- Fixed panels updating their relevant children too much when the scroll bar is hidden
+- Fixed a stack overflow exception when a panel's scroll bar auto-hiding causes elements to gain height
+
 ### MLEM.Data
 Fixes
 - Fixed various exception types not being wrapped by ContentLoadExceptions when loading raw or JSON content

MLEM.Ui/Elements/Panel.cs

@@ -56,6 +56,7 @@ namespace MLEM.Ui.Elements {
 
         private readonly List<Element> relevantChildren = new List<Element>();
         private readonly HashSet<Element> scrolledChildren = new HashSet<Element>();
+        private readonly float[] scrollBarMaxHistory;
         private readonly bool scrollOverflow;
 
         private RenderTarget2D renderTarget;
@@ -96,6 +97,10 @@ namespace MLEM.Ui.Elements {
                     this.ScrollToElement(e);
                 };
                 this.AddChild(this.ScrollBar);
+
+                this.scrollBarMaxHistory = new float[3];
+                for (var i = 0; i < this.scrollBarMaxHistory.Length; i++)
+                    this.scrollBarMaxHistory[i] = -1;
             }
         }
 
@@ -110,7 +115,13 @@ namespace MLEM.Ui.Elements {
                         throw new NotSupportedException($"A panel that handles overflow can't contain non-automatic anchors ({child})");
                 }
             }
+
             base.ForceUpdateArea();
+            if (this.scrollOverflow) {
+                for (var i = 0; i < this.scrollBarMaxHistory.Length; i++)
+                    this.scrollBarMaxHistory[i] = -1;
+            }
+
             this.SetScrollBarStyle();
         }
 
@@ -271,10 +282,17 @@ namespace MLEM.Ui.Elements {
             }
 
             // the max value of the scroll bar is the amount of non-scaled pixels taken up by overflowing components
-            var scrollBarMax = (childrenHeight - this.ChildPaddedArea.Height) / this.Scale;
+            var scrollBarMax = Math.Max(0, (childrenHeight - this.ChildPaddedArea.Height) / this.Scale);
             if (!this.ScrollBar.MaxValue.Equals(scrollBarMax, Element.Epsilon)) {
-                this.ScrollBar.MaxValue = scrollBarMax;
+                // avoid a show/hide oscillation that occurs while updating our area with children that can lose height when the scroll bar is shown (like long paragraphs)
-                this.relevantChildrenDirty = true;
+                if (!this.scrollBarMaxHistory[0].Equals(this.scrollBarMaxHistory[2], Element.Epsilon) || !this.scrollBarMaxHistory[1].Equals(scrollBarMax, Element.Epsilon)) {
+                    this.scrollBarMaxHistory[0] = this.scrollBarMaxHistory[1];
+                    this.scrollBarMaxHistory[1] = this.scrollBarMaxHistory[2];
+                    this.scrollBarMaxHistory[2] = scrollBarMax;
+
+                    this.ScrollBar.MaxValue = scrollBarMax;
+                    this.relevantChildrenDirty = true;
+                }
             }
 
             // update child padding based on whether the scroll bar is visible

MLEM/Formatting/Codes/Code.cs

@@ -23,9 +23,9 @@ namespace MLEM.Formatting.Codes {
         public readonly Match Match;
         /// <summary>
         /// The tokens that this formatting code is a part of.
-        /// Note that this array only has multiple entries if additional tokens have to be started while this code is still applied.
+        /// Note that this collection only has multiple entries if additional tokens have to be started while this code is still applied.
         /// </summary>
-        public IList<Token> Tokens { get; internal set; }
+        public readonly List<Token> Tokens = new List<Token>();
 
         /// <summary>
         /// Creates a new formatting code based on a formatting code regex and its match.

MLEM/Formatting/TextFormatter.cs

@@ -156,11 +156,12 @@ namespace MLEM.Formatting {
             // resolve macros
             s = this.ResolveMacros(s);
             var tokens = new List<Token>();
-            var codes = new List<Code>();
+            var applied = new List<Code>();
+            var allCodes = new List<Code>();
             // add the formatting code right at the start of the string
             var firstCode = this.GetNextCode(s, 0, 0);
             if (firstCode != null)
-                codes.Add(firstCode);
+                applied.Add(firstCode);
             var index = 0;
             var rawIndex = 0;
             while (rawIndex < s.Length) {
@@ -168,24 +169,25 @@ namespace MLEM.Formatting {
                 // if we've reached the end of the string
                 if (next == null) {
                     var sub = s.Substring(rawIndex, s.Length - rawIndex);
-                    tokens.Add(new Token(codes.ToArray(), index, rawIndex, TextFormatter.StripFormatting(font, sub, codes), sub));
+                    tokens.Add(new Token(applied.ToArray(), index, rawIndex, TextFormatter.StripFormatting(font, sub, applied), sub));
                     break;
                 }
+                allCodes.Add(next);
 
                 // create a new token for the content up to the next code
                 var ret = s.Substring(rawIndex, next.Match.Index - rawIndex);
-                var strippedRet = TextFormatter.StripFormatting(font, ret, codes);
+                var strippedRet = TextFormatter.StripFormatting(font, ret, applied);
-                tokens.Add(new Token(codes.ToArray(), index, rawIndex, strippedRet, ret));
+                tokens.Add(new Token(applied.ToArray(), index, rawIndex, strippedRet, ret));
 
                 // move to the start of the next code
                 rawIndex = next.Match.Index;
                 index += strippedRet.Length;
 
                 // remove all codes that are incompatible with the next one and apply it
-                codes.RemoveAll(c => c.EndsHere(next) || next.EndsOther(c));
+                applied.RemoveAll(c => c.EndsHere(next) || next.EndsOther(c));
-                codes.Add(next);
+                applied.Add(next);
             }
-            return new TokenizedString(font, alignment, s, TextFormatter.StripFormatting(font, s, tokens.SelectMany(t => t.AppliedCodes)), tokens.ToArray());
+            return new TokenizedString(font, alignment, s, TextFormatter.StripFormatting(font, s, allCodes), tokens.ToArray(), allCodes.ToArray());
         }
 
         /// <summary>

MLEM/Formatting/Token.cs

@@ -50,6 +50,8 @@ namespace MLEM.Formatting {
             this.RawIndex = rawIndex;
             this.Substring = substring;
             this.RawSubstring = rawSubstring;
+            foreach (var code in appliedCodes)
+                code.Tokens.Add(this);
         }
 
         /// <summary>

MLEM/Formatting/TokenizedString.cs

@@ -1,11 +1,9 @@
 using System;
 using System.Collections.Generic;
-using System.Collections.ObjectModel;
 using System.Linq;
 using System.Text;
 using Microsoft.Xna.Framework;
 using Microsoft.Xna.Framework.Graphics;
-using MLEM.Extensions;
 using MLEM.Font;
 using MLEM.Formatting.Codes;
 using MLEM.Misc;
@@ -42,17 +40,11 @@ namespace MLEM.Formatting {
         private float initialInnerOffset;
         private RectangleF area;
 
-        internal TokenizedString(GenericFont font, TextAlignment alignment, string rawString, string strg, Token[] tokens) {
+        internal TokenizedString(GenericFont font, TextAlignment alignment, string rawString, string strg, Token[] tokens, Code[] allCodes) {
             this.RawString = rawString;
             this.String = strg;
             this.Tokens = tokens;
-
+            this.AllCodes = allCodes;
-            // since a code can be present in multiple tokens, we use Distinct here
-            this.AllCodes = tokens.SelectMany(t => t.AppliedCodes).Distinct().ToArray();
-            // TODO this can probably be optimized by keeping track of a code's tokens while tokenizing
-            foreach (var code in this.AllCodes)
-                code.Tokens = new ReadOnlyCollection<Token>(this.Tokens.Where(t => t.AppliedCodes.Contains(code)).ToList());
-
             this.Realign(font, alignment);
         }