diff --git a/CHANGELOG.md b/CHANGELOG.md
index 933c2b0..fdae5cc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,7 +35,6 @@ Additions
 Improvements
 - Allow scrolling panels to contain other scrolling panels
 - Allow dropdowns to have scrolling panels
-- Don't unnecessarily set areas dirty when removing a root element from the ui
 - Improved Panel performance when adding and removing a lot of children
 
 Fixes
diff --git a/MLEM.Ui/Elements/Element.cs b/MLEM.Ui/Elements/Element.cs
index 842d788..3daa80d 100644
--- a/MLEM.Ui/Elements/Element.cs
+++ b/MLEM.Ui/Elements/Element.cs
@@ -591,6 +591,7 @@ namespace MLEM.Ui.Elements {
             element.AndChildren(e => e.AddedToUi(this.System, this.Root));
             this.OnChildAdded?.Invoke(this, element);
             this.SetSortedChildrenDirty();
+            element.SetAreaDirty();
             return element;
         }
 
@@ -1341,7 +1342,6 @@ namespace MLEM.Ui.Elements {
         protected internal virtual void AddedToUi(UiSystem system, RootElement root) {
             this.Root = root;
             this.System = system;
-            this.SetAreaDirty();
             this.OnAddedToUi?.Invoke(this);
             root?.InvokeOnElementAdded(this);
         }
diff --git a/MLEM.Ui/Elements/Panel.cs b/MLEM.Ui/Elements/Panel.cs
index 1d702eb..585dc9a 100644
--- a/MLEM.Ui/Elements/Panel.cs
+++ b/MLEM.Ui/Elements/Panel.cs
@@ -245,13 +245,12 @@ namespace MLEM.Ui.Elements {
         /// <inheritdoc />
         protected override void OnChildAreaDirty(Element child, bool grandchild) {
             base.OnChildAreaDirty(child, grandchild);
-            // we only need to scroll when a grandchild changes, since all of our children are forced
-            // to be auto-anchored and so will automatically propagate their changes up to us
-            if (grandchild) {
+            if (grandchild && !this.AreaDirty) {
+                // we only need to scroll when a grandchild changes, since all of our children are forced
+                // to be auto-anchored and so will automatically propagate their changes up to us
                 this.ScrollChildren();
                 // we also need to re-setup here in case the child is involved in a special GetTotalCoveredArea
-                if (!this.AreaDirty)
-                    this.ScrollSetup();
+                this.ScrollSetup();
             }
         }
 
@@ -358,17 +357,16 @@ namespace MLEM.Ui.Elements {
             if (!this.scrollOverflow)
                 return;
 
-            // we ignore false grandchildren so that the children of the scroll bar stay in place
-            var currentChildren = new HashSet<Element>(this.GetChildren(c => c != this.ScrollBar, true, true));
-
+            var currentChildren = new HashSet<Element>();
             // scroll all our children (and cache newly added ones)
-            foreach (var child in currentChildren) {
+            // we ignore false grandchildren so that the children of the scroll bar stay in place
+            foreach (var child in this.GetChildren(c => c != this.ScrollBar, true, true)) {
                 // if a child was newly added later, the last scroll offset was never applied
                 if (this.scrolledChildren.Add(child))
                     child.ScrollOffset.Y -= this.lastScrollOffset;
                 child.ScrollOffset.Y += (this.lastScrollOffset - this.ScrollBar.CurrentValue);
+                currentChildren.Add(child);
             }
-
             // remove cached scrolled children that aren't our children anymore
             this.scrolledChildren.IntersectWith(currentChildren);
 
diff --git a/MLEM.Ui/UiSystem.cs b/MLEM.Ui/UiSystem.cs
index 1dd2752..2e80d72 100644
--- a/MLEM.Ui/UiSystem.cs
+++ b/MLEM.Ui/UiSystem.cs
@@ -351,7 +351,10 @@ namespace MLEM.Ui {
                 return null;
             var root = new RootElement(name, element, this);
             this.rootElements.Add(root);
-            root.Element.AndChildren(e => e.AddedToUi(this, root));
+            root.Element.AndChildren(e => {
+                e.AddedToUi(this, root);
+                e.SetAreaDirty();
+            });
             this.OnRootAdded?.Invoke(root);
             root.InvokeOnAddedToUi(this);
             this.SortRoots();
@@ -368,8 +371,10 @@ namespace MLEM.Ui {
                 return;
             this.rootElements.Remove(root);
             this.Controls.SelectElement(root, null);
-            // we don't need to set dirty here since re-adding to a ui will cause dirty state anyway
-            root.Element.AndChildren(e => e.RemovedFromUi());
+            root.Element.AndChildren(e => {
+                e.RemovedFromUi();
+                e.SetAreaDirty();
+            });
             this.OnRootRemoved?.Invoke(root);
             root.InvokeOnRemovedFromUi(this);
         }