diff --git a/src/nvim/extmark.c b/src/nvim/extmark.c
index 0321a11b0f8ccc8cec1f90475cf33b3d9411676f..73a54729f84acdc2bbeee3e779b80555b4c5e343 100644
--- a/src/nvim/extmark.c
+++ b/src/nvim/extmark.c
@@ -150,7 +150,11 @@ void extmark_del(buf_T *buf, MarkTreeIter *itr, MTKey key, bool restore)
   }
 
   if (mt_decor_any(key)) {
-    buf_decor_remove(buf, key.pos.row, key2.pos.row, mt_decor(key), true);
+    if (mt_invalid(key)) {
+      decor_free(mt_decor(key));
+    } else {
+      buf_decor_remove(buf, key.pos.row, key2.pos.row, mt_decor(key), true);
+    }
   }
 
   // TODO(bfredl): delete it from current undo header, opportunistically?
@@ -393,10 +397,9 @@ void extmark_apply_undo(ExtmarkUndoObject undo_info, bool undo)
   } else if (undo_info.type == kExtmarkSavePos) {
     ExtmarkSavePos pos = undo_info.data.savepos;
     if (undo) {
-      if (pos.old_row >= 0) {
-        extmark_setraw(curbuf, pos.mark, pos.old_row, pos.old_col);
-      }
-      if (pos.invalidated) {
+      if (pos.old_row >= 0
+          && extmark_setraw(curbuf, pos.mark, pos.old_row, pos.old_col)
+          && pos.invalidated) {
         MarkTreeIter itr[1] = { 0 };
         MTKey mark = marktree_lookup(curbuf->b_marktree, pos.mark, itr);
         mt_itr_rawkey(itr).flags &= (uint16_t) ~MT_FLAG_INVALID;
diff --git a/test/functional/api/extmark_spec.lua b/test/functional/api/extmark_spec.lua
index 54f4aaab0335985f81c9392bf9e66b5391ba4fcd..19ad26fa861b1d80239789e7aaa0fc82aa829cb4 100644
--- a/test/functional/api/extmark_spec.lua
+++ b/test/functional/api/extmark_spec.lua
@@ -1712,6 +1712,10 @@ describe('API/extmarks', function()
           aaa bbb ccc                         |*2
                                               |
     ]])
+    -- decor is not removed twice
+    command('d3')
+    api.nvim_buf_del_extmark(0, ns, 1)
+    command('silent undo')
     -- mark is deleted with undo_restore == false
     set_extmark(ns, 1, 0, 0, { invalidate = true, undo_restore = false, sign_text = 'S1' })
     set_extmark(ns, 2, 1, 0, { invalidate = true, undo_restore = false, sign_text = 'S2' })