diff --git a/dom/canvas/WebGLContext.cpp b/dom/canvas/WebGLContext.cpp
index 667ef4303731..669e131bd456 100644
--- a/dom/canvas/WebGLContext.cpp
+++ b/dom/canvas/WebGLContext.cpp
@@ -1126,7 +1126,7 @@ bool WebGLContext::PresentIntoXR(gl::SwapChain& swapChain,
 
 // Initialize a swap chain's surface factory given the desired surface type.
 void InitSwapChain(gl::GLContext& gl, gl::SwapChain& swapChain,
-                   const layers::TextureType consumerType) {
+                   const layers::TextureType consumerType, bool useAsync) {
   if (!swapChain.mFactory) {
     auto typedFactory = gl::SurfaceFactory::Create(&gl, consumerType);
     if (typedFactory) {
@@ -1138,6 +1138,11 @@ void InitSwapChain(gl::GLContext& gl, gl::SwapChain& swapChain,
     swapChain.mFactory = MakeUnique<gl::SurfaceFactory_Basic>(gl);
   }
   MOZ_ASSERT(swapChain.mFactory);
+  if (useAsync) {
+    // RemoteTextureMap will handle recycling any surfaces, so don't rely on the
+    // SwapChain's internal pooling.
+    swapChain.DisablePool();
+  }
 }
 
 void WebGLContext::Present(WebGLFramebuffer* const xrFb,
@@ -1158,7 +1163,10 @@ void WebGLContext::Present(WebGLFramebuffer* const xrFb,
     mResolvedDefaultFB = nullptr;
   }
 
-  InitSwapChain(*gl, *swapChain, consumerType);
+  bool useAsync = options.remoteTextureOwnerId.IsValid() &&
+                  options.remoteTextureId.IsValid();
+
+  InitSwapChain(*gl, *swapChain, consumerType, useAsync);
 
   bool valid =
       maybeFB ? PresentIntoXR(*swapChain, *maybeFB) : PresentInto(*swapChain);
@@ -1167,8 +1175,6 @@ void WebGLContext::Present(WebGLFramebuffer* const xrFb,
     return;
   }
 
-  bool useAsync = options.remoteTextureOwnerId.IsValid() &&
-                  options.remoteTextureId.IsValid();
   if (useAsync) {
     PushRemoteTexture(nullptr, *swapChain, swapChain->FrontBuffer(), options);
   }
@@ -1205,10 +1211,11 @@ bool WebGLContext::CopyToSwapChain(
   }
   gfx::IntSize size(info->width, info->height);
 
-  InitSwapChain(*gl, srcFb->mSwapChain, consumerType);
-
   bool useAsync = options.remoteTextureOwnerId.IsValid() &&
                   options.remoteTextureId.IsValid();
+
+  InitSwapChain(*gl, srcFb->mSwapChain, consumerType, useAsync);
+
   // If we're using async present and if there is no way to serialize surfaces,
   // then a readback is required to do the copy. In this case, there's no reason
   // to copy into a separate shared surface for the front buffer. Just directly
@@ -2429,7 +2436,7 @@ webgl::LinkActiveInfo GetLinkActiveInfo(
 
         ret.activeUniforms.push_back(std::move(info));
       }  // for i
-    }    // anon
+    }  // anon
 
     if (webgl2) {
       // -------------------------------------
@@ -2475,7 +2482,7 @@ webgl::LinkActiveInfo GetLinkActiveInfo(
 
           ret.activeUniformBlocks.push_back(std::move(info));
         }  // for i
-      }    // anon
+      }  // anon
 
       // -------------------------------------
       // active tf varyings
diff --git a/gfx/gl/GLScreenBuffer.cpp b/gfx/gl/GLScreenBuffer.cpp
index 9dda11b41a3a..6550698eb516 100644
--- a/gfx/gl/GLScreenBuffer.cpp
+++ b/gfx/gl/GLScreenBuffer.cpp
@@ -10,22 +10,13 @@
 #include "gfx2DGlue.h"
 #include "MozFramebuffer.h"
 #include "SharedSurface.h"
+#include "mozilla/gfx/BuildConstants.h"
 
 namespace mozilla::gl {
 
 // -
 // SwapChainPresenter
 
-// We need to apply pooling on Android because of the AndroidSurface slow
-// destructor bugs. They cause a noticeable performance hit. See bug
-// #1646073.
-static constexpr size_t kPoolSize =
-#if defined(MOZ_WIDGET_ANDROID)
-    4;
-#else
-    0;
-#endif
-
 UniquePtr<SwapChainPresenter> SwapChain::Acquire(
     const gfx::IntSize& size, const gfx::ColorSpace2 colorSpace) {
   MOZ_ASSERT(mFactory);
@@ -42,11 +33,9 @@ UniquePtr<SwapChainPresenter> SwapChain::Acquire(
     }
   }
 
-  // When mDestroyedCallback exists, recycling of SharedSurfaces is managed by
-  // the owner of the SwapChain by calling StoreRecycledSurface().
-  const auto poolSize = mDestroyedCallback ? 0 : kPoolSize;
-
-  if (!mPool.empty() && (!poolSize || mPool.size() == poolSize)) {
+  // When pooling is disabled, recycling of SharedSurfaces is managed by the
+  // owner of the SwapChain by calling StoreRecycledSurface().
+  if (!mPool.empty() && (!mPoolLimit || mPool.size() >= mPoolLimit)) {
     surf = mPool.front();
     mPool.pop();
   }
@@ -55,9 +44,11 @@ UniquePtr<SwapChainPresenter> SwapChain::Acquire(
     if (!uniquePtrSurf) return nullptr;
     surf.reset(uniquePtrSurf.release());
   }
-  mPool.push(surf);
-  while (mPool.size() > poolSize) {
-    mPool.pop();
+  if (mPoolLimit > 0) {
+    mPool.push(surf);
+    while (mPool.size() > mPoolLimit) {
+      mPool.pop();
+    }
   }
 
   auto ret = MakeUnique<SwapChainPresenter>(*this);
@@ -74,7 +65,10 @@ void SwapChain::ClearPool() {
 bool SwapChain::StoreRecycledSurface(
     const std::shared_ptr<SharedSurface>& surf) {
   MOZ_ASSERT(mFactory);
-  if (!mFactory || NS_WARN_IF(surf->mDesc.gl != mFactory->mDesc.gl)) {
+  // Don't allow external recycled surfaces if SwapChain is managing own pool.
+  MOZ_ASSERT(!mPoolLimit);
+  if (mPoolLimit > 0 || !mFactory ||
+      NS_WARN_IF(surf->mDesc.gl != mFactory->mDesc.gl)) {
     // Ensure we don't accidentally store an expired shared surface or from a
     // different context.
     return false;
@@ -130,7 +124,11 @@ GLuint SwapChainPresenter::Fb() const {
 // -
 // SwapChain
 
-SwapChain::SwapChain() = default;
+SwapChain::SwapChain()
+    :  // We need to apply pooling on Android because of the AndroidSurface slow
+       // destructor bugs. They cause a noticeable performance hit. See bug
+       // #1646073.
+      mPoolLimit(kIsAndroid ? 4 : 0) {}
 
 SwapChain::~SwapChain() {
   if (mPresenter) {
diff --git a/gfx/gl/GLScreenBuffer.h b/gfx/gl/GLScreenBuffer.h
index d2812497716b..ace3145f7ac3 100644
--- a/gfx/gl/GLScreenBuffer.h
+++ b/gfx/gl/GLScreenBuffer.h
@@ -55,6 +55,7 @@ class SwapChain final {
   UniquePtr<SurfaceFactory> mFactory;
 
  private:
+  size_t mPoolLimit;
   std::queue<std::shared_ptr<SharedSurface>> mPool;
   std::shared_ptr<SharedSurface> mFrontBuffer;
   std::function<void()> mDestroyedCallback;
@@ -69,6 +70,14 @@ class SwapChain final {
   SwapChain();
   virtual ~SwapChain();
 
+  void DisablePool() {
+    if (mPoolLimit) {
+      MOZ_ASSERT(mPool.empty());
+      mPool = {};
+      mPoolLimit = 0;
+    }
+  }
+
   void ClearPool();
   bool StoreRecycledSurface(const std::shared_ptr<SharedSurface>& surf);
   const auto& FrontBuffer() const { return mFrontBuffer; }