Fix FSAA in UI and lower VRAM consumption

ogre2/include/ignition/rendering/ogre2/Ogre2RenderTarget.hh

@@ -115,8 +115,25 @@ namespace ignition
       /// \see Camera::SetShadowsNodeDefDirty
       public: void SetShadowsNodeDefDirty();
 
-      /// \brief Get a pointer to the ogre render target
-      public: virtual Ogre::RenderTarget *RenderTarget() const = 0;
+      /// \brief Get a pointer to the ogre render target containing
+      /// the results of the render
+      public: virtual Ogre::RenderTarget *RenderTarget() const;
+
+      /// \brief Returns true if this is a render window
+      /// TODO(anyone): this function should be virtual.
+      /// We didn't do it to preserve ABI.
+      /// Look in commit history for '#Ogre2IsRenderWindowABI' to
+      /// see changes made and revert
+      public: bool IsRenderWindow() const;
+
+      // Documentation inherited
+      public: unsigned int GLIdImpl() const;
+
+      /// \brief Destroy the render texture
+      protected: void DestroyTargetImpl();
+
+      /// \brief Build the render texture
+      protected: void BuildTargetImpl();
 
       /// \brief Get visibility mask for the viewport associated with this
       /// render target
@@ -128,12 +145,23 @@ namespace ignition
       /// \param[in] _mask Visibility mask
       public: virtual void SetVisibilityMask(uint32_t _mask);
 
+      /// \brief Deprecated. Use other overloads.
+      public: static IGN_DEPRECATED(5) void UpdateRenderPassChain(
+          Ogre::CompositorWorkspace *_workspace,
+          const std::string &_workspaceDefName,
+          const std::string &_baseNode, const std::string &_finalNode,
+          const std::vector<RenderPassPtr> &_renderPasses,
+          bool _recreateNodes);
+
       /// \brief Update the render pass chain
       public: static void UpdateRenderPassChain(
           Ogre::CompositorWorkspace *_workspace,
           const std::string &_workspaceDefName,
           const std::string &_baseNode, const std::string &_finalNode,
-          const std::vector<RenderPassPtr> &_renderPasses, bool _recreateNodes);
+          const std::vector<RenderPassPtr> &_renderPasses,
+          bool _recreateNodes,
+          Ogre::Texture *(*_ogreTextures)[2],
+          bool _isRenderWindow);
 
       /// \brief Update the background color
       protected: virtual void UpdateBackgroundColor();
@@ -168,6 +196,10 @@ namespace ignition
       /// \sa BaseRenderTarget::Rebuild()
       protected: void RebuildMaterial();
 
+      /// Calls Ogre2RenderTexture::SetOgreTexture if appropiate to ensure
+      /// Ogre2RenderTexture::ogreTexture always has our outputs
+      protected: void SyncOgreTextureVars();
+
       /// \brief Pointer to the internal ogre camera
       protected: Ogre::Camera *ogreCamera = nullptr;
 
@@ -187,7 +219,11 @@ namespace ignition
       /// \brief a material used by for the render target
       protected: MaterialPtr material;
 
-      /// \brief Helper class that applies the material to the render target
+      /// \brief Unused. Kept for ABI reasons.
+      ///
+      /// Just in case we set this value to
+      /// Ogre2RenderTargetPrivate::materialApplicator[0] which is what
+      /// most client applications may want.
       protected: Ogre2RenderTargetMaterialPtr materialApplicator;
 
       /// \brief Flag to indicate if the render target color has changed
@@ -229,7 +265,9 @@ namespace ignition
       // Documentation inherited
       public: virtual unsigned int GLId() const override;
 
-      // Documentation inherited.
+      // Documentation inherited
+      // TODO(anyone): this function should be removed.
+      // We didn't do it to preserve ABI.
       public: virtual Ogre::RenderTarget *RenderTarget() const override;
 
       // Documentation inherited.
@@ -241,8 +279,20 @@ namespace ignition
       /// \brief Build the render texture
       protected: virtual void BuildTarget();
 
-      /// \brief Pointer to the internal ogre render texture object
-      protected: Ogre::Texture *ogreTexture = nullptr;
+      /// \brief Do not call this function directly.
+      ///
+      /// It's used to keep ABI compatibility to sync ogreTexture
+      /// with the internal pointer from our base class.
+      /// \param[in] _ogreTexture texture from
+      /// Ogre2RenderTargetPrivate::ogreTexture[1]
+      public: void SetOgreTexture(Ogre::Texture *_ogreTexture);
+
+      /// \brief Unused. Kept for ABI reasons.
+      ///
+      /// Just in case we set this value to
+      /// Ogre2RenderTargetPrivate::ogreTexture[1] which is what most client
+      /// applications may want.
+      protected: IGN_DEPRECATED(5) Ogre::Texture * ogreTexture = nullptr;
 
       /// \brief Make scene our friend so it can create a ogre2 render texture
       private: friend class Ogre2Scene;
@@ -261,6 +311,12 @@ namespace ignition
       // Documentation inherited.
       public: virtual void Destroy() override;
 
+      // TODO(anyone): this function should be virtual.
+      // We didn't do it to preserve ABI.
+      // Looks in commit history for '#Ogre2IsRenderWindowABI' to
+      // see changes made and revert
+      public: bool IsRenderWindow() const;
+
       // Documentation inherited.
       public: virtual Ogre::RenderTarget *RenderTarget() const override;
 

ogre2/include/ignition/rendering/ogre2/Ogre2RenderTargetMaterial.hh

@@ -96,6 +96,9 @@ namespace ignition
                   Ogre::Material *_originalMaterial, uint16_t _lodIndex,
                   const Ogre::Renderable *_rend) override;
 
+      /// \brief Returns this->renderTarget == _renderTarget
+      public: bool IsSameRenderTarget(Ogre::RenderTarget *_renderTarget);
+
       /// \brief scene manager responsible for rendering
       private: Ogre::SceneManager *scene = nullptr;
 

ogre2/src/Ogre2DepthCamera.cc

@@ -104,7 +104,7 @@ class ignition::rendering::Ogre2DepthCameraPrivate
   public: Ogre::CompositorWorkspace *ogreCompositorWorkspace = nullptr;
 
   /// \brief Output texture with depth and color data
-  public: Ogre::TexturePtr ogreDepthTexture;
+  public: Ogre::TexturePtr ogreDepthTexture[2];
 
   /// \brief Dummy render texture for the depth data
   public: RenderTexturePtr depthTexture;
@@ -297,10 +297,13 @@ void Ogre2DepthCamera::Destroy()
   Ogre::CompositorManager2 *ogreCompMgr = ogreRoot->getCompositorManager2();
 
   // remove depth texture, material, compositor
-  if (this->dataPtr->ogreDepthTexture)
+  for( size_t i = 0u; i < 2u; ++i )
   {
-    Ogre::TextureManager::getSingleton().remove(
-        this->dataPtr->ogreDepthTexture->getName());
+    if (this->dataPtr->ogreDepthTexture[i])
+    {
+      Ogre::TextureManager::getSingleton().remove(
+            this->dataPtr->ogreDepthTexture[i]->getName());
+    }
   }
   if (this->dataPtr->ogreCompositorWorkspace)
   {
@@ -588,41 +591,11 @@ void Ogre2DepthCamera::CreateDepthTexture()
     this->dataPtr->ogreCompositorBaseNodeDef = baseNodeDefName;
     Ogre::CompositorNodeDef *baseNodeDef =
         ogreCompMgr->addNodeDefinition(baseNodeDefName);
-    Ogre::TextureDefinitionBase::TextureDefinition *rt0TexDef =
-        baseNodeDef->addTextureDefinition("rt0");
-    rt0TexDef->textureType = Ogre::TEX_TYPE_2D;
-    rt0TexDef->width = 0;
-    rt0TexDef->height = 0;
-    rt0TexDef->depth = 1;
-    rt0TexDef->numMipmaps = 0;
-    rt0TexDef->widthFactor = 1;
-    rt0TexDef->heightFactor = 1;
-    rt0TexDef->formatList = {Ogre::PF_FLOAT32_RGBA};
-    rt0TexDef->fsaa = 0;
-    rt0TexDef->uav = false;
-    rt0TexDef->automipmaps = false;
-    rt0TexDef->hwGammaWrite = Ogre::TextureDefinitionBase::BoolFalse;
-    rt0TexDef->depthBufferId = Ogre::DepthBuffer::POOL_INVALID;
-    rt0TexDef->depthBufferFormat = Ogre::PF_UNKNOWN;
-    rt0TexDef->fsaaExplicitResolve = false;
-
-    Ogre::TextureDefinitionBase::TextureDefinition *rt1TexDef =
-        baseNodeDef->addTextureDefinition("rt1");
-    rt1TexDef->textureType = Ogre::TEX_TYPE_2D;
-    rt1TexDef->width = 0;
-    rt1TexDef->height = 0;
-    rt1TexDef->depth = 1;
-    rt1TexDef->numMipmaps = 0;
-    rt1TexDef->widthFactor = 1;
-    rt1TexDef->heightFactor = 1;
-    rt1TexDef->formatList = {Ogre::PF_FLOAT32_RGBA};
-    rt1TexDef->fsaa = 0;
-    rt1TexDef->uav = false;
-    rt1TexDef->automipmaps = false;
-    rt1TexDef->hwGammaWrite = Ogre::TextureDefinitionBase::BoolFalse;
-    rt1TexDef->depthBufferId = Ogre::DepthBuffer::POOL_INVALID;
-    rt1TexDef->depthBufferFormat = Ogre::PF_UNKNOWN;
-    rt1TexDef->fsaaExplicitResolve = false;
+
+    baseNodeDef->addTextureSourceName(
+          "rt0", 0u, Ogre::TextureDefinitionBase::TEXTURE_INPUT);
+    baseNodeDef->addTextureSourceName(
+          "rt1", 1u, Ogre::TextureDefinitionBase::TEXTURE_INPUT);
 
     Ogre::TextureDefinitionBase::TextureDefinition *depthTexDef =
         baseNodeDef->addTextureDefinition("depthTexture");
@@ -853,10 +826,10 @@ void Ogre2DepthCamera::CreateDepthTexture()
     Ogre::CompositorNodeDef *finalNodeDef =
         ogreCompMgr->addNodeDefinition(finalNodeDefName);
 
-    // output texture
-    finalNodeDef->addTextureSourceName("rt_output", 0,
+    finalNodeDef->addTextureSourceName("rt_input", 0,
         Ogre::TextureDefinitionBase::TEXTURE_INPUT);
-    finalNodeDef->addTextureSourceName("rt_input", 1,
+    // output texture
+    finalNodeDef->addTextureSourceName("rt_output", 1,
         Ogre::TextureDefinitionBase::TEXTURE_INPUT);
 
     finalNodeDef->setNumTargetPass(1);
@@ -892,8 +865,9 @@ void Ogre2DepthCamera::CreateDepthTexture()
     Ogre::CompositorWorkspaceDef *workDef =
         ogreCompMgr->addWorkspaceDefinition(wsDefName);
 
-    workDef->connect(baseNodeDefName, 0,  finalNodeDefName, 1);
-    workDef->connectExternal(0, finalNodeDefName, 0);
+    workDef->connectExternal(0, baseNodeDefName, 0);
+    workDef->connectExternal(1, baseNodeDefName, 1);
+    workDef->connect(baseNodeDefName, finalNodeDefName);
   }
   Ogre::CompositorWorkspaceDef *wsDef =
       ogreCompMgr->getWorkspaceDefinition(wsDefName);
@@ -905,12 +879,19 @@ void Ogre2DepthCamera::CreateDepthTexture()
   }
 
   // create render texture - these textures pack the range data
-  this->dataPtr->ogreDepthTexture =
-    Ogre::TextureManager::getSingleton().createManual(
-    this->Name() + "_depth", "General", Ogre::TEX_TYPE_2D,
-    this->ImageWidth(), this->ImageHeight(), 1, 0,
-    Ogre::PF_FLOAT32_RGBA, Ogre::TU_RENDERTARGET,
-    0, false, 0, Ogre::BLANKSTRING, false, true);
+  for( size_t i = 0u; i < 2u; ++i )
+  {
+    this->dataPtr->ogreDepthTexture[i] =
+      Ogre::TextureManager::getSingleton().createManual(
+      this->Name() + "_depth" + std::to_string(i), "General",
+      Ogre::TEX_TYPE_2D, this->ImageWidth(), this->ImageHeight(), 1, 0,
+      Ogre::PF_FLOAT32_RGBA, Ogre::TU_RENDERTARGET,
+      0, false, 0, Ogre::BLANKSTRING, false, true);
+
+    Ogre::RenderTarget *rt =
+        this->dataPtr->ogreDepthTexture[i]->getBuffer()->getRenderTarget();
+    rt->setDepthBufferPool(Ogre::DepthBuffer::POOL_INVALID);
+  }
 
   CreateWorkspaceInstance();
 }
@@ -922,13 +903,19 @@ void Ogre2DepthCamera::CreateWorkspaceInstance()
   auto ogreRoot = engine->OgreRoot();
   Ogre::CompositorManager2 *ogreCompMgr = ogreRoot->getCompositorManager2();
 
-  Ogre::RenderTarget *rt =
-    this->dataPtr->ogreDepthTexture->getBuffer()->getRenderTarget();
+  Ogre::CompositorChannelVec externalTargets(2u);
+  for( size_t i = 0u; i < 2u; ++i )
+  {
+    externalTargets[i].target =
+        this->dataPtr->ogreDepthTexture[i]->getBuffer()->getRenderTarget();
+    externalTargets[i].textures.push_back(this->dataPtr->ogreDepthTexture[i]);
+  }
 
   // create compositor worksspace
   this->dataPtr->ogreCompositorWorkspace =
       ogreCompMgr->addWorkspace(this->scene->OgreSceneManager(),
-      rt, this->ogreCamera, this->dataPtr->ogreCompositorWorkspaceDef, false);
+      externalTargets, this->ogreCamera,
+      this->dataPtr->ogreCompositorWorkspaceDef, false);
 
   // add the listener
   Ogre::CompositorNode *node =
@@ -963,20 +950,35 @@ void Ogre2DepthCamera::Render()
 //////////////////////////////////////////////////
 void Ogre2DepthCamera::PreRender()
 {
-  if (!this->dataPtr->ogreDepthTexture)
+  if (!this->dataPtr->ogreDepthTexture[0])
     this->CreateDepthTexture();
 
   if (!this->dataPtr->ogreCompositorWorkspace)
     this->CreateWorkspaceInstance();
 
+  Ogre::Texture *rawDepthTextures[2] =
+  {
+    this->dataPtr->ogreDepthTexture[0].get(),
+    this->dataPtr->ogreDepthTexture[1].get()
+  };
+
   // update depth camera render passes
   Ogre2RenderTarget::UpdateRenderPassChain(
       this->dataPtr->ogreCompositorWorkspace,
       this->dataPtr->ogreCompositorWorkspaceDef,
       this->dataPtr->ogreCompositorBaseNodeDef,
       this->dataPtr->ogreCompositorFinalNodeDef,
       this->dataPtr->renderPasses,
-      this->dataPtr->renderPassDirty);
+      this->dataPtr->renderPassDirty,
+      &rawDepthTextures,
+      false);
+
+  if (rawDepthTextures[0] != this->dataPtr->ogreDepthTexture[0].get())
+  {
+    std::swap( this->dataPtr->ogreDepthTexture[0],
+               this->dataPtr->ogreDepthTexture[1] );
+  }
+
   for (auto &pass : this->dataPtr->renderPasses)
     pass->PreRender();
 
@@ -1028,7 +1030,7 @@ void Ogre2DepthCamera::PostRender()
         1, imageFormat, this->dataPtr->depthBuffer);
 
   // blit data from gpu to cpu
-  auto rt = this->dataPtr->ogreDepthTexture->getBuffer()->getRenderTarget();
+  auto rt = this->dataPtr->ogreDepthTexture[1]->getBuffer()->getRenderTarget();
   rt->copyContentsToMemory(dstBox, Ogre::RenderTarget::FB_AUTO);
 
   if (!this->dataPtr->depthImage)