[Metal] matching depth stencil state

cocos/renderer/CCRenderer.cpp

@@ -199,7 +199,6 @@ void Renderer::init()
     auto device = backend::Device::getInstance();
     _commandBuffer = device->newCommandBuffer();
     // @MTL: the depth stencil flags must same render target and _depthStencilDescriptor
-    // TODO: can use one?
     _depthStencilDescriptor.depthStencilFlags = TargetBufferFlags::DEPTH_AND_STENCIL;
     _defaultRT = device->newDefaultRenderTarget(TargetBufferFlags::COLOR | _depthStencilDescriptor.depthStencilFlags);
     
@@ -352,7 +351,6 @@ void Renderer::visitRenderQueue(RenderQueue& queue)
     //Process Global-Z > 0 Queue
     //
     doVisitRenderQueue(queue.getSubQueue(RenderQueue::QUEUE_GROUP::GLOBALZ_POS));
-
 }
 
 void Renderer::doVisitRenderQueue(const std::vector<RenderCommand*>& renderCommands)

cocos/renderer/backend/DepthStencilState.cpp

@@ -40,6 +40,10 @@ bool StencilDescriptor::operator==(const StencilDescriptor &rhs) const
 DepthStencilState::~DepthStencilState()
 {}
 
+bool DepthStencilState::isEnabled() const  {
+    return bitmask::any(_depthStencilInfo.depthStencilFlags, TargetBufferFlags::DEPTH_AND_STENCIL);
+}
+
 void DepthStencilState::update(const DepthStencilDescriptor& descriptor) {
     _depthStencilInfo = descriptor;
     _isBackFrontStencilEqual = descriptor.backFaceStencil == descriptor.frontFaceStencil;

cocos/renderer/backend/DepthStencilState.h

@@ -74,7 +74,7 @@ class DepthStencilState : public cocos2d::Ref
 public:
     virtual void update(const DepthStencilDescriptor& descriptor);
     const DepthStencilDescriptor& getDepthStencilInfo()const { return _depthStencilInfo; }
-    bool isEnabled() const { return bitmask::any(_depthStencilInfo.depthStencilFlags, TargetBufferFlags::DEPTH_AND_STENCIL); }
+    bool isEnabled() const;
 protected:
     /**
      * @param descriptor Specifies depth and stencil descriptor.

cocos/renderer/backend/metal/CommandBufferMTL.h

@@ -226,6 +226,7 @@ private:
     dispatch_semaphore_t _frameBoundarySemaphore;
     const RenderTarget* _currentRenderTarget = nil; // weak ref
     RenderPassParams _currentRenderPassParams;
+    TargetBufferFlags _currentRenderTargetFlags = TargetBufferFlags::NONE;
     NSAutoreleasePool* _autoReleasePool = nil;
     
     std::vector<std::pair<TextureBackend*,std::function<void(const PixelBufferDescriptor&)>>> _captureCallbacks;

cocos/renderer/backend/metal/CommandBufferMTL.mm

@@ -200,15 +200,17 @@ void CommandBufferMTL::beginFrame()
 
 id<MTLRenderCommandEncoder> CommandBufferMTL::getRenderCommandEncoder(const RenderTarget* renderTarget, const RenderPassParams& renderPassParams)
 {
-    if(_mtlRenderEncoder != nil && _currentRenderPassParams == renderPassParams && _currentRenderTarget == renderTarget)
+    if(_mtlRenderEncoder != nil &&
+       _currentRenderPassParams == renderPassParams &&
+       _currentRenderTarget == renderTarget &&
+       _currentRenderTargetFlags == renderTarget->getTargetFlags())
     {
         return _mtlRenderEncoder;
     }
-    else
+    
-    {
+    _currentRenderTarget = renderTarget;
-        _currentRenderTarget = renderTarget;
+    _currentRenderPassParams = renderPassParams;
-        _currentRenderPassParams = renderPassParams;
+    _currentRenderTargetFlags = renderTarget->getTargetFlags();
-    }
     
     if(_mtlRenderEncoder != nil)
     {

cocos/renderer/backend/metal/RenderPipelineMTL.h

@@ -62,7 +62,7 @@ private:
     void setBlendState(MTLRenderPipelineColorAttachmentDescriptor*, const BlendDescriptor&);
     void setShaderModules(const PipelineDescriptor&);
     void setBlendStateAndFormat(const BlendDescriptor&);
-    void getAttachmentFormat(const RenderTarget* renderTarget, PixelFormat colorAttachmentsFormat[MAX_COLOR_ATTCHMENT], PixelFormat&, PixelFormat&);
+    void chooseAttachmentFormat(const RenderTarget* renderTarget, PixelFormat colorAttachmentsFormat[MAX_COLOR_ATTCHMENT], PixelFormat&, PixelFormat&);
     
     id<MTLRenderPipelineState> _mtlRenderPipelineState = nil;
     id<MTLDevice> _mtlDevice = nil;

cocos/renderer/backend/metal/RenderPipelineMTL.mm

@@ -183,7 +183,7 @@ void RenderPipelineMTL::update(const RenderTarget* renderTarget, const PipelineD
     
     memset(&hashMe, 0, sizeof(hashMe));
     const auto& blendDescriptor = pipelineDescirptor.blendDescriptor;
-    getAttachmentFormat(renderTarget, _colorAttachmentsFormat, _depthAttachmentFormat, _stencilAttachmentFormat);
+    chooseAttachmentFormat(renderTarget, _colorAttachmentsFormat, _depthAttachmentFormat, _stencilAttachmentFormat);
     auto program = static_cast<ProgramMTL*>(pipelineDescirptor.programState->getProgram());
     hashMe.vertexShaderHash = program->getVertexShader()->getHashValue();
     hashMe.fragmentShaderHash = program->getFragmentShader()->getHashValue();
@@ -292,33 +292,21 @@ void RenderPipelineMTL::setShaderModules(const PipelineDescriptor& descriptor)
     _mtlRenderPipelineDescriptor.fragmentFunction = fragShaderModule->getMTLFunction();
 }
 
-void RenderPipelineMTL::getAttachmentFormat(const RenderTarget* renderTarget,
+void RenderPipelineMTL::chooseAttachmentFormat(const RenderTarget* renderTarget,
                                             PixelFormat colorAttachmentsFormat[MAX_COLOR_ATTCHMENT],
                                             PixelFormat& depthFormat,
                                             PixelFormat& stencilFormat)
 {
+    // Sets color attachment format
     auto rtMTL = static_cast<const RenderTargetMTL*>(renderTarget);
     auto rtflags = rtMTL->getTargetFlags();
     for(auto i = 0; i < MAX_COLOR_ATTCHMENT; ++i) {
-        if (bitmask::any(rtflags, getMRTColorFlag(i)))
+        colorAttachmentsFormat[i] = bitmask::any(rtflags, getMRTColorFlag(i)) ? rtMTL->getColorAttachmentPixelFormat(i) : PixelFormat::NONE;
-        {
-            colorAttachmentsFormat[i] = rtMTL->getColorAttachmentPixelFormat(i);
-        }
-        else
-        {
-            colorAttachmentsFormat[i] = PixelFormat::NONE;
-        }
     }
     
-    if (bitmask::any(rtflags, RenderTargetFlag::DEPTH_AND_STENCIL))
+    // Sets depth and stencil attachment format, match RenderTargetMTL::applyRenderPassAttachments
-    {
+    depthFormat = bitmask::any(rtflags, RenderTargetFlag::DEPTH) ? rtMTL->getDepthAttachmentPixelFormat() : PixelFormat::NONE;
-        depthFormat = rtMTL->getDepthAttachmentPixelFormat();
+    stencilFormat = bitmask::any(rtflags, RenderTargetFlag::STENCIL) ? rtMTL->getStencilAttachmentPixelFormat() : PixelFormat::NONE;
-        stencilFormat =rtMTL->getStencilAttachmentPixelFormat();
-    }
-    else
-    {
-        depthFormat = stencilFormat = PixelFormat::NONE;
-    }
 }
 
 void RenderPipelineMTL::setBlendStateAndFormat(const BlendDescriptor& blendDescriptor)

cocos/renderer/backend/metal/RenderTargetMTL.mm

@@ -96,7 +96,8 @@ void RenderTargetMTL::applyRenderPassAttachments(const RenderPassParams& params,
 #endif
     }
     
-    if(bitmask::any(this->_flags, RenderTargetFlag::DEPTH_AND_STENCIL)) {
+    // Sets descriptor depth and stencil params, should match RenderTargetMTL::chooseAttachmentFormat
+    if(bitmask::any(this->_flags, RenderTargetFlag::DEPTH)) {
         auto depthAttachment = getDepthAttachment();
         if(depthAttachment){
             descriptor.depthAttachment.texture = depthAttachment.texture;
@@ -106,7 +107,9 @@ void RenderTargetMTL::applyRenderPassAttachments(const RenderPassParams& params,
             descriptor.depthAttachment.storeAction = getStoreAction(params, TargetBufferFlags::DEPTH);
             descriptor.depthAttachment.clearDepth = params.clearDepthValue;
         }
+    }
     
+    if(bitmask::any(this->_flags, RenderTargetFlag::STENCIL)) {
         auto stencilAttachment = getStencilAttachment();
         if(stencilAttachment) {
             descriptor.stencilAttachment.texture = stencilAttachment.texture;
@@ -174,7 +177,7 @@ PixelFormat RenderTargetMTL::getColorAttachmentPixelFormat(int index) const
 
 PixelFormat RenderTargetMTL::getDepthAttachmentPixelFormat() const
 { // FIXME: engine-x only support D24S8
-    if(bitmask::any(_flags, TargetBufferFlags::DEPTH_AND_STENCIL)) {
+    if(bitmask::any(_flags, TargetBufferFlags::DEPTH)) {
         if(isDefaultRenderTarget() || !_depth)
             return PixelFormat::D24S8;
         return _depth.texture->getTextureFormat();
@@ -184,7 +187,7 @@ PixelFormat RenderTargetMTL::getDepthAttachmentPixelFormat() const
 
 PixelFormat RenderTargetMTL::getStencilAttachmentPixelFormat() const
 { // FIXME: engine-x only support D24S8
-    if(bitmask::any(_flags, TargetBufferFlags::DEPTH_AND_STENCIL)) {
+    if(bitmask::any(_flags, TargetBufferFlags::STENCIL)) {
         if(isDefaultRenderTarget() || !_stencil)
             return PixelFormat::D24S8;
         return _stencil.texture->getTextureFormat();