What is a stencil buffer?

It's been some time since I've done graphics programming, as part of my refresher I was recently doing some research on stencil buffers. This entry simply goes over what I've recalled so far.

A stencil buffer is like a mask used in computer graphics to control where certain parts of an image should be drawn. It works together with the depth buffer, which helps display 3D objects correctly. The stencil buffer stores a number for each pixel on the screen, and you can use these numbers to define different areas where you want specific things to happen, like showing reflections or shadows.

When you use a stencil buffer, you can set up rules to decide whether a pixel should be drawn or not, and you can also decide how to update the numbers in the stencil buffer as you draw more things on the screen.

Some cool things you can do with a stencil buffer include:

Clipping and masking: Show only specific parts of an object or scene, hiding the rest.

Reflections: Create reflections of objects on shiny surfaces by masking out where they should appear.

Shadows: Make shadows for objects by figuring out which parts of a scene are in shadow.

Decals: Add things like stickers or dirt to objects, using the stencil buffer to put them in the right place.

Key-points:

Stencil buffer can vary in depth, usually 8 bits per pixel.

Stencil buffer is an extension of the z-buffer.

Different tests can be configured, pass fail tests determine update/ignore/replace operations.

Used for clipping, masking, shadows, reflections.

Using a stencil buffer can make your graphics look more realistic, but it might also make your program use more memory and run slower, so be careful when using it and try to optimize its usage.

In Vulkan:

VkImageCreateInfo depthStencilImageInfo{};
depthStencilImageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
depthStencilImageInfo.imageType = VK_IMAGE_TYPE_2D;
depthStencilImageInfo.extent.width = swapChainExtent.width;
depthStencilImageInfo.extent.height = swapChainExtent.height;
depthStencilImageInfo.extent.depth = 1;
depthStencilImageInfo.mipLevels = 1;
depthStencilImageInfo.arrayLayers = 1;
depthStencilImageInfo.format = VK_FORMAT_D32_SFLOAT_S8_UINT;
depthStencilImageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
depthStencilImageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
depthStencilImageInfo.usage = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
depthStencilImageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
depthStencilImageInfo.samples = VK_SAMPLE_COUNT_1_BIT;

VkImageViewCreateInfo depthStencilImageViewInfo{};
depthStencilImageViewInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
depthStencilImageViewInfo.image = depthStencilImage;
depthStencilImageViewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D;
depthStencilImageViewInfo.format = VK_FORMAT_D32_SFLOAT_S8_UINT;
depthStencilImageViewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT;
depthStencilImageViewInfo.subresourceRange.baseMipLevel = 0;
depthStencilImageViewInfo.subresourceRange.levelCount = 1;
depthStencilImageViewInfo.subresourceRange.baseArrayLayer = 0;
depthStencilImageViewInfo.subresourceRange.layerCount = 1;

VkAttachmentDescription depthStencilAttachment{};
depthStencilAttachment.format = VK_FORMAT_D32_SFLOAT_S8_UINT;
depthStencilAttachment.samples = VK_SAMPLE_COUNT_1_BIT;
depthStencilAttachment.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
depthStencilAttachment.storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
depthStencilAttachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
depthStencilAttachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
depthStencilAttachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
depthStencilAttachment.finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;

VkAttachmentReference depthStencilAttachmentRef{};
depthStencilAttachmentRef.attachment = 2; // Index of depthStencilAttachment in the pAttachments array
depthStencilAttachmentRef.layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;

VkStencilOpState stencilOpState{};
stencilOpState.failOp = VK_STENCIL_OP_KEEP;
stencilOpState.passOp = VK_STENCIL_OP_KEEP;
stencilOpState.depthFailOp = VK_STENCIL_OP_KEEP;
stencilOpState.compareOp = VK_COMPARE_OP_ALWAYS;
stencilOpState.compareMask = 0xFF;
stencilOpState.writeMask = 0xFF;
stencilOpState.reference = 1;

VkPipelineDepthStencilStateCreateInfo depthStencilState{};
depthStencilState.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO;
depthStencilState.depthTestEnable = VK_TRUE;
depthStencilState.depthWriteEnable = VK_TRUE;
depthStencilState.depthCompareOp = VK_COMPARE_OP_LESS;
depthStencilState.depthBoundsTestEnable = VK_FALSE;
depthStencilState.minDepthBounds = 0.0f;
depthStencilState.maxDepthBounds = 1.0f;
depthStencilState.stencilTestEnable = VK_TRUE;
depthStencilState.front = stencilOpState;
depthStencilState.back = stencilOpState;

VkClearValue clearValues[3];
clearValues[0].color = {0.0f, 0.0f, 0.0f, 1.0f}; // Color attachment clear value
clearValues[1].depthStencil = {1.0f, 0};         // Depth attachment clear value
clearValues[2].depthStencil = {0.0f, 1};         // Stencil attachment clear value

VkRenderPassBeginInfo renderPassInfo{};
renderPassInfo.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
renderPassInfo.renderPass = renderPass;
renderPassInfo.framebuffer = swapChainFramebuffers[i];
renderPassInfo.renderArea.offset = {0, 0};
renderPassInfo.renderArea.extent = swapChainExtent;
renderPassInfo.clearValueCount = 3;
renderPassInfo.pClearValues = clearValues;