ATItoTGA normal map generation

[Josh]

I have always loved the normal maps ATItoDOT3 produces. Here is an implementation using their algorithm grabbed from here:
https://github.com/mojocorp/ATINormalMapper/blob/master/TGAtoDOT3/TGAtoDOT3.cpp

 

#include "UltraEngine.h"

using namespace UltraEngine;

typedef struct _pixel
{
    uint8 red;
    uint8 blue;
    uint8 green;
    uint8 alpha;
} pixel;

inline void
TGAReadPixel(uint8* image, int width, int off, pixel* pix, int x, int y)
{
#ifdef _DEBUG
    if ((image == NULL) || (pix == NULL)) {
        //NmPrint("ERROR: NULL pointer passed to Readpixel!\n");
        exit(-1);
    }
#endif
    int idx = y * width * off + x * off;
    if (off > 0) {
        pix->red = image[idx];
    }
    if (off > 1) {
        pix->blue = image[idx + 1];
    }
    if (off > 2) {
        pix->green = image[idx + 2];
    }
}

int gWidth, gHeight;

void
WritePixel(uint8* image, int bpp, const pixel* pix, int x, int y)
{
    const int idx = (x + y * gWidth) * (bpp / 8);
    if (bpp >= 8) {
        image[idx + 0] = pix->blue;
    }
    if (bpp >= 16) {
        image[idx + 1] = pix->green;
    }
    if (bpp >= 24) {
        image[idx + 2] = pix->red;
    }
    if (bpp >= 32) {
        image[idx + 3] = pix->alpha;
    }
}

uint8
PackFloatInByte(float in)
{
    return (uint8)((in + 1.0f) / 2.0f * 255.0f);
}

void
SobelFilter(uint8* dstImage, uint8* srcImage, int width, int height, int bpp)
{
    gWidth = width;
    gHeight = height;
    pixel pix;
    for (int y = 0; y < height; y++) {
        for (int x = 0; x < width; x++) {
            // Do Y Sobel filter
            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x - 1 + width) % width, (y + 1) % height);
            float dY = ((float)pix.red) / 255.0f * -1.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, x % width, (y + 1) % height);
            dY += ((float)pix.red) / 255.0f * -2.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x + 1) % width, (y + 1) % height);
            dY += ((float)pix.red) / 255.0f * -1.0f;

            TGAReadPixel(
                srcImage, width, bpp / 8, &pix, (x - 1 + width) % width, (y - 1 + height) % height);
            dY += ((float)pix.red) / 255.0f * 1.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, x % width, (y - 1 + height) % height);
            dY += ((float)pix.red) / 255.0f * 2.0f;

            TGAReadPixel(
                srcImage, width, bpp / 8, &pix, (x + 1) % width, (y - 1 + height) % height);
            dY += ((float)pix.red) / 255.0f * 1.0f;

            // Do X Sobel filter
            TGAReadPixel(
                srcImage, width, bpp / 8, &pix, (x - 1 + width) % width, (y - 1 + height) % height);
            float dX = ((float)pix.red) / 255.0f * -1.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x - 1 + width) % width, y % height);
            dX += ((float)pix.red) / 255.0f * -2.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x - 1 + width) % width, (y + 1) % height);
            dX += ((float)pix.red) / 255.0f * -1.0f;

            TGAReadPixel(
                srcImage, width, bpp / 8, &pix, (x + 1) % width, (y - 1 + height) % height);
            dX += ((float)pix.red) / 255.0f * 1.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x + 1) % width, y % height);
            dX += ((float)pix.red) / 255.0f * 2.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x + 1) % width, (y + 1) % height);
            dX += ((float)pix.red) / 255.0f * 1.0f;

            // Cross Product of components of gradient reduces to
            float nX = -dX;
            float nY = -dY;
            float nZ = 1;

            // Normalize
            float oolen = 1.0f / ((float)sqrt(nX * nX + nY * nY + nZ * nZ));
            nX *= oolen;
            nY *= oolen;
            nZ *= oolen;

            pix.red = (uint8)PackFloatInByte(nX);
            pix.green = (uint8)PackFloatInByte(nY);
            pix.blue = (uint8)PackFloatInByte(nZ);
            pix.alpha = 255;
            pix.red = 128;

            WritePixel(dstImage, bpp, &pix, x, y);
        }
    }
}

int main(int argc, const char* argv[])
{
    auto src = LoadPixmap("https://github.com/UltraEngine/Documentation/raw/master/Assets/Materials/Brick/brickwall01.dds");
    if (src->format != TEXTURE_BGRA) src = src->Convert(TEXTURE_BGRA);
    
    auto dest = CreatePixmap(src->size.x, src->size.y, src->format);
    
    SobelFilter((uint8*)dest->pixels->Data(), (uint8*)src->pixels->Data(), src->size.x, src->size.y, 32);

    src->Save(GetPath(PATH_DESKTOP) + "/test.dds");
    dest->Save(GetPath(PATH_DESKTOP) + "/testDOT3.dds");

    return 0;
}

Input

Output

[Dreikblack]

Modified to make it get drag&dropped textures on .exe, added FITextureLoader support for .png and commented "pix.red = 128;" because it seems to just removing x offset

#include "UltraEngine.h"

using namespace UltraEngine;

typedef struct _pixel {
    uint8_t red;
    uint8_t blue;
    uint8_t green;
    uint8_t alpha;
} pixel;

inline void TGAReadPixel(uint8_t* image, int width, int off, pixel* pix, int x, int y) {
#ifdef _DEBUG
    if ((image == NULL) || (pix == NULL)) {
        //NmPrint("ERROR: NULL pointer passed to Readpixel!\n");
        exit(-1);
    }
#endif
    int idx = y * width * off + x * off;
    if (off > 0) {
        pix->red = image[idx];
    }
    if (off > 1) {
        pix->blue = image[idx + 1];
    }
    if (off > 2) {
        pix->green = image[idx + 2];
    }
}

int gWidth, gHeight;

void
WritePixel(uint8_t* image, int bpp, const pixel* pix, int x, int y) {
    const int idx = (x + y * gWidth) * (bpp / 8);
    if (bpp >= 8) {
        image[idx + 0] = pix->blue;
    }
    if (bpp >= 16) {
        image[idx + 1] = pix->green;
    }
    if (bpp >= 24) {
        image[idx + 2] = pix->red;
    }
    if (bpp >= 32) {
        image[idx + 3] = pix->alpha;
    }
}


uint8_t PackFloatInByte(float in) {
    return (uint8_t)((in + 1.0f) / 2.0f * 255.0f);
}


void SobelFilter(uint8_t* dstImage, uint8_t* srcImage, int width, int height, int bpp) {
    gWidth = width;
    gHeight = height;
    pixel pix;
    for (int y = 0; y < height; y++) {
        for (int x = 0; x < width; x++) {
            // Do Y Sobel filter
            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x - 1 + width) % width, (y + 1) % height);
            float dY = ((float)pix.red) / 255.0f * -1.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, x % width, (y + 1) % height);
            dY += ((float)pix.red) / 255.0f * -2.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x + 1) % width, (y + 1) % height);
            dY += ((float)pix.red) / 255.0f * -1.0f;

            TGAReadPixel(
                srcImage, width, bpp / 8, &pix, (x - 1 + width) % width, (y - 1 + height) % height);
            dY += ((float)pix.red) / 255.0f * 1.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, x % width, (y - 1 + height) % height);
            dY += ((float)pix.red) / 255.0f * 2.0f;

            TGAReadPixel(
                srcImage, width, bpp / 8, &pix, (x + 1) % width, (y - 1 + height) % height);
            dY += ((float)pix.red) / 255.0f * 1.0f;

            // Do X Sobel filter
            TGAReadPixel(
                srcImage, width, bpp / 8, &pix, (x - 1 + width) % width, (y - 1 + height) % height);
            float dX = ((float)pix.red) / 255.0f * -1.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x - 1 + width) % width, y % height);
            dX += ((float)pix.red) / 255.0f * -2.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x - 1 + width) % width, (y + 1) % height);
            dX += ((float)pix.red) / 255.0f * -1.0f;

            TGAReadPixel(
                srcImage, width, bpp / 8, &pix, (x + 1) % width, (y - 1 + height) % height);
            dX += ((float)pix.red) / 255.0f * 1.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x + 1) % width, y % height);
            dX += ((float)pix.red) / 255.0f * 2.0f;

            TGAReadPixel(srcImage, width, bpp / 8, &pix, (x + 1) % width, (y + 1) % height);
            dX += ((float)pix.red) / 255.0f * 1.0f;

            // Cross Product of components of gradient reduces to
            float nX = -dX;
            float nY = -dY;
            float nZ = 1;

            // Normalize
            float oolen = 1.0f / ((float)sqrt(nX * nX + nY * nY + nZ * nZ));
            nX *= oolen;
            nY *= oolen;
            nZ *= oolen;

            pix.red = (uint8_t)PackFloatInByte(nX);
            pix.green = (uint8_t)PackFloatInByte(nY);
            pix.blue = (uint8_t)PackFloatInByte(nZ);
            pix.alpha = 255;
            //pix.red = 128;

            WritePixel(dstImage, bpp, &pix, x, y);
        }
    }
}

int main(int argc, const char* argv[]) {
    auto textureLoaderPlugin = LoadPlugin("Plugins/FITextureLoader");
    for (int i = 1; i < argc; ++i) {
        Print(argv[i]);
        WString filePath = argv[i];
        auto src = LoadPixmap(filePath);
        if (!src) {
            continue;
        }
        if (src->format != TEXTURE_BGRA) {
            src = src->Convert(TEXTURE_BGRA);
        }
        auto dest = CreatePixmap(src->size.x, src->size.y, src->format);
        SobelFilter((uint8_t*)dest->pixels->Data(), (uint8_t*)src->pixels->Data(), src->size.x, src->size.y, 32);
        dest->Save(StripExt(filePath) + "_normal.dds");
    }
    return 0;
}

Diffuse2Normal.zip

[Josh]

20 minutes ago, Dreikblack said:

and commented "pix.red = 128;" because it seems to just removing x offset

Why did I do this??? 

[reepblue]

You need to generate the mipmaps and the compression needs to be BC5. Replace the bottom with this.

        auto dest = CreatePixmap(src->size.x, src->size.y, src->format);
        SobelFilter((uint8_t*)dest->pixels->Data(), (uint8_t*)src->pixels->Data(), src->size.x, src->size.y, 32);
        //dest->Save(StripExt(filePath) + "_normal.dds");

        //Build mipmap chain
        auto mipchain = dest->BuildMipchain();
        auto ddsformat = TEXTURE_BC5;
        auto blocksize = 4;
        auto count = mipchain.size();
        auto block = dest->blocksize;
        if (ddsformat != 0)
        {
            std::vector<shared_ptr<Pixmap>> mipchain2;
            for (int i = 0; i < count; i++)
            {
                if (mipchain[i]->size.x < blocksize or mipchain[i]->size.y < blocksize)
                    break;
                else
                {
                    mipchain2.push_back(mipchain[i]->Convert(ddsformat));
                    if (mipchain2[i] == NULL)
                    {
                        Print("Error: Failed to convert \"" + path + "\" to format " + String(ddsformat));
                    }
                }
            }
            mipchain = mipchain2;
        }

        SaveTexture(StripExt(filePath) + "_normal.dds", TEXTURE_2D, mipchain);

 

[Dreikblack]

@reepblue, i tried that, but result have no blue channel?

[reepblue]

That's how normal maps should look. The yellowing is due to BC5 compression. All normals should look like that as a game texture. 

[Dreikblack]

But how Z vector is determinated then? Also normal maps were blue everywhere where i read about them

[Josh]

If the material detects the normal map is using BC5 compression, a flag is set and the Z axis will be extracted from the pixel with this equation (or something close to it):

z = 1.0f - sqrt( x*x + y*y );

BC5 uses the same space as DXT5 but since it has only two channels it has more precision to store normals. This page has some more info:
https://www.ultraengine.com/learn/pbrmaterials

[Dreikblack]

X and Y vectors calculates in usual way (127 as 0 offset)? And Z is bigger if x & y vectors are closer to 127?

[Dreikblack]

And what if want to change z without changing horizontal and vertical offset?

[Dreikblack]

1 hour ago, Josh said:

z = 1.0f - sqrt( x*x + y*y );

Also it leads to different results for z when x & y are 255 and 0

[Josh]

vec4 nsample = texture(sampler2D(material.textureHandle[TEXTURE_NORMAL]), texcoords.xy);
n = nsample.xyz * 2.0f - 1.0f;

//Extract normal z
if ((materialFlags & MATERIAL_EXTRACTNORMALMAPZ) != 0) n.z = sqrt(max(0.0f, 1.0f - (n.x * n.x + n.y * n.y)));