OpenAI Image Generation - Final Version

[Josh]

The OpenAI image generation plugin stopped working, doubtlessly due to changes in their web API, and I am not going to bother trying to fix it. I am uploading this here as a possible resource to learn from.

OpenAI.lua

local extension = {}

function extension.hook(event, extension)
    
    if event.id == EVENT_WINDOWCLOSE then
        if event.source == extension.window then
            extension.window:SetHidden(true)
            program.window:Activate()
        end
    
    elseif event.id == EVENT_INITASSETEDITOR then

        local asseteditor = AssetEditor(event.source)
        local menu = asseteditor.texturemenu:FindChild("Tools", false)
        if menu == nil then
            menu = CreateMenu("Tools", asseteditor.texturemenu)
        else
            CreateMenu("", menu)-- add divider
        end
        local submenu = menu:FindChild("OpenAI", false)
        if submenu == nil then
            submenu = CreateMenu("OpenAI", menu)
        end
        local item = CreateMenu("Make Seamless", submenu)
        ListenEvent(EVENT_WIDGETACTION, item, extension.hook, extension)
        item = CreateMenu("Make Variation", submenu)
        ListenEvent(EVENT_WIDGETACTION, item, extension.hook, extension)
        
    elseif event.id == EVENT_WIDGETACTION then
        
        local collect = false
        
        if event.source == extension.menuitem then

            extension.window:SetHidden(false)
            extension.window:Activate()

        elseif event.source == extension.button then

            local apikey = extension.apikeyfield:GetText()
            if apikey == "" then
                Notify("API key is missing", "Error", true)
                return false
            end

            local dir = GetPath(PATH_PROGRAMDATA).."/Ultra Engine/Temp/OpenAI"
            CreateDir(dir, true)
            local name = Uuid()..".png"
            local savepath = dir.."/"..name

            local prompt = extension.promptfield:GetText()
            prompt = Replace(prompt, "\"", "")

            local sz = 1024--512
            --[[local i = extension.sizefield:GetSelectedItem()
            if i == 1 then
                sz = 256
            elseif i == 3 then
                sz = 1024
            end]]

            --Save settings
            if type(program.settings.extensions) ~= "userdata" then
                program.settings.extensions = {}
            end
            if type(program.settings.extensions.openai) ~= "userdata" then
                program.settings.extensions.openai = {}
            end
            program.settings.extensions.openai.apikey = apikey
            program.settings.extensions.openai.prompt = prompt
            program.settings.extensions.openai.size = {}
            program.settings.extensions.openai.size[1] = sz
            program.settings.extensions.openai.size[2] = sz            
            if extension.detailbutton:GetState() == WIDGETSTATE_SELECTED then
                program.settings.extensions.openai.detail = true
            else
                program.settings.extensions.openai.detail = false
            end

            local quality = ""
            if extension.detailbutton:GetState() == WIDGETSTATE_SELECTED then
                quality = "\\\"quality\\\":\\\"hd\\\", "
            end
            local cmd = "curl https://api.openai.com/v1/images/generations -H \"Content-Type: application/json\" -H \"Authorization: Bearer "..apikey.."\" -d \"{\\\"model\\\": \\\"dall-e-3\\\", "..quality.."\\\"prompt\\\":\\\""..prompt.."\\\", \\\"n\\\":1, \\\"size\\\":\\\""..tostring(sz).."x"..tostring(sz).."\\\"}\""
            --Print(cmd)
            local output = CreateBufferStream()
            local result = Command(cmd, output)
            if result ~= 0 then
                Print("Error: curl returned error "..tostring(result))
            end

            local success
            local t = LoadTable(output)
            if type(t) == "table" then
                if type(t["error"]) == "table" then
                    if type(t["error"]["message"]) == "string" then
                        local msg = t["error"]["message"]
                        Print("Error: "..msg)
                        return false
                    end
                elseif type(t["data"]) == "table" then
                    if type(t["data"][1]["url"]) == "string" then
                        local path = t["data"][1]["url"]
                        success = DownloadFile(path, savepath)                        
                    end
                end
            end

            if not success then
                Print("Error: Image generation failed")
                return false
            end

            local pixmap = LoadPixmap(savepath)
            if pixmap ~= nil then
                local tex = CreateTexture(TEXTURE_2D, pixmap.size.x, pixmap.size.y, pixmap.format, {pixmap})
                tex.name = "New texture"
                local asseteditor = program:OpenAsset(tex)
                if asseteditor ~= nil then
                    asseteditor:Modify()
                else
                    Print("Error: Failed to open texture in asset editor.");
                end
            end
            
        else

            local menu = Widget(event.source)
            if menu ~= nil then
                if menu.text == "Make Seamless" then

                    local apikey = extension.apikeyfield:GetText()
                    if apikey == "" then
                        Notify("API key is missing", "Error", true)
                        return false
                    end
        
                    local texture, assetwindow                
                    local n

                    --Find the texture to operate on
                    local interface = menu:GetInterface()
                    local window = interface:GetWindow()
                    for n = 1, #program.asseteditors do                        
                        if program.asseteditors[n].window == window then
                            assetwindow = program.asseteditors[n]                          
                            texture = Texture(assetwindow.asset)
                            break
                        end
                    end

                    if assetwindow == nil then
                        Print("Error: Could not find asset window.")-- this should never happen
                        return false
                    end

                    if texture == nil then
                        Print("Error: Could not find texture.")-- this should never happen
                        return false
                    end

                    local info = TextureInfo(texture.extra)
                    if info == nil then
                        Print("Error: Could not find texture information.")-- this should never happen
                        return false
                    end

                    local base = info.mipchain[1]

                    --Verify dimensions
                    if base.size.x ~= base.size.y or (base.size.x ~= 256 and base.size.x ~= 512 and base.size.x ~= 1024) then
                        Print("Error: Image dimensions must be 256 x 256, 512 x 512, or 1024 x 1024")
                        return false
                    end
                    
                    --Convert pixel format
                    if base.format ~= TEXTURE_RGBA then
                        base = base:Convert(TEXTURE_RGBA)
                    end
                    
                    --Create offset image
                    local pixmap = CreatePixmap(base.size.x, base.size.y, TEXTURE_RGBA)
                    local hs = base.size.x / 2
                    base:CopyRect(0, 0, hs, hs, pixmap, hs, hs)
                    base:CopyRect(hs, 0, hs, hs, pixmap, 0, hs)
                    base:CopyRect(0, hs, hs, hs, pixmap, hs, 0)
                    base:CopyRect(hs, hs, hs, hs, pixmap, 0, 0)

                    --Temp directory
                    local savedir = GetPath(PATH_PROGRAMDATA).."/Ultra Engine/Temp/OpenAI/"
                    if FileType(savedir) ~= 2 then
                        if not CreateDir(savedir, true) then
                            Print("Error: Failed to create temporary folder")
                            return false
                        end
                    end

                    --Create mask image
                    local border = base.size.x / 8
                    local maskpath = savedir.."/maskimage_"..tostring(base.size.x).."_"..tostring(border)..".png"
                    if FileType(maskpath) ~= 1 then
                        local mask = CreatePixmap(base.size.x, base.size.y, TEXTURE_RGBA)                        
                        local hb = border / 2
                        local x, y, rgba, l
                        mask:Fill(255,255,255,255)
                        local maskcolor = Rgba(255,255,255,0)
                        for x = base.size.x / 2 - hb, base.size.x / 2 + hb do
                            for y = 0, base.size.y - 1 do
                                mask:WritePixel(x, y, maskcolor)
                                mask:WritePixel(y, x, maskcolor)
                            end
                        end
                        if not mask:Save(maskpath) then
                            return false-- this should never happen
                        end
                    end

                    --Save image as PNG                                        
                    local imagepath = savedir.."/"..Uuid()..".png"
                    if not pixmap:Save(imagepath) then
                        return false-- this should never happen
                    end
                    
                    local prompt = ""
                    local savepath = savedir.."/"..Uuid()..".png"
                    local outputpath = savedir.."/"..Uuid()..".png"
                    local cmd = "curl https://api.openai.com/v1/images/edits -H \"Authorization: Bearer "..apikey.."\" -F image=\"@"..imagepath.."\" -F mask=\"@"..maskpath.."\" -F model=\"dall-e-2\" -F prompt=\"tiling seamless\" -F n=1 -F size=\""..tostring(base.size.x).."x"..tostring(base.size.y).."\""
                    local output = CreateBufferStream()
                    local result = Command(cmd, output)
                    if result ~= 0 then
                        Print("Error: curl returned error "..tostring(result))
                    end

                    local result
                    local t = LoadTable(output)
                    if type(t) == "table" then
                        if type(t["error"]) == "table" then
                            if type(t["error"]["message"]) == "string" then
                                local msg = t["error"]["message"]
                                Print("Error: "..msg)
                                return false
                            end
                        elseif type(t["data"]) == "table" then
                            if type(t["data"][1]["url"]) == "string" then
                                local path = t["data"][1]["url"]
                                DownloadFile(path, outputpath)
                                result = LoadPixmap(outputpath)
                            end
                        end
                    end
                    
                    if result == nil then
                        return false
                    end

                    --Load output pixmap
                    base = CreatePixmap(base.size.x, base.size.y, TEXTURE_RGBA)                    
                    result:CopyRect(0, 0, hs, hs, base, hs, hs)
                    result:CopyRect(hs, 0, hs, hs, base, 0, hs)
                    result:CopyRect(0, hs, hs, hs, base, hs, 0)
                    result:CopyRect(hs, hs, hs, hs, base, 0, 0)

                    local format = info.mipchain[1].format
                    for n = 1, #info.mipchain do
                        if n > 1 then
                            base = base:Resize(base.size.x / 2, base.size.y / 2)
                        end
                        local intermediate = base
                        if intermediate.format ~= format then
                            intermediate = intermediate:Convert(format)
                        end
                        intermediate:CopyRect(0, 0, intermediate.size.x, intermediate.size.y, info.mipchain[n], 0, 0)
                        texture:SetPixels(intermediate, n)                      
                    end
                    assetwindow.texturepanel:Redraw()
                    assetwindow:Modify()
                    
                    collect = true

                elseif menu.text == "Make Variation" then

                    local apikey = extension.apikeyfield:GetText()
                    if apikey == "" then
                        Notify("API key is missing", "Error", true)
                        return false
                    end
        
                    local texture, assetwindow                
                    local n

                    --Find the texture to operate on
                    local interface = menu:GetInterface()
                    local window = interface:GetWindow()
                    for n = 1, #program.asseteditors do                        
                        if program.asseteditors[n].window == window then
                            assetwindow = program.asseteditors[n]                          
                            texture = Texture(assetwindow.asset)
                            break
                        end
                    end

                    if assetwindow == nil then
                        Print("Error: Could not find asset window.")-- this should never happen
                        return false
                    end

                    if texture == nil then
                        Print("Error: Could not find texture.")-- this should never happen
                        return false
                    end

                    local info = TextureInfo(texture.extra)
                    if info == nil then
                        Print("Error: Could not find texture information.")-- this should never happen
                        return false
                    end

                    local base = info.mipchain[1]

                    --Verify dimensions
                    if base.size.x ~= base.size.y or (base.size.x ~= 256 and base.size.x ~= 512 and base.size.x ~= 1024) then
                        Print("Error: Image dimensions must be 256 x 256, 512 x 512, or 1024 x 1024")
                        return false
                    end
                    
                    --Convert pixel format
                    if base.format ~= TEXTURE_RGBA then
                        base = base:Convert(TEXTURE_RGBA)
                    end

                    --Temp directory
                    local savedir = GetPath(PATH_PROGRAMDATA).."/Ultra Engine/Temp/OpenAI/"
                    if FileType(savedir) ~= 2 then
                        if not CreateDir(savedir, true) then
                            Print("Error: Failed to create temporary folder")
                            return false
                        end
                    end

                    --Save image as PNG                                        
                    local imagepath = savedir.."/"..Uuid()..".png"
                    if not base:Save(imagepath) then
                        return false-- this should never happen
                    end
                    
                    local prompt = ""
                    local savepath = savedir.."/"..Uuid()..".png"
                    local outputpath = savedir.."/"..Uuid()..".png"

                    local cmd = "curl https://api.openai.com/v1/images/variations -H \"Authorization: Bearer "..apikey.."\" -F image=\"@"..imagepath.."\" -F model=\"dall-e-2\" -F n=1 -F size=\""..tostring(base.size.x).."x"..tostring(base.size.y).."\""
                    local output = CreateBufferStream()
                    local result = Command(cmd, output)
                    if result ~= 0 then
                        Print("Error: curl returned error "..tostring(result))
                    end

                    base = nil
                    local t = LoadTable(output)
                    if type(t) == "table" then
                        if type(t["error"]) == "table" then
                            if type(t["error"]["message"]) == "string" then
                                local msg = t["error"]["message"]
                                Print("Error: "..msg)
                                return false
                            end
                        elseif type(t["data"]) == "table" then
                            if type(t["data"][1]["url"]) == "string" then
                                local path = t["data"][1]["url"]
                                DownloadFile(path, outputpath)
                                base = LoadPixmap(outputpath)
                            end
                        end
                    end
                    
                    if base == nil then
                        return false
                    end

                    local mipchain = {}
                    local sz = base.size
                    while true do
                        table.insert(mipchain, base)
                        if base.size.x == 1 then break end
                        base = base:Resize(base.size.x / 2, base.size.y / 2)
                    end

                    local tex = CreateTexture(TEXTURE_2D, sz.x, sz.y, base.format, mipchain, 1, TEXTURE_MIPMAPS)
                    local editor = program:OpenAsset(tex)
                    if editor then
                        editor.window:SetShape(assetwindow.window.position.x + 100, assetwindow.window.position.y + 100, editor.window.size.x, editor.window.size.y)
                    end
                    
                    collect = true
                end
            end
        end
        if collect then collectgarbage() end
        return false
    end
end

--------------------------------------------------------------------
-- Add menu item
--------------------------------------------------------------------

local menu = program.menu:FindChild("Scripting", false)
if menu ~= nil then
    local submenu = menu:FindChild("OpenAI", false)
    if submenu == nil then
        submenu = CreateMenu("OpenAI", menu)
    end
    extension.menuitem = CreateMenu("Text to Image", submenu)
end

ListenEvent(EVENT_WIDGETACTION, extension.menuitem, extension.hook, extension)

--------------------------------------------------------------------
-- Create tool window
--------------------------------------------------------------------

local winx = 500
local winy = 300 - 46
local x = 20
local y = 20
local h = 28
local lh = 20
local spacing = 36

extension.window = CreateWindow("Text to Image", 0, 0, winx, winy, program.window, WINDOW_HIDDEN | WINDOW_CENTER | WINDOW_TITLEBAR)
extension.ui = CreateInterface(extension.window)

local label = CreateLabel("Description", x, y, extension.ui.background.size.x - 2 * x, lh, extension.ui.background)
label:SetLayout(1,1,1,0)

y = y + lh

extension.promptfield = CreateTextField(x, y, extension.ui.background.size.x - 2 * x, h, extension.ui.background)
extension.promptfield:SetText("tiling seamless texture painted concrete wall abandoned dirty")
extension.promptfield:SetLayout(1,1,1,0)

y = y + spacing

--Supported sizes changed
--[[label = CreateLabel("Size", x, y, extension.ui.background.size.x - 2 * x, lh, extension.ui.background)
label:SetLayout(1,1,1,0)

y = y + lh

extension.sizefield = CreateComboBox(x, y, extension.ui.background.size.x - 2 * x, h, extension.ui.background)
extension.sizefield:AddItem("256 x 256")
extension.sizefield:AddItem("512 x 512", true)
extension.sizefield:AddItem("1024 x 1024")
extension.sizefield:SetLayout(1,1,1,0)

y = y + spacing]]

label = CreateLabel("API key", x, y, extension.ui.background.size.x - 2 * x, lh, extension.ui.background)
label:SetLayout(1,1,1,0)

y = y + lh

extension.apikeyfield = CreateTextField(x, y, extension.ui.background.size.x - 2 * x, h, extension.ui.background, TEXTFIELD_PASSWORD)
extension.apikeyfield:SetLayout(1,1,1,0)

y = y + spacing

extension.detailbutton = CreateButton("Detailed", x, y, 120, h, extension.ui.background, BUTTON_CHECKBOX)
extension.detailbutton:SetLayout(1,0,1,0)

y = y + spacing

local bw = 80
local bh = 32

extension.button = CreateButton("Generate", extension.ui.background.size.x / 2 - bw / 2, extension.ui.background.size.y - bh - 10, bw, bh, extension.ui.background)
extension.button:SetLayout(1,1,0,1)

--Rescale the window and interface if needed
local scale = program.window.display.scale
if scale ~= 1 then
    local w = winx * scale
    local h = winy * scale
    extension.window:SetShape((program.window.display.size.x - w) / 2, (program.window.display.size.y - h) / 2, w, h)
    extension.ui:SetScale(scale)
end

--Load settings
if type(program.settings.extensions) == "userdata" then
    if type(program.settings.extensions.openai) == "userdata" then
        if type(program.settings.extensions.openai.prompt) == "string" then
            extension.promptfield:SetText(program.settings.extensions.openai.prompt)
        end
        if type(program.settings.extensions.openai.apikey) == "string" then
            extension.apikeyfield:SetText(program.settings.extensions.openai.apikey)
        end
        if type(program.settings.extensions.openai.detail) == "boolean" then
            extension.detailbutton:SetState(program.settings.extensions.openai.detail)
        end
        --[[if type(program.settings.extensions.openai.size) == "userdata" then
            if type(program.settings.extensions.openai.size[1]) == "number" then
                if program.settings.extensions.openai.size[1] == 256 then
                    extension.sizefield:SelectItem(1)
                elseif program.settings.extensions.openai.size[1] == 512 then
                    extension.sizefield:SelectItem(2)
                elseif program.settings.extensions.openai.size[1] == 1024 then
                    extension.sizefield:SelectItem(3)
                end
            end
        end]]
    end
end

ListenEvent(EVENT_WINDOWCLOSE, extension.window, extension.hook, extension)
ListenEvent(EVENT_WIDGETACTION, extension.button, extension.hook, extension)
ListenEvent(EVENT_INITASSETEDITOR, nil, extension.hook, extension)