[EXPERIMENT] AGS 4: Custom Shaders support

[Vincent]

ok i added the .d3ddef file in the shader folder and edited the glsl and hlsl:

hlsl:

Spoiler

Code: ags

// Pixel shader input structure
struct PS_INPUT
{
    float2 Texture : TEXCOORD0;
};

// Pixel shader output structure
struct PS_OUTPUT
{
    float4 Color : COLOR0;
};

// Configuration uniforms
const float USE_XBR : register(c4);
const float USE_CRT_SCANLINES : register(c5);
const float USE_SUBPIXEL_AA : register(c6);
const float2 iOutputDim : register(c7); 

// Engine-provided uniforms
sampler2D iTexture : register(s0);
const float2 iTextureDim : register(c2);
const float iAlpha : register(c3);

// XBR (eXperimental Batch Rendering) edge detection
float4 getXBRColor(float2 uv)
{
    float2 texel = 1.0 / iTextureDim;
    
    // Sample 3x3 grid
    float4 c11 = tex2D(iTexture, uv); // Center
    float4 c00 = tex2D(iTexture, uv + texel * float2(-1, -1));
    float4 c20 = tex2D(iTexture, uv + texel * float2(1, -1));
    float4 c02 = tex2D(iTexture, uv + texel * float2(-1, 1));
    float4 c22 = tex2D(iTexture, uv + texel * float2(1, 1));
    
    // Calculate edge weights
    float d_edge = (dot(abs(c00 - c22), 1) + dot(abs(c20 - c02), 1)) * 0.25;
    float h_edge = (c20.r + c20.g + c20.b - c00.r - c00.g - c00.b) * 0.5;
    float v_edge = (c02.r + c02.g + c02.b - c00.r - c00.g - c00.b) * 0.5;
    
    // Blend based on edges
    float blend_factor = smoothstep(0.0, 0.5, d_edge);
    float4 result = c11;
    
    if (abs(h_edge) > abs(v_edge))
    {
        result = lerp(result, (c20 + c00) * 0.5, blend_factor);
    }
    else
    {
        result = lerp(result, (c02 + c00) * 0.5, blend_factor);
    }
    
    return result;
}

// CRT scanline effect (now resolution-aware)
float3 applyScanlines(float2 uv, float3 color)
{
    // Base scanline density on output resolution
    float scanlineDensity = iOutputDim.y / 600.0; // 600 = reference resolution
    float scanline = sin(uv.y * scanlineDensity * 3.14159 * 2.0);
    return color * (0.9 + 0.1 * scanline * scanline);
}

// Subpixel anti-aliasing (resolution-aware)
float3 applySubpixelAA(float2 uv, float3 color)
{
    // Use output dimensions for AA scaling
    float2 texel = 1.0 / iOutputDim;
    float2 subCoord = frac(uv * iOutputDim);
    
    float4 c = tex2D(iTexture, uv);
    float4 r = tex2D(iTexture, uv + float2(texel.x, 0));
    float4 l = tex2D(iTexture, uv - float2(texel.x, 0));
    float4 u = tex2D(iTexture, uv + float2(0, texel.y));
    float4 d = tex2D(iTexture, uv - float2(0, texel.y));
    
    // Weighted blend based on subpixel position
    float3 result = color;
    result = lerp(result, 0.5 * (c.rgb + r.rgb), smoothstep(0.3, 0.7, subCoord.x));
    result = lerp(result, 0.5 * (c.rgb + l.rgb), smoothstep(0.7, 0.3, subCoord.x));
    result = lerp(result, 0.5 * (c.rgb + u.rgb), smoothstep(0.3, 0.7, subCoord.y));
    result = lerp(result, 0.5 * (c.rgb + d.rgb), smoothstep(0.7, 0.3, subCoord.y));
    
    return result;
}

PS_OUTPUT main(in PS_INPUT In)
{
    float2 uv = In.Texture;
    float4 color = tex2D(iTexture, uv);
    
    // Advanced upscaling techniques
    if (USE_XBR > 0.5)
    {
        color = getXBRColor(uv);
    }
    
    if (USE_SUBPIXEL_AA > 0.5)
    {
        color.rgb = applySubpixelAA(uv, color.rgb);
    }
    
    if (USE_CRT_SCANLINES > 0.5)
    {
        color.rgb = applyScanlines(uv, color.rgb);
    }
    
    // Output with alpha
    PS_OUTPUT Out;
    Out.Color = float4(color.rgb, color.a * iAlpha);
    return Out;
}

[close]

glsl:

Spoiler

Code: ags

// Configuration uniforms (controlled from AGS)
uniform float USE_XBR;
uniform float USE_CRT_SCANLINES;
uniform float USE_SUBPIXEL_AA;

uniform vec2 iOutputDim;

// Standard engine-provided uniforms
uniform float iTime;
uniform int iGameFrame;
uniform sampler2D iTexture;
uniform vec2 iTextureDim;
uniform float iAlpha;

varying vec2 vTexCoord;

// XBR (eXperimental Batch Rendering) edge detection
vec4 getXBRColor(vec2 uv)
{
    vec2 texel = 1.0 / iTextureDim;
    
    // Sample 3x3 grid
    vec4 c11 = texture2D(iTexture, uv); // Center
    vec4 c00 = texture2D(iTexture, uv + texel * vec2(-1, -1));
    vec4 c20 = texture2D(iTexture, uv + texel * vec2(1, -1));
    vec4 c02 = texture2D(iTexture, uv + texel * vec2(-1, 1));
    vec4 c22 = texture2D(iTexture, uv + texel * vec2(1, 1));
    
    // Calculate edge weights
    float d_edge = (dot(abs(c00 - c22), vec4(1)) + dot(abs(c20 - c02), vec4(1))) * 0.25;
    float h_edge = (c20.r + c20.g + c20.b - c00.r - c00.g - c00.b) * 0.5;
    float v_edge = (c02.r + c02.g + c02.b - c00.r - c00.g - c00.b) * 0.5;
    
    // Blend based on edges
    float blend_factor = smoothstep(0.0, 0.5, d_edge);
    vec4 result = c11;
    
    if (abs(h_edge) > abs(v_edge))
    {
        result = mix(result, (c20 + c00) * 0.5, blend_factor);
    }
    else
    {
        result = mix(result, (c02 + c00) * 0.5, blend_factor);
    }
    
    return result;
}

// CRT scanline effect
vec3 applyScanlines(vec2 uv, vec3 color)
{
    float scanline = sin(uv.y * iTextureDim.y * 3.14159 * 2.0);
    return color * (0.9 + 0.1 * scanline * scanline);
}

// Subpixel anti-aliasing
vec3 applySubpixelAA(vec2 uv, vec3 color)
{
    vec2 texel = 1.0 / iTextureDim;
    vec2 subCoord = fract(uv * iTextureDim);
    
    vec4 c = texture2D(iTexture, uv);
    vec4 r = texture2D(iTexture, uv + vec2(texel.x, 0));
    vec4 l = texture2D(iTexture, uv - vec2(texel.x, 0));
    vec4 u = texture2D(iTexture, uv + vec2(0, texel.y));
    vec4 d = texture2D(iTexture, uv - vec2(0, texel.y));
    
    // Weighted blend based on subpixel position
    vec3 result = color;
    result = mix(result, 0.5 * (c.rgb + r.rgb), smoothstep(0.3, 0.7, subCoord.x));
    result = mix(result, 0.5 * (c.rgb + l.rgb), smoothstep(0.7, 0.3, subCoord.x));
    result = mix(result, 0.5 * (c.rgb + u.rgb), smoothstep(0.3, 0.7, subCoord.y));
    result = mix(result, 0.5 * (c.rgb + d.rgb), smoothstep(0.7, 0.3, subCoord.y));
    
    return result;
}

void main()
{
    vec2 uv = vTexCoord;
    vec4 color = texture2D(iTexture, uv);
    
    // Calculate scaling based on output dimensions
    vec2 pixelSize = 1.0/iOutputDim;
    
    // Advanced upscaling techniques
    if (USE_XBR > 0.5) {
        color = getXBRColor(uv);
    }
    
    if (USE_SUBPIXEL_AA > 0.5) {
        color.rgb = applySubpixelAA(uv, color.rgb);
    }
    
    if (USE_CRT_SCANLINES > 0.5) {
        // Make scanlines scale with output resolution
        float scanlineDensity = iOutputDim.y / 600.0; // Base on 600p
        color.rgb = applyScanlines(uv*scanlineDensity, color.rgb);
    }
    
    gl_FragColor = vec4(color.rgb, color.a * iAlpha);
}

[close]

d3ddef:

Spoiler

Code: ags

[compiler]
target = ps_2_b


[constants]
iGameFrame = 1
iTextureDim = 2
iAlpha = 3
USE_XBR = 4
USE_CRT_SCANLINES = 5
USE_SUBPIXEL_AA = 6
iOutputDim = 7  ; 

[close]

room script:

Spoiler

Code: ags

function SetPixelSettings(float useXBR, float useScanlines, float useSubpixelAA)
{
    pixelInstance.SetConstantF("USE_XBR", 1.);
    pixelInstance.SetConstantF("USE_CRT_SCANLINES", 1.);
    pixelInstance.SetConstantF("USE_SUBPIXEL_AA", 0.);
}

function initialize_shaders()
{
  PixelShader = ShaderProgram.CreateFromFile("$DATA$/shaders/PixelShader.glsl");
  pixelInstance = PixelShader.CreateInstance();
  SetPixelSettings(1., 1., 1.);
}

function room_RepExec()
{
  if (pixelactive) Room.BackgroundShader = pixelInstance;
  else Room.BackgroundShader = null;
}

[close]

it was just a quick test tho but its working all good now indeed, thanks. I might come out with a proper demo soon or later. But in the meantime I hope you took the demo I shared.

[Crimson Wizard]

Updated the first post with the new download link, now this version supports up to 3 secondary textures attached to a shader. Instructions are also updated in the first post.

For a quicker reference, the update contents are explained in this comment:
https://www.adventuregamestudio.co.uk/forums/engine-development/experiment-ags-4-custom-shaders-support/msg636683542/#msg636683542

[Crimson Wizard]

I have a small request, can someone download the demo game (linked in this thread's first post), enable "background" shader ("Bg" button), and tell how it looks like?

You should be seeing "ripple" effect, but does it look like smooth waves, or plain circles expanding from the center?

It worked well for me in the past, but when I test this now, I have a different result. I am in doubts whether there's something wrong with the game which I did not notice earlier, or something wrong with my video card (I had suspicions that it might be malfunctioning lately).

EDIT: okay, got this fixed by rebooting PC. No idea what caused this, either a random glitch or some other software put gfx card into a wrong state... :/

[Crimson Wizard]

Anyway, the shaders feature is now a part of AGS 4 since Alpha 22 update:
https://www.adventuregamestudio.co.uk/forums/index.php?msg=636683993

[eri0o]

I've had issues with shaders existing beyond when they should when debugging and halting AGS engine in Visual Studio, ages ago when I played with the OpenGL ES code for shaders used in the tint and stuff. I could solve it by restarting usually. I don't know how to explain but the game would boot and have ghosts tints, shaped like sprites, and only happened in my PC with Nvidia. My video card was new at the time.

[eri0o]

I don't have a target use for this now, but if there was a way to load a shader from a string, with somehow accounting for the different graphics driver, it would be possible to have shaders in script modules without packaging them separately.

Edit: nevermind, just remembered that DirectX exists.

Edit 2: I guess a possible progression of this is adding the text editor for glsl and hsls right in the editor under a new project node named shader.

[Crimson Wizard]

I don't have a target use for this now, but if there was a way to load a shader from a string, with somehow accounting for the different graphics driver, it would be possible to have shaders in script modules without packaging them separately.

I don't know yet, but as a workaround you may write files from script, and then create shaders from them.

[eri0o]

That is a good alternative for script modules that I didn't thought about. But it may work.