[EXPERIMENT] AGS 4: Custom Shaders support

[Vincent]

ok i added the .d3ddef file in the shader folder and edited the glsl and hlsl:

hlsl:

Spoiler

Code: ags

// Pixel shader input structure
struct PS_INPUT
{
    float2 Texture : TEXCOORD0;
};

// Pixel shader output structure
struct PS_OUTPUT
{
    float4 Color : COLOR0;
};

// Configuration uniforms
const float USE_XBR : register(c4);
const float USE_CRT_SCANLINES : register(c5);
const float USE_SUBPIXEL_AA : register(c6);
const float2 iOutputDim : register(c7); 

// Engine-provided uniforms
sampler2D iTexture : register(s0);
const float2 iTextureDim : register(c2);
const float iAlpha : register(c3);

// XBR (eXperimental Batch Rendering) edge detection
float4 getXBRColor(float2 uv)
{
    float2 texel = 1.0 / iTextureDim;
    
    // Sample 3x3 grid
    float4 c11 = tex2D(iTexture, uv); // Center
    float4 c00 = tex2D(iTexture, uv + texel * float2(-1, -1));
    float4 c20 = tex2D(iTexture, uv + texel * float2(1, -1));
    float4 c02 = tex2D(iTexture, uv + texel * float2(-1, 1));
    float4 c22 = tex2D(iTexture, uv + texel * float2(1, 1));
    
    // Calculate edge weights
    float d_edge = (dot(abs(c00 - c22), 1) + dot(abs(c20 - c02), 1)) * 0.25;
    float h_edge = (c20.r + c20.g + c20.b - c00.r - c00.g - c00.b) * 0.5;
    float v_edge = (c02.r + c02.g + c02.b - c00.r - c00.g - c00.b) * 0.5;
    
    // Blend based on edges
    float blend_factor = smoothstep(0.0, 0.5, d_edge);
    float4 result = c11;
    
    if (abs(h_edge) > abs(v_edge))
    {
        result = lerp(result, (c20 + c00) * 0.5, blend_factor);
    }
    else
    {
        result = lerp(result, (c02 + c00) * 0.5, blend_factor);
    }
    
    return result;
}

// CRT scanline effect (now resolution-aware)
float3 applyScanlines(float2 uv, float3 color)
{
    // Base scanline density on output resolution
    float scanlineDensity = iOutputDim.y / 600.0; // 600 = reference resolution
    float scanline = sin(uv.y * scanlineDensity * 3.14159 * 2.0);
    return color * (0.9 + 0.1 * scanline * scanline);
}

// Subpixel anti-aliasing (resolution-aware)
float3 applySubpixelAA(float2 uv, float3 color)
{
    // Use output dimensions for AA scaling
    float2 texel = 1.0 / iOutputDim;
    float2 subCoord = frac(uv * iOutputDim);
    
    float4 c = tex2D(iTexture, uv);
    float4 r = tex2D(iTexture, uv + float2(texel.x, 0));
    float4 l = tex2D(iTexture, uv - float2(texel.x, 0));
    float4 u = tex2D(iTexture, uv + float2(0, texel.y));
    float4 d = tex2D(iTexture, uv - float2(0, texel.y));
    
    // Weighted blend based on subpixel position
    float3 result = color;
    result = lerp(result, 0.5 * (c.rgb + r.rgb), smoothstep(0.3, 0.7, subCoord.x));
    result = lerp(result, 0.5 * (c.rgb + l.rgb), smoothstep(0.7, 0.3, subCoord.x));
    result = lerp(result, 0.5 * (c.rgb + u.rgb), smoothstep(0.3, 0.7, subCoord.y));
    result = lerp(result, 0.5 * (c.rgb + d.rgb), smoothstep(0.7, 0.3, subCoord.y));
    
    return result;
}

PS_OUTPUT main(in PS_INPUT In)
{
    float2 uv = In.Texture;
    float4 color = tex2D(iTexture, uv);
    
    // Advanced upscaling techniques
    if (USE_XBR > 0.5)
    {
        color = getXBRColor(uv);
    }
    
    if (USE_SUBPIXEL_AA > 0.5)
    {
        color.rgb = applySubpixelAA(uv, color.rgb);
    }
    
    if (USE_CRT_SCANLINES > 0.5)
    {
        color.rgb = applyScanlines(uv, color.rgb);
    }
    
    // Output with alpha
    PS_OUTPUT Out;
    Out.Color = float4(color.rgb, color.a * iAlpha);
    return Out;
}

[close]

glsl:

Spoiler

Code: ags

// Configuration uniforms (controlled from AGS)
uniform float USE_XBR;
uniform float USE_CRT_SCANLINES;
uniform float USE_SUBPIXEL_AA;

uniform vec2 iOutputDim;

// Standard engine-provided uniforms
uniform float iTime;
uniform int iGameFrame;
uniform sampler2D iTexture;
uniform vec2 iTextureDim;
uniform float iAlpha;

varying vec2 vTexCoord;

// XBR (eXperimental Batch Rendering) edge detection
vec4 getXBRColor(vec2 uv)
{
    vec2 texel = 1.0 / iTextureDim;
    
    // Sample 3x3 grid
    vec4 c11 = texture2D(iTexture, uv); // Center
    vec4 c00 = texture2D(iTexture, uv + texel * vec2(-1, -1));
    vec4 c20 = texture2D(iTexture, uv + texel * vec2(1, -1));
    vec4 c02 = texture2D(iTexture, uv + texel * vec2(-1, 1));
    vec4 c22 = texture2D(iTexture, uv + texel * vec2(1, 1));
    
    // Calculate edge weights
    float d_edge = (dot(abs(c00 - c22), vec4(1)) + dot(abs(c20 - c02), vec4(1))) * 0.25;
    float h_edge = (c20.r + c20.g + c20.b - c00.r - c00.g - c00.b) * 0.5;
    float v_edge = (c02.r + c02.g + c02.b - c00.r - c00.g - c00.b) * 0.5;
    
    // Blend based on edges
    float blend_factor = smoothstep(0.0, 0.5, d_edge);
    vec4 result = c11;
    
    if (abs(h_edge) > abs(v_edge))
    {
        result = mix(result, (c20 + c00) * 0.5, blend_factor);
    }
    else
    {
        result = mix(result, (c02 + c00) * 0.5, blend_factor);
    }
    
    return result;
}

// CRT scanline effect
vec3 applyScanlines(vec2 uv, vec3 color)
{
    float scanline = sin(uv.y * iTextureDim.y * 3.14159 * 2.0);
    return color * (0.9 + 0.1 * scanline * scanline);
}

// Subpixel anti-aliasing
vec3 applySubpixelAA(vec2 uv, vec3 color)
{
    vec2 texel = 1.0 / iTextureDim;
    vec2 subCoord = fract(uv * iTextureDim);
    
    vec4 c = texture2D(iTexture, uv);
    vec4 r = texture2D(iTexture, uv + vec2(texel.x, 0));
    vec4 l = texture2D(iTexture, uv - vec2(texel.x, 0));
    vec4 u = texture2D(iTexture, uv + vec2(0, texel.y));
    vec4 d = texture2D(iTexture, uv - vec2(0, texel.y));
    
    // Weighted blend based on subpixel position
    vec3 result = color;
    result = mix(result, 0.5 * (c.rgb + r.rgb), smoothstep(0.3, 0.7, subCoord.x));
    result = mix(result, 0.5 * (c.rgb + l.rgb), smoothstep(0.7, 0.3, subCoord.x));
    result = mix(result, 0.5 * (c.rgb + u.rgb), smoothstep(0.3, 0.7, subCoord.y));
    result = mix(result, 0.5 * (c.rgb + d.rgb), smoothstep(0.7, 0.3, subCoord.y));
    
    return result;
}

void main()
{
    vec2 uv = vTexCoord;
    vec4 color = texture2D(iTexture, uv);
    
    // Calculate scaling based on output dimensions
    vec2 pixelSize = 1.0/iOutputDim;
    
    // Advanced upscaling techniques
    if (USE_XBR > 0.5) {
        color = getXBRColor(uv);
    }
    
    if (USE_SUBPIXEL_AA > 0.5) {
        color.rgb = applySubpixelAA(uv, color.rgb);
    }
    
    if (USE_CRT_SCANLINES > 0.5) {
        // Make scanlines scale with output resolution
        float scanlineDensity = iOutputDim.y / 600.0; // Base on 600p
        color.rgb = applyScanlines(uv*scanlineDensity, color.rgb);
    }
    
    gl_FragColor = vec4(color.rgb, color.a * iAlpha);
}

[close]

d3ddef:

Spoiler

Code: ags

[compiler]
target = ps_2_b


[constants]
iGameFrame = 1
iTextureDim = 2
iAlpha = 3
USE_XBR = 4
USE_CRT_SCANLINES = 5
USE_SUBPIXEL_AA = 6
iOutputDim = 7  ; 

[close]

room script:

Spoiler

Code: ags

function SetPixelSettings(float useXBR, float useScanlines, float useSubpixelAA)
{
    pixelInstance.SetConstantF("USE_XBR", 1.);
    pixelInstance.SetConstantF("USE_CRT_SCANLINES", 1.);
    pixelInstance.SetConstantF("USE_SUBPIXEL_AA", 0.);
}

function initialize_shaders()
{
  PixelShader = ShaderProgram.CreateFromFile("$DATA$/shaders/PixelShader.glsl");
  pixelInstance = PixelShader.CreateInstance();
  SetPixelSettings(1., 1., 1.);
}

function room_RepExec()
{
  if (pixelactive) Room.BackgroundShader = pixelInstance;
  else Room.BackgroundShader = null;
}

[close]

it was just a quick test tho but its working all good now indeed, thanks. I might come out with a proper demo soon or later. But in the meantime I hope you took the demo I shared.

[Crimson Wizard]

Updated the first post with the new download link, now this version supports up to 3 secondary textures attached to a shader. Instructions are also updated in the first post.

For a quicker reference, the update contents are explained in this comment:
https://www.adventuregamestudio.co.uk/forums/engine-development/experiment-ags-4-custom-shaders-support/msg636683542/#msg636683542