fix some bugs in the HLSL shader, mostly to do with r_shadow_deferred

[xonotic/darkplaces.git] / gl_rmain.c

diff --git a/gl_rmain.c b/gl_rmain.c
index ff26de8f3a5964bb229871f99f23f96af7b0c9ab..9f8ab2401d274003ed2d2d5a1a0f999274bc1c60 100644 (file)
--- a/gl_rmain.c
+++ b/gl_rmain.c
@@ -1833,9 +1833,9 @@ const char *builtincgshaderstring =
 ")\n"
 "{\n"
 "//    float3 temp = float3(Depth,Depth*(65536.0/255.0),Depth*(16777216.0/255.0));\n"
-"      float3 temp = float3(Depth,Depth*256.0,Depth*65536.0);\n"
+"      float4 temp = float4(Depth,Depth*256.0,Depth*65536.0,0.0);\n"
 "      temp.yz -= floor(temp.yz);\n"
-"      gl_FragColor = float4(temp,0);\n"
+"      gl_FragColor = temp;\n"
 "//    gl_FragColor = float4(Depth,0,0,0);\n"
 "}\n"
 "#endif\n"
@@ -2468,7 +2468,7 @@ const char *builtincgshaderstring =
 "#endif\n"
 "out float3 VectorS : TEXCOORD5, // direction of S texcoord (sometimes crudely called tangent)\n"
 "out float3 VectorT : TEXCOORD6, // direction of T texcoord (sometimes crudely called binormal)\n"
-"out float3 VectorR : TEXCOORD7 // direction of R texcoord (surface normal)\n"
+"out float4 VectorR : TEXCOORD7 // direction of R texcoord (surface normal), Depth value\n"
 ")\n"
 "{\n"
 "      TexCoordBoth = mul(TexMatrix, gl_MultiTexCoord0);\n"
@@ -2491,8 +2491,9 @@ const char *builtincgshaderstring =
 "\n"
 "      VectorS = mul(ModelViewMatrix, float4(gl_MultiTexCoord1.xyz, 0)).xyz;\n"
 "      VectorT = mul(ModelViewMatrix, float4(gl_MultiTexCoord2.xyz, 0)).xyz;\n"
-"      VectorR = mul(ModelViewMatrix, float4(gl_MultiTexCoord3.xyz, 0)).xyz;\n"
+"      VectorR.xyz = mul(ModelViewMatrix, float4(gl_MultiTexCoord3.xyz, 0)).xyz;\n"
 "      gl_Position = mul(ModelViewProjectionMatrix, gl_Vertex);\n"
+"      VectorR.w = gl_Position.z;\n"
 "}\n"
 "#endif // VERTEX_SHADER\n"
 "\n"
@@ -2503,7 +2504,7 @@ const char *builtincgshaderstring =
 "float3 EyeVector : TEXCOORD2,\n"
 "float3 VectorS : TEXCOORD5, // direction of S texcoord (sometimes crudely called tangent)\n"
 "float3 VectorT : TEXCOORD6, // direction of T texcoord (sometimes crudely called binormal)\n"
-"float3 VectorR : TEXCOORD7, // direction of R texcoord (surface normal)\n"
+"float4 VectorR : TEXCOORD7, // direction of R texcoord (surface normal), Depth value\n"
 "uniform sampler Texture_Normal : register(s0),\n"
 "#ifdef USEALPHAKILL\n"
 "uniform sampler Texture_Color : register(s1),\n"
@@ -2517,7 +2518,12 @@ const char *builtincgshaderstring =
 "uniform float OffsetMapping_Scale : register(c24),\n"
 "#endif\n"
 "uniform half SpecularPower : register(c36),\n"
+"#ifdef HLSL\n"
+"out float4 gl_FragData0 : COLOR0,\n"
+"out float4 gl_FragData1 : COLOR1\n"
+"#else\n"
 "out float4 gl_FragColor : COLOR\n"
+"#endif\n"
 ")\n"
 "{\n"
 "      float2 TexCoord = TexCoordBoth.xy;\n"
@@ -2547,7 +2553,16 @@ const char *builtincgshaderstring =
 "      float a = tex2D(Texture_Gloss, TexCoord).a;\n"
 "#endif\n"
 "\n"
+"#ifdef HLSL\n"
+"      gl_FragData0 = float4(normalize(surfacenormal.x * VectorS + surfacenormal.y * VectorT + surfacenormal.z * VectorR.xyz) * 0.5 + float3(0.5, 0.5, 0.5), a);\n"
+"      float Depth = VectorR.w / 256.0;\n"
+"      float4 depthcolor = float4(Depth,Depth*65536.0/255.0,Depth*16777216.0/255.0,0.0);\n"
+"//    float4 depthcolor = float4(Depth,Depth*256.0,Depth*65536.0,0.0);\n"
+"      depthcolor.yz -= floor(depthcolor.yz);\n"
+"      gl_FragData1 = depthcolor;\n"
+"#else\n"
 "      gl_FragColor = float4(normalize(surfacenormal.x * VectorS + surfacenormal.y * VectorT + surfacenormal.z * VectorR) * 0.5 + float3(0.5, 0.5, 0.5), a);\n"
+"#endif\n"
 "}\n"
 "#endif // FRAGMENT_SHADER\n"
 "#else // !MODE_DEFERREDGEOMETRY\n"
@@ -2600,9 +2615,9 @@ const char *builtincgshaderstring =
 "\n"
 "#ifdef USESHADOWMAP2D\n"
 "# ifdef USESHADOWSAMPLER\n"
-"uniform sampler Texture_ShadowMap2D : register(s11),\n"
+"uniform sampler Texture_ShadowMap2D : register(s15),\n"
 "# else\n"
-"uniform sampler Texture_ShadowMap2D : register(s11),\n"
+"uniform sampler Texture_ShadowMap2D : register(s15),\n"
 "# endif\n"
 "#endif\n"
 "\n"
@@ -2623,7 +2638,11 @@ const char *builtincgshaderstring =
 "      float2 ScreenTexCoord = Pixel * PixelToScreenTexCoord;\n"
 "      //ScreenTexCoord.y = ScreenTexCoord.y * -1 + 1; // Cg is opposite?\n"
 "      float3 position;\n"
+"#ifdef HLSL\n"
+"      position.z = texDepth2D(Texture_ScreenDepth, ScreenTexCoord) * 256.0;\n"
+"#else\n"
 "      position.z = ScreenToDepth.y / (texDepth2D(Texture_ScreenDepth, ScreenTexCoord) + ScreenToDepth.x);\n"
+"#endif\n"
 "      position.xy = ModelViewPosition.xy * (position.z / ModelViewPosition.z);\n"
 "      // decode viewspace pixel normal\n"
 "      half4 normalmap = half4(tex2D(Texture_ScreenNormalMap, ScreenTexCoord));\n"
@@ -2643,19 +2662,19 @@ const char *builtincgshaderstring =
 "      // calculate directional shading\n"
 "      float3 eyevector = position * -1.0;\n"
 "#  ifdef USEEXACTSPECULARMATH\n"
-"      half specular = pow(half(max(float(dot(reflect(lightnormal, surfacenormal), normalize(eyevector)))*-1.0, 0.0)), SpecularPower * normalmap.a);\n"
+"      half specular = half(pow(half(max(float(dot(reflect(lightnormal, surfacenormal), normalize(eyevector)))*-1.0, 0.0)), SpecularPower * normalmap.a));\n"
 "#  else\n"
 "      half3 specularnormal = half3(normalize(lightnormal + half3(normalize(eyevector))));\n"
-"      half specular = pow(half(max(float(dot(surfacenormal, specularnormal)), 0.0)), SpecularPower * normalmap.a);\n"
+"      half specular = half(pow(half(max(float(dot(surfacenormal, specularnormal)), 0.0)), SpecularPower * normalmap.a));\n"
 "#  endif\n"
 "#endif\n"
 "\n"
 "#if defined(USESHADOWMAP2D)\n"
-"      fade *= ShadowMapCompare(CubeVector, Texture_ShadowMap2D, ShadowMap_Parameters, ShadowMap_TextureScale\n"
+"      fade *= half(ShadowMapCompare(CubeVector, Texture_ShadowMap2D, ShadowMap_Parameters, ShadowMap_TextureScale\n"
 "#ifdef USESHADOWMAPVSDCT\n"
 ", Texture_CubeProjection\n"
 "#endif\n"
-"      );\n"
+"      ));\n"
 "#endif\n"
 "\n"
 "#ifdef USEDIFFUSE\n"
@@ -2904,6 +2923,8 @@ const char *builtincgshaderstring =
 "uniform sampler Texture_ScreenNormalMap : register(s14),\n"
 "#endif\n"
 "#ifdef USEDEFERREDLIGHTMAP\n"
+"uniform sampler Texture_ScreenDepth : register(s13),\n"
+"uniform sampler Texture_ScreenNormalMap : register(s14),\n"
 "uniform sampler Texture_ScreenDiffuse : register(s11),\n"
 "uniform sampler Texture_ScreenSpecular : register(s12),\n"
 "#endif\n"
@@ -2963,9 +2984,9 @@ const char *builtincgshaderstring =
 "\n"
 "#ifdef USESHADOWMAP2D\n"
 "# ifdef USESHADOWSAMPLER\n"
-"uniform sampler Texture_ShadowMap2D : register(s11),\n"
+"uniform sampler Texture_ShadowMap2D : register(s15),\n"
 "# else\n"
-"uniform sampler Texture_ShadowMap2D : register(s11),\n"
+"uniform sampler Texture_ShadowMap2D : register(s15),\n"
 "# endif\n"
 "#endif\n"
 "\n"
@@ -3072,21 +3093,24 @@ const char *builtincgshaderstring =
 "\n"
 "#ifdef USESHADOWMAP2D\n"
 "#ifdef USESHADOWMAPVSDCT\n"
-"//    color.rgb = half3(tex2D(Texture_ShadowMap2D, float2(0.1,0.1)).rgb);\n"
-"//    color.rgb = half3(tex2D(Texture_ShadowMap2D, GetShadowMapTC2D(CubeVector, ShadowMap_Parameters, Texture_CubeProjection).xy * ShadowMap_TextureScale).rgb);\n"
-"//    color.rgb = half3(GetShadowMapTC2D(CubeVector, ShadowMap_Parameters, Texture_CubeProjection).xyz * float3(ShadowMap_TextureScale,1.0));\n"
-"//    color.r = half(texDepth2D(Texture_ShadowMap2D, GetShadowMapTC2D(CubeVector, ShadowMap_Parameters, Texture_CubeProjection).xy * ShadowMap_TextureScale));\n"
+"//    float3 shadowmaptc = GetShadowMapTC2D(CubeVector, ShadowMap_Parameters, Texture_CubeProjection);\n"
 "#else\n"
 "//    float3 shadowmaptc = GetShadowMapTC2D(CubeVector, ShadowMap_Parameters);\n"
+"#endif\n"
 "//    color.rgb = half3(tex2D(Texture_ShadowMap2D, float2(0.1,0.1)).rgb);\n"
-"//    color.rgb = half3(tex2D(Texture_ShadowMap2D, GetShadowMapTC2D(CubeVector, ShadowMap_Parameters).xy * ShadowMap_TextureScale).rgb);\n"
-"//    color.rgb = half3(GetShadowMapTC2D(CubeVector, ShadowMap_Parameters).xyz * float3(ShadowMap_TextureScale,1.0));\n"
-"//    color.r = half(texDepth2D(Texture_ShadowMap2D, GetShadowMapTC2D(CubeVector, ShadowMap_Parameters).xy * ShadowMap_TextureScale));\n"
+"//    color.rgb = half3(tex2D(Texture_ShadowMap2D, shadowmaptc.xy * ShadowMap_TextureScale).rgb);\n"
+"//    color.rgb = half3(shadowmaptc.xyz * float3(ShadowMap_TextureScale,1.0));\n"
+"//    color.r = half(texDepth2D(Texture_ShadowMap2D, shadowmaptc.xy * ShadowMap_TextureScale));\n"
+"//    color.rgb = half3(tex2D(Texture_ShadowMap2D, float2(0.1,0.1)).rgb);\n"
+"//    color.rgb = half3(tex2D(Texture_ShadowMap2D, shadowmaptc.xy * ShadowMap_TextureScale).rgb);\n"
+"//    color.rgb = half3(shadowmaptc.xyz * float3(ShadowMap_TextureScale,1.0));\n"
+"//    color.r = half(texDepth2D(Texture_ShadowMap2D, shadowmaptc.xy * ShadowMap_TextureScale));\n"
 "//    color.r = half(shadowmaptc.z - texDepth2D(Texture_ShadowMap2D, shadowmaptc.xy * ShadowMap_TextureScale));\n"
+"//    color.r = half(shadowmaptc.z);\n"
 "//    color.r = half(texDepth2D(Texture_ShadowMap2D, shadowmaptc.xy * ShadowMap_TextureScale));\n"
 "//    color.r = half(shadowmaptc.z);\n"
-"#endif\n"
 "//    color.r = 1;\n"
+"//    color.rgb = abs(CubeVector);\n"
 "#endif\n"
 "//    color.rgb = half3(1,1,1);\n"
 "#endif // MODE_LIGHTSOURCE\n"
@@ -3165,13 +3189,15 @@ const char *builtincgshaderstring =
 "#endif\n"
 "\n"
 "#ifdef USESHADOWMAPORTHO\n"
-"      color.rgb *= ShadowMapCompare(ShadowMapTC, Texture_ShadowMap2D, ShadowMap_Parameters, ShadowMap_TextureScale);\n"
+"      color.rgb *= half(ShadowMapCompare(ShadowMapTC, Texture_ShadowMap2D, ShadowMap_Parameters, ShadowMap_TextureScale));\n"
 "#endif\n"
 "\n"
 "#ifdef USEDEFERREDLIGHTMAP\n"
 "      float2 ScreenTexCoord = Pixel * PixelToScreenTexCoord;\n"
 "      color.rgb += diffusetex * half3(tex2D(Texture_ScreenDiffuse, ScreenTexCoord).rgb) * DeferredMod_Diffuse;\n"
 "      color.rgb += glosstex.rgb * half3(tex2D(Texture_ScreenSpecular, ScreenTexCoord).rgb) * DeferredMod_Specular;\n"
+"//    color.rgb = half3(tex2D(Texture_ScreenDepth, ScreenTexCoord).rgb);\n"
+"//    color.r = half(texDepth2D(Texture_ScreenDepth, ScreenTexCoord)) * 1.0;\n"
 "#endif\n"
 "\n"
 "#ifdef USEGLOW\n"
@@ -3735,7 +3761,7 @@ static void R_GLSL_CompilePermutation(r_glsl_permutation_t *p, unsigned int mode
                if (p->loc_Texture_Cube            >= 0) qglUniform1iARB(p->loc_Texture_Cube           , GL20TU_CUBE);
                if (p->loc_Texture_Refraction      >= 0) qglUniform1iARB(p->loc_Texture_Refraction     , GL20TU_REFRACTION);
                if (p->loc_Texture_Reflection      >= 0) qglUniform1iARB(p->loc_Texture_Reflection     , GL20TU_REFLECTION);
-               if (p->loc_Texture_ShadowMap2D     >= 0) qglUniform1iARB(p->loc_Texture_ShadowMap2D    , permutation & SHADERPERMUTATION_SHADOWMAPORTHO ? GL20TU_SHADOWMAPORTHO2D : GL20TU_SHADOWMAP2D);
+               if (p->loc_Texture_ShadowMap2D     >= 0) qglUniform1iARB(p->loc_Texture_ShadowMap2D    , GL20TU_SHADOWMAP2D);
                if (p->loc_Texture_CubeProjection  >= 0) qglUniform1iARB(p->loc_Texture_CubeProjection , GL20TU_CUBEPROJECTION);
                if (p->loc_Texture_ScreenDepth     >= 0) qglUniform1iARB(p->loc_Texture_ScreenDepth    , GL20TU_SCREENDEPTH);
                if (p->loc_Texture_ScreenNormalMap >= 0) qglUniform1iARB(p->loc_Texture_ScreenNormalMap, GL20TU_SCREENNORMALMAP);
@@ -4482,7 +4508,7 @@ static void R_HLSL_CacheShader(r_hlsl_permutation_t *p, const char *cachename, c
                vsbin = (DWORD *)FS_LoadFile(va("%s.vsbin", cachename), r_main_mempool, true, &vsbinsize);
                psbin = (DWORD *)FS_LoadFile(va("%s.psbin", cachename), r_main_mempool, true, &psbinsize);
        }
-       if (debugshader || (!vsbin && vertstring) || (!psbin && fragstring))
+       if ((!vsbin && vertstring) || (!psbin && fragstring))
        {
                const char* dllnames_d3dx9 [] =
                {
@@ -4575,14 +4601,11 @@ static void R_HLSL_CacheShader(r_hlsl_permutation_t *p, const char *cachename, c
                else
                        Con_Printf("Unable to compile shader - D3DXCompileShader function not found\n");
        }
-       if (vsbin)
+       if (vsbin && psbin)
        {
                vsresult = IDirect3DDevice9_CreateVertexShader(vid_d3d9dev, vsbin, &p->vertexshader);
                if (FAILED(vsresult))
                        Con_Printf("HLSL CreateVertexShader failed for %s (hresult = %8x)\n", cachename, vsresult);
-       }
-       if (psbin)
-       {
                psresult = IDirect3DDevice9_CreatePixelShader(vid_d3d9dev, psbin, &p->pixelshader);
                if (FAILED(psresult))
                        Con_Printf("HLSL CreatePixelShader failed for %s (hresult = %8x)\n", cachename, psresult);
@@ -5063,6 +5086,7 @@ extern int r_shadow_prepass_width;
 extern int r_shadow_prepass_height;
 extern rtexture_t *r_shadow_prepassgeometrydepthtexture;
 extern rtexture_t *r_shadow_prepassgeometrynormalmaptexture;
+extern rtexture_t *r_shadow_prepassgeometrydepthcolortexture;
 extern rtexture_t *r_shadow_prepasslightingdiffusetexture;
 extern rtexture_t *r_shadow_prepasslightingspeculartexture;
 extern cvar_t gl_mesh_separatearrays;
@@ -5633,7 +5657,7 @@ void R_SetupShader_Surface(const vec3_t lightcolorbase, qboolean modellighting,
                if (permutation & SHADERPERMUTATION_DEFERREDLIGHTMAP  ) R_Mesh_TexBind(GL20TU_SCREENSPECULAR    , r_shadow_prepasslightingspeculartexture             );
                if (rsurface.rtlight || (r_shadow_usingshadowmaportho && !(rsurface.ent_flags & RENDER_NOSELFSHADOW)))
                {
-                       R_Mesh_TexBind((permutation & SHADERPERMUTATION_SHADOWMAPORTHO) ? GL20TU_SHADOWMAPORTHO2D : GL20TU_SHADOWMAP2D, (permutation & SHADERPERMUTATION_SHADOWSAMPLER) ? r_shadow_shadowmap2dtexture : r_shadow_shadowmap2dcolortexture);
+                       R_Mesh_TexBind(GL20TU_SHADOWMAP2D, r_shadow_shadowmap2dcolortexture);
                        if (rsurface.rtlight)
                        {
                                if (permutation & SHADERPERMUTATION_CUBEFILTER        ) R_Mesh_TexBind(GL20TU_CUBE              , rsurface.rtlight->currentcubemap                    );
@@ -5789,7 +5813,7 @@ void R_SetupShader_Surface(const vec3_t lightcolorbase, qboolean modellighting,
                if (r_glsl_permutation->loc_Texture_ScreenSpecular  >= 0) R_Mesh_TexBind(GL20TU_SCREENSPECULAR    , r_shadow_prepasslightingspeculartexture             );
                if (rsurface.rtlight || (r_shadow_usingshadowmaportho && !(rsurface.ent_flags & RENDER_NOSELFSHADOW)))
                {
-                       if (r_glsl_permutation->loc_Texture_ShadowMap2D     >= 0) R_Mesh_TexBind(r_shadow_usingshadowmaportho ? GL20TU_SHADOWMAPORTHO2D : GL20TU_SHADOWMAP2D, r_shadow_shadowmap2dtexture                         );
+                       if (r_glsl_permutation->loc_Texture_ShadowMap2D     >= 0) R_Mesh_TexBind(GL20TU_SHADOWMAP2D, r_shadow_shadowmap2dtexture                         );
                        if (rsurface.rtlight)
                        {
                                if (r_glsl_permutation->loc_Texture_Cube            >= 0) R_Mesh_TexBind(GL20TU_CUBE              , rsurface.rtlight->currentcubemap                    );
@@ -6038,10 +6062,10 @@ void R_SetupShader_DeferredLight(const rtlight_t *rtlight)
                hlslPSSetParameter2f(D3DPSREGISTER_PixelToScreenTexCoord, 1.0f/vid.width, 1.0/vid.height);
 
                R_Mesh_TexBind(GL20TU_ATTENUATION        , r_shadow_attenuationgradienttexture                 );
-               R_Mesh_TexBind(GL20TU_SCREENDEPTH        , r_shadow_prepassgeometrydepthtexture                );
+               R_Mesh_TexBind(GL20TU_SCREENDEPTH        , r_shadow_prepassgeometrydepthcolortexture           );
                R_Mesh_TexBind(GL20TU_SCREENNORMALMAP    , r_shadow_prepassgeometrynormalmaptexture            );
                R_Mesh_TexBind(GL20TU_CUBE               , rsurface.rtlight->currentcubemap                    );
-               R_Mesh_TexBind(GL20TU_SHADOWMAP2D        , r_shadow_shadowmap2dtexture                         );
+               R_Mesh_TexBind(GL20TU_SHADOWMAP2D        , r_shadow_shadowmap2dcolortexture                    );
                R_Mesh_TexBind(GL20TU_CUBEPROJECTION     , r_shadow_shadowmapvsdcttexture                      );
 #endif
                break;
@@ -12159,7 +12183,7 @@ static void R_DrawTextureSurfaceList_Sky(int texturenumsurfaces, const msurface_
                        // just to make sure that braindead drivers don't draw
                        // anything despite that colormask...
                        GL_BlendFunc(GL_ZERO, GL_ONE);
-                       RSurf_PrepareVerticesForBatch(BATCHNEED_VERTEXPOSITION, texturenumsurfaces, texturesurfacelist);
+                       RSurf_PrepareVerticesForBatch(BATCHNEED_VERTEXPOSITION | BATCHNEED_NOGAPS, texturenumsurfaces, texturesurfacelist);
                        R_Mesh_PrepareVertices_Position(rsurface.batchnumvertices, rsurface.batchvertexposition, rsurface.batchvertexpositionbuffer);
                }
                else
@@ -12167,7 +12191,7 @@ static void R_DrawTextureSurfaceList_Sky(int texturenumsurfaces, const msurface_
                        R_SetupShader_Generic(NULL, NULL, GL_MODULATE, 1);
                        // fog sky
                        GL_BlendFunc(GL_ONE, GL_ZERO);
-                       RSurf_PrepareVerticesForBatch(BATCHNEED_ARRAY_VERTEX, texturenumsurfaces, texturesurfacelist);
+                       RSurf_PrepareVerticesForBatch(BATCHNEED_ARRAY_VERTEX | BATCHNEED_NOGAPS, texturenumsurfaces, texturesurfacelist);
                        GL_Color(r_refdef.fogcolor[0], r_refdef.fogcolor[1], r_refdef.fogcolor[2], 1);
                        R_Mesh_PrepareVertices_Generic_Arrays(rsurface.batchnumvertices, rsurface.batchvertex3f, NULL, NULL);
                }