fix an issue with glStencilFuncSeparate being misused (strange that I don't think...

fixed deferred rendering in D3D9 renderer

git-svn-id: svn://svn.icculus.org/twilight/trunk/darkplaces@11512 d7cf8633-e32d-0410-b094-e92efae38249
::stable-branch::merge=707fc1c53d8c315f29582d58573372f9ed36f444

diff --git a/gl_backend.c b/gl_backend.c
index 4b2bd905d6f04326c03cbb40176aa666ee01657d..8c7eb2d9176a0031a273e7290a894cd68cb56f00 100644 (file)
--- a/gl_backend.c
+++ b/gl_backend.c
@@ -1928,7 +1928,8 @@ void R_SetStencilSeparate(qboolean enable, int writemask, int frontfail, int fro
                        qglStencilMask(writemask);CHECKGLERROR
                        qglStencilOpSeparate(GL_FRONT, frontfail, frontzfail, frontzpass);CHECKGLERROR
                        qglStencilOpSeparate(GL_BACK, backfail, backzfail, backzpass);CHECKGLERROR
-                       qglStencilFuncSeparate(frontcompare, backcompare, comparereference, comparereference);CHECKGLERROR
+                       qglStencilFuncSeparate(GL_FRONT, frontcompare, comparereference, comparereference);CHECKGLERROR
+                       qglStencilFuncSeparate(GL_BACK, backcompare, comparereference, comparereference);CHECKGLERROR
                }
                else if (vid.support.ext_stencil_two_side)
                {

diff --git a/shader_hlsl.h b/shader_hlsl.h
index a1d537d4bc3faeb358e40961eedcca577fe50eba..f20f8329e7ad7d45a651175e789ad3fb857b84e3 100644 (file)
--- a/shader_hlsl.h
+++ b/shader_hlsl.h
@@ -75,17 +75,17 @@
 "void main\n"
 "(\n"
 "float Depth : TEXCOORD0,\n"
-"out float4 gl_FragColor : COLOR\n"
+"out float4 dp_FragColor : COLOR\n"
 ")\n"
 "{\n"
 "//    float4 temp = float4(Depth,Depth*(65536.0/255.0),Depth*(16777216.0/255.0),0.0);\n"
 "      float4 temp = float4(Depth,Depth*256.0,Depth*65536.0,0.0);\n"
 "      temp.yz -= floor(temp.yz);\n"
-"      gl_FragColor = temp;\n"
-"//    gl_FragColor = float4(Depth,0,0,0);\n"
+"      dp_FragColor = temp;\n"
+"//    dp_FragColor = float4(Depth,0,0,0);\n"
 "}\n"
 "#endif\n"
-"#else // !MODE_DEPTH_ORSHADOW\n"
+"#else // !MODE_DEPTH_OR_SHADOW\n"
 "\n"
 "\n"
 "\n"
@@ -112,10 +112,10 @@
 "void main\n"
 "(\n"
 "float4 gl_FrontColor : COLOR0,\n"
-"out float4 gl_FragColor : COLOR\n"
+"out float4 dp_FragColor : COLOR\n"
 ")\n"
 "{\n"
-"      gl_FragColor = gl_FrontColor;\n"
+"      dp_FragColor = gl_FrontColor;\n"
 "}\n"
 "#endif\n"
 "#else // !MODE_SHOWDEPTH\n"
@@ -170,15 +170,15 @@
 "uniform float ClientTime : register(c2),\n"
 "uniform float2 PixelSize : register(c25),\n"
 "uniform float4 BloomColorSubtract : register(c43),\n"
-"out float4 gl_FragColor : COLOR\n"
+"out float4 dp_FragColor : COLOR\n"
 ")\n"
 "{\n"
-"      gl_FragColor = tex2D(Texture_First, TexCoord1);\n"
+"      dp_FragColor = tex2D(Texture_First, TexCoord1);\n"
 "#ifdef USEBLOOM\n"
-"      gl_FragColor += max(float4(0,0,0,0), tex2D(Texture_Second, TexCoord2) - BloomColorSubtract);\n"
+"      dp_FragColor += max(float4(0,0,0,0), tex2D(Texture_Second, TexCoord2) - BloomColorSubtract);\n"
 "#endif\n"
 "#ifdef USEVIEWTINT\n"
-"      gl_FragColor = lerp(gl_FragColor, ViewTintColor, ViewTintColor.a);\n"
+"      dp_FragColor = lerp(dp_FragColor, ViewTintColor, ViewTintColor.a);\n"
 "#endif\n"
 "\n"
 "#ifdef USEPOSTPROCESSING\n"
@@ -213,34 +213,34 @@
 "      float py5 =  2.0 * dot(float3(0.3, 0.59, 0.11), y5);\n"
 "      float py6 =  1.0 * dot(float3(0.3, 0.59, 0.11), y6);\n"
 "      sobel = 0.25 * abs(px1 + px2 + px3 + px4 + px5 + px6) + 0.25 * abs(py1 + py2 + py3 + py4 + py5 + py6);\n"
-"      gl_FragColor += tex2D(Texture_First, TexCoord1 + PixelSize*UserVec1.x*float2(-0.987688, -0.156434)) * UserVec1.y;\n"
-"      gl_FragColor += tex2D(Texture_First, TexCoord1 + PixelSize*UserVec1.x*float2(-0.156434, -0.891007)) * UserVec1.y;\n"
-"      gl_FragColor += tex2D(Texture_First, TexCoord1 + PixelSize*UserVec1.x*float2( 0.891007, -0.453990)) * UserVec1.y;\n"
-"      gl_FragColor += tex2D(Texture_First, TexCoord1 + PixelSize*UserVec1.x*float2( 0.707107,  0.707107)) * UserVec1.y;\n"
-"      gl_FragColor += tex2D(Texture_First, TexCoord1 + PixelSize*UserVec1.x*float2(-0.453990,  0.891007)) * UserVec1.y;\n"
-"      gl_FragColor /= (1.0 + 5.0 * UserVec1.y);\n"
-"      gl_FragColor.rgb = gl_FragColor.rgb * (1.0 + UserVec2.x) + float3(1,1,1)*max(0.0, sobel - UserVec2.z)*UserVec2.y;\n"
+"      dp_FragColor += tex2D(Texture_First, TexCoord1 + PixelSize*UserVec1.x*float2(-0.987688, -0.156434)) * UserVec1.y;\n"
+"      dp_FragColor += tex2D(Texture_First, TexCoord1 + PixelSize*UserVec1.x*float2(-0.156434, -0.891007)) * UserVec1.y;\n"
+"      dp_FragColor += tex2D(Texture_First, TexCoord1 + PixelSize*UserVec1.x*float2( 0.891007, -0.453990)) * UserVec1.y;\n"
+"      dp_FragColor += tex2D(Texture_First, TexCoord1 + PixelSize*UserVec1.x*float2( 0.707107,  0.707107)) * UserVec1.y;\n"
+"      dp_FragColor += tex2D(Texture_First, TexCoord1 + PixelSize*UserVec1.x*float2(-0.453990,  0.891007)) * UserVec1.y;\n"
+"      dp_FragColor /= (1.0 + 5.0 * UserVec1.y);\n"
+"      dp_FragColor.rgb = dp_FragColor.rgb * (1.0 + UserVec2.x) + float3(1,1,1)*max(0.0, sobel - UserVec2.z)*UserVec2.y;\n"
 "#endif\n"
 "\n"
 "#ifdef USESATURATION\n"
 "      //apply saturation BEFORE gamma ramps, so v_glslgamma value does not matter\n"
-"      float y = dot(gl_FragColor.rgb, float3(0.299, 0.587, 0.114));\n"
+"      float y = dot(dp_FragColor.rgb, float3(0.299, 0.587, 0.114));\n"
 "      // 'vampire sight' effect, wheres red is compensated\n"
 "      #ifdef SATURATION_REDCOMPENSATE\n"
-"              float rboost = max(0.0, (gl_FragColor.r - max(gl_FragColor.g, gl_FragColor.b))*(1.0 - Saturation));\n"
-"              gl_FragColor.rgb = lerp(float3(y,y,y), gl_FragColor.rgb, Saturation);\n"
-"              gl_FragColor.r += r;\n"
+"              float rboost = max(0.0, (dp_FragColor.r - max(dp_FragColor.g, dp_FragColor.b))*(1.0 - Saturation));\n"
+"              dp_FragColor.rgb = lerp(float3(y,y,y), dp_FragColor.rgb, Saturation);\n"
+"              dp_FragColor.r += r;\n"
 "      #else\n"
 "              // normal desaturation\n"
-"              //gl_FragColor = float3(y,y,y) + (gl_FragColor.rgb - float3(y)) * Saturation;\n"
-"              gl_FragColor.rgb = lerp(float3(y,y,y), gl_FragColor.rgb, Saturation);\n"
+"              //dp_FragColor = float3(y,y,y) + (dp_FragColor.rgb - float3(y)) * Saturation;\n"
+"              dp_FragColor.rgb = lerp(float3(y,y,y), dp_FragColor.rgb, Saturation);\n"
 "      #endif\n"
 "#endif\n"
 "\n"
 "#ifdef USEGAMMARAMPS\n"
-"      gl_FragColor.r = tex2D(Texture_GammaRamps, float2(gl_FragColor.r, 0)).r;\n"
-"      gl_FragColor.g = tex2D(Texture_GammaRamps, float2(gl_FragColor.g, 0)).g;\n"
-"      gl_FragColor.b = tex2D(Texture_GammaRamps, float2(gl_FragColor.b, 0)).b;\n"
+"      dp_FragColor.r = tex2D(Texture_GammaRamps, float2(dp_FragColor.r, 0)).r;\n"
+"      dp_FragColor.g = tex2D(Texture_GammaRamps, float2(dp_FragColor.g, 0)).g;\n"
+"      dp_FragColor.b = tex2D(Texture_GammaRamps, float2(dp_FragColor.b, 0)).b;\n"
 "#endif\n"
 "}\n"
 "#endif\n"
@@ -298,33 +298,33 @@
 "#ifdef USEGAMMARAMPS\n"
 "uniform sampler Texture_GammaRamps : register(s2),\n"
 "#endif\n"
-"out float4 gl_FragColor : COLOR\n"
+"out float4 dp_FragColor : COLOR\n"
 ")\n"
 "{\n"
 "#ifdef USEVIEWTINT\n"
-"      gl_FragColor = gl_FrontColor;\n"
+"      dp_FragColor = gl_FrontColor;\n"
 "#else\n"
-"      gl_FragColor = vec4(1.0, 1.0, 1.0, 1.0);\n"
+"      dp_FragColor = vec4(1.0, 1.0, 1.0, 1.0);\n"
 "#endif\n"
 "#ifdef USEDIFFUSE\n"
 "# ifdef USEREFLECTCUBE\n"
 "      // suppress texture alpha\n"
 "      dp_FragColor.rgb *= tex2D(Texture_First, TexCoord1).rgb;\n"
 "# else\n"
-"      gl_FragColor *= tex2D(Texture_First, TexCoord1);\n"
+"      dp_FragColor *= tex2D(Texture_First, TexCoord1);\n"
 "# endif\n"
 "#endif\n"
 "\n"
 "#ifdef USESPECULAR\n"
 "      float4 tex2 = tex2D(Texture_Second, TexCoord2);\n"
 "# ifdef USECOLORMAPPING\n"
-"      gl_FragColor *= tex2;\n"
+"      dp_FragColor *= tex2;\n"
 "# endif\n"
 "# ifdef USEGLOW\n"
-"      gl_FragColor += tex2;\n"
+"      dp_FragColor += tex2;\n"
 "# endif\n"
 "# ifdef USEVERTEXTEXTUREBLEND\n"
-"      gl_FragColor = lerp(gl_FragColor, tex2, tex2.a);\n"
+"      dp_FragColor = lerp(dp_FragColor, tex2, tex2.a);\n"
 "# endif\n"
 "#endif\n"
 "#ifdef USEGAMMARAMPS\n"
@@ -362,7 +362,7 @@
 "float2 TexCoord : TEXCOORD0,\n"
 "uniform sampler Texture_First : register(s0),\n"
 "uniform float4 BloomBlur_Parameters : register(c1),\n"
-"out float4 gl_FragColor : COLOR\n"
+"out float4 dp_FragColor : COLOR\n"
 ")\n"
 "{\n"
 "      int i;\n"
@@ -374,7 +374,7 @@
 "              color += tex2D(Texture_First, tc).rgb;\n"
 "              tc += BloomBlur_Parameters.xy;\n"
 "      }\n"
-"      gl_FragColor = float4(color * BloomBlur_Parameters.z + float3(BloomBlur_Parameters.w), 1);\n"
+"      dp_FragColor = float4(color * BloomBlur_Parameters.z + float3(BloomBlur_Parameters.w), 1);\n"
 "}\n"
 "#endif\n"
 "#else // !MODE_BLOOMBLUR\n"
@@ -415,7 +415,7 @@
 "uniform float4 ScreenScaleRefractReflect : register(c32),\n"
 "uniform float4 ScreenCenterRefractReflect : register(c31),\n"
 "uniform float4 RefractColor : register(c29),\n"
-"out float4 gl_FragColor : COLOR\n"
+"out float4 dp_FragColor : COLOR\n"
 ")\n"
 "{\n"
 "      float2 ScreenScaleRefractReflectIW = ScreenScaleRefractReflect.xy * (1.0 / ModelViewProjectionPosition.w);\n"
@@ -432,7 +432,7 @@
 "      f      *= min(1.0, length(tex2D(Texture_Refraction, ScreenTexCoord + float2(-0.01, 0.01)).rgb) / 0.05);\n"
 "      f      *= min(1.0, length(tex2D(Texture_Refraction, ScreenTexCoord + float2(-0.01, -0.01)).rgb) / 0.05);\n"
 "      ScreenTexCoord = lerp(SafeScreenTexCoord, ScreenTexCoord, f);\n"
-"      gl_FragColor = float4(tex2D(Texture_Refraction, ScreenTexCoord).rgb, 1) * RefractColor;\n"
+"      dp_FragColor = float4(tex2D(Texture_Refraction, ScreenTexCoord).rgb, 1) * RefractColor;\n"
 "}\n"
 "#endif\n"
 "#else // !MODE_REFRACTION\n"
@@ -486,7 +486,7 @@
 "uniform float4 ReflectColor : register(c26),\n"
 "uniform float ReflectFactor : register(c27),\n"
 "uniform float ReflectOffset : register(c28),\n"
-"out float4 gl_FragColor : COLOR\n"
+"out float4 dp_FragColor : COLOR\n"
 ")\n"
 "{\n"
 "      float4 ScreenScaleRefractReflectIW = ScreenScaleRefractReflect * (1.0 / ModelViewProjectionPosition.w);\n"
@@ -510,7 +510,7 @@
 "      f *= min(1.0, length(tex2D(Texture_Reflection, ScreenTexCoord.zw + float2(-0.01, -0.01)).rgb) / 0.05);\n"
 "      ScreenTexCoord.zw = lerp(SafeScreenTexCoord.zw, ScreenTexCoord.zw, f);\n"
 "      float Fresnel = pow(min(1.0, 1.0 - float(normalize(EyeVector).z)), 2.0) * ReflectFactor + ReflectOffset;\n"
-"      gl_FragColor = lerp(float4(tex2D(Texture_Refraction, ScreenTexCoord.xy).rgb, 1) * RefractColor, float4(tex2D(Texture_Reflection, ScreenTexCoord.zw).rgb, 1) * ReflectColor, Fresnel);\n"
+"      dp_FragColor = lerp(float4(tex2D(Texture_Refraction, ScreenTexCoord.xy).rgb, 1) * RefractColor, float4(tex2D(Texture_Reflection, ScreenTexCoord.zw).rgb, 1) * ReflectColor, Fresnel);\n"
 "}\n"
 "#endif\n"
 "#else // !MODE_WATER\n"
@@ -770,7 +770,7 @@
 "#ifdef USETRIPPY\n"
 "      gl_Position = TrippyVertex(gl_Position);\n"
 "#endif\n"
-"      VectorR.w = gl_Position.z;\n"
+"      VectorR.w = mul(ModelViewMatrix, gl_Vertex).z;\n"
 "}\n"
 "#endif // VERTEX_SHADER\n"
 "\n"
@@ -797,12 +797,7 @@
 "uniform float OffsetMapping_Bias : register(c54),\n"
 "#endif\n"
 "uniform half SpecularPower : register(c36),\n"
-"#ifdef HLSL\n"
-"out float4 gl_FragData0 : COLOR0,\n"
-"out float4 gl_FragData1 : COLOR1\n"
-"#else\n"
-"out float4 gl_FragColor : COLOR\n"
-"#endif\n"
+"out float4 dp_FragColor : COLOR\n"
 ")\n"
 "{\n"
 "      float2 TexCoord = TexCoordBoth.xy;\n"
@@ -836,16 +831,8 @@
 "      float a = offsetMappedTexture2D(Texture_Gloss).a;\n"
 "#endif\n"
 "\n"
-"#ifdef HLSL\n"
-"      gl_FragData0 = float4(normalize(surfacenormal.x * VectorS + surfacenormal.y * VectorT + surfacenormal.z * VectorR.xyz) * 0.5 + float3(0.5, 0.5, 0.5), a);\n"
-"      float Depth = VectorR.w / 256.0;\n"
-"      float4 depthcolor = float4(Depth,Depth*65536.0/255.0,Depth*16777216.0/255.0,0.0);\n"
-"//    float4 depthcolor = float4(Depth,Depth*256.0,Depth*65536.0,0.0);\n"
-"      depthcolor.yz -= floor(depthcolor.yz);\n"
-"      gl_FragData1 = depthcolor;\n"
-"#else\n"
-"      gl_FragColor = float4(normalize(surfacenormal.x * VectorS + surfacenormal.y * VectorT + surfacenormal.z * VectorR) * 0.5 + float3(0.5, 0.5, 0.5), a);\n"
-"#endif\n"
+"      float3 pixelnormal = normalize(surfacenormal.x * VectorS.xyz + surfacenormal.y * VectorT.xyz + surfacenormal.z * VectorR.xyz);\n"
+"      dp_FragColor = float4(pixelnormal.x, pixelnormal.y, VectorR.w, a);\n"
 "}\n"
 "#endif // FRAGMENT_SHADER\n"
 "#else // !MODE_DEFERREDGEOMETRY\n"
@@ -919,17 +906,19 @@
 "{\n"
 "      // calculate viewspace pixel position\n"
 "      float2 ScreenTexCoord = Pixel * PixelToScreenTexCoord;\n"
-"      //ScreenTexCoord.y = ScreenTexCoord.y * -1 + 1; // Cg is opposite?\n"
 "      float3 position;\n"
-"#ifdef HLSL\n"
-"      position.z = texDepth2D(Texture_ScreenDepth, ScreenTexCoord) * 256.0;\n"
-"#else\n"
-"      position.z = ScreenToDepth.y / (texDepth2D(Texture_ScreenDepth, ScreenTexCoord) + ScreenToDepth.x);\n"
-"#endif\n"
-"      position.xy = ModelViewPosition.xy * (position.z / ModelViewPosition.z);\n"
-"      // decode viewspace pixel normal\n"
+"      // get the geometry information (depth, normal, specular exponent)\n"
 "      half4 normalmap = half4(tex2D(Texture_ScreenNormalMap, ScreenTexCoord));\n"
-"      half3 surfacenormal = half3(normalize(normalmap.rgb - half3(0.5,0.5,0.5)));\n"
+"      // decode viewspace pixel normal\n"
+"//    float3 surfacenormal = normalize(normalmap.rgb - cast_myhalf3(0.5,0.5,0.5));\n"
+"      float3 surfacenormal = half3(normalmap.rg, sqrt(1.0-dot(normalmap.rg, normalmap.rg)));\n"
+"      // decode viewspace pixel position\n"
+"//    position.z = decodedepthmacro(dp_texture2D(Texture_ScreenDepth, ScreenTexCoord));\n"
+"      position.z = normalmap.b;\n"
+"//    position.z = ScreenToDepth.y / (dp_texture2D(Texture_ScreenDepth, ScreenTexCoord).r + ScreenToDepth.x);\n"
+"      position.xy = ModelViewPosition.xy * (position.z / ModelViewPosition.z);\n"
+"\n"
+"      // now do the actual shading\n"
 "      // surfacenormal = pixel normal in viewspace\n"
 "      // LightVector = pixel to light in viewspace\n"
 "      // CubeVector = position in lightspace\n"
@@ -1278,7 +1267,7 @@
 "#endif\n"
 "#endif // !defined(MODE_LIGHTSOURCE) && !defined(MODE_DEFERREDLIGHTSOURCE) && !defined(USESHADOWMAPORTHO)\n"
 "\n"
-"out float4 gl_FragColor : COLOR\n"
+"out float4 dp_FragColor : COLOR\n"
 ")\n"
 "{\n"
 "      float2 TexCoord = TexCoordBoth.xy;\n"
@@ -1531,7 +1520,7 @@
 "      color.rgb = lerp(color.rgb, half3(tex2D(Texture_Reflection, ScreenTexCoord).rgb) * ReflectColor.rgb, ReflectColor.a);\n"
 "#endif\n"
 "\n"
-"      gl_FragColor = float4(color);\n"
+"      dp_FragColor = float4(color);\n"
 "}\n"
 "#endif // FRAGMENT_SHADER\n"
 "\n"