r_shadow: work around a realtime world light crash

Fudges a bit more memory to support edge cases, prints a warn instead of
crashing if it's not enough.

see https://github.com/DarkPlacesEngine/darkplaces/issues/119
and 46964b3848eb7471d2f0e2284ae389b4b53337c1

Signed-off-by: bones_was_here <bones_was_here@xonotic.au>

diff --git a/gl_rsurf.c b/gl_rsurf.c
index 723e9fb4ce3ee2028ffcc20b6924f1e02955dedd..a68daf5b68e539e032441a0d715e48d3977e0a5d 100644 (file)
--- a/gl_rsurf.c
+++ b/gl_rsurf.c
@@ -1295,7 +1295,10 @@ void R_Mod_CompileShadowMap(entity_render_t *ent, vec3_t relativelightorigin, ve
        // exceeding the number of triangles in a single mesh) we have to make sure
        // that we make only a single mesh - so over-estimate the size of the mesh
        // to match the model.
-       r_shadow_compilingrtlight->static_meshchain_shadow_shadowmap = Mod_ShadowMesh_Begin(r_main_mempool, model->surfmesh.num_vertices, model->surfmesh.num_triangles);
+       // bones_was_here: +128 works around BUG https://github.com/DarkPlacesEngine/darkplaces/issues/119
+       // +64 was enough to start the map without triggering ASan, +96 was enough to also edit the light.
+       // See also: warning in Mod_ShadowMesh_AddMesh
+       r_shadow_compilingrtlight->static_meshchain_shadow_shadowmap = Mod_ShadowMesh_Begin(r_main_mempool, model->surfmesh.num_vertices, model->surfmesh.num_triangles + 128);
        R_Shadow_PrepareShadowSides(model->surfmesh.num_triangles);
        for (surfacelistindex = 0;surfacelistindex < numsurfaces;surfacelistindex++)
        {

diff --git a/model_shared.c b/model_shared.c
index fd181f355685198e8b6aea6815380855d2606bce..866ee56ee5a2c2332f2fe213aad1753e6b95ec80 100644 (file)
--- a/model_shared.c
+++ b/model_shared.c
@@ -1044,6 +1044,12 @@ void Mod_ShadowMesh_AddMesh(shadowmesh_t *mesh, const float *vertex3f, int numtr
 
        for (i = 0;i < numtris;i++)
        {
+               if ((mesh->numtriangles * 3 + 2) * sizeof(int) + 1 >= ((memheader_t *)((unsigned char *)mesh->element3i - sizeof(memheader_t)))->size)
+               {
+                       // FIXME: we didn't allocate enough space for all the tris, see R_Mod_CompileShadowMap
+                       Con_Print(CON_WARN "Mod_ShadowMesh_AddMesh: insufficient memory allocated!\n");
+                       return;
+               }
                mesh->element3i[mesh->numtriangles * 3 + 0] = Mod_ShadowMesh_AddVertex(mesh, vertex3f + 3 * element3i[i * 3 + 0]);
                mesh->element3i[mesh->numtriangles * 3 + 1] = Mod_ShadowMesh_AddVertex(mesh, vertex3f + 3 * element3i[i * 3 + 1]);
                mesh->element3i[mesh->numtriangles * 3 + 2] = Mod_ShadowMesh_AddVertex(mesh, vertex3f + 3 * element3i[i * 3 + 2]);