From 7704b6100e410ceba9634fd92bd7ee7a45dd0e25 Mon Sep 17 00:00:00 2001 From: Ell Date: Wed, 8 Mar 2017 15:01:51 -0500 Subject: [PATCH] Bug 779326 - GIMP Crash on Merging/flattening image ... ... possibly due to small win32 stack Limit the number of samples processed in one go by gimp_composite_blend() so that we don't overflow the stack when we alloca() buffers on it. --- .../layer-modes/gimpoperationlayermode.c | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/app/operations/layer-modes/gimpoperationlayermode.c b/app/operations/layer-modes/gimpoperationlayermode.c index e61adcbf0a..ef8a5bb42b 100644 --- a/app/operations/layer-modes/gimpoperationlayermode.c +++ b/app/operations/layer-modes/gimpoperationlayermode.c @@ -35,6 +35,14 @@ #include "gimpoperationlayermode.h" +/* the maximum number of samples to process in one go. used to limit + * the size of the buffers we allocate on the stack. + */ +#define GIMP_COMPOSITE_BLEND_MAX_SAMPLES ((1 << 19) /* 0.5 MiB */ / \ + 16 /* bytes per pixel */ / \ + 2 /* max number of buffers */) + + enum { PROP_0, @@ -1009,9 +1017,9 @@ gimp_composite_blend (GimpOperationLayerMode *layer_mode, GimpLayerColorSpace composite_space = layer_mode->composite_space; GimpLayerCompositeMode composite_mode = layer_mode->composite_mode; - gfloat *blend_in = in; - gfloat *blend_layer = layer; - gfloat *blend_out = out; + gfloat *blend_in; + gfloat *blend_layer; + gfloat *blend_out; gboolean composite_needs_in_color = composite_mode == GIMP_LAYER_COMPOSITE_SRC_OVER || @@ -1020,6 +1028,30 @@ gimp_composite_blend (GimpOperationLayerMode *layer_mode, const Babl *composite_to_blend_fish = NULL; const Babl *blend_to_composite_fish = NULL; + /* make sure we don't process more than GIMP_COMPOSITE_BLEND_MAX_SAMPLES + * at a time, so that we don't overflow the stack if we allocate buffers + * on it. note that this has to be done with a nested function call, + * because alloca'd buffers remain for the duration of the stack frame. + */ + while (samples > GIMP_COMPOSITE_BLEND_MAX_SAMPLES) + { + gimp_composite_blend (layer_mode, + in, layer, mask, out, + GIMP_COMPOSITE_BLEND_MAX_SAMPLES, + blend_func); + + in += 4 * GIMP_COMPOSITE_BLEND_MAX_SAMPLES; + layer += 4 * GIMP_COMPOSITE_BLEND_MAX_SAMPLES; + mask += GIMP_COMPOSITE_BLEND_MAX_SAMPLES; + out += 4 * GIMP_COMPOSITE_BLEND_MAX_SAMPLES; + + samples -= GIMP_COMPOSITE_BLEND_MAX_SAMPLES; + } + + blend_in = in; + blend_layer = layer; + blend_out = out; + if (blend_space != GIMP_LAYER_COLOR_SPACE_AUTO) { g_assert (composite_space >= 1 && composite_space < 4);