libgimpbase and core: fix #12711 plugin hang on MacOS

This a bandaid for an issue on MacOS:
an IO event is received that says a read pipe from a plugin has data,
but the pipe is actually empty and reads hang, and hang the app.

Possibly this is an issue with GLib and could be fixed there,
to not send such a spurious IO event.

Possibly an alternative fix is to determine whether the app side of the
plugin protocol is in a state where no messages from the plugin
are expected.  In that case, the read pipe should be flushed
i.e. data discarded, with non-blocking reads, until a read returns nothing.

app/plug-in/gimpplugin.c

@@ -180,6 +180,10 @@ gimp_plug_in_finalize (GObject *object)
   G_OBJECT_CLASS (parent_class)->finalize (object);
 }
 
+/* Always returns TRUE.
+ * This is of type GIOFunc, for which returning TRUE
+ * means the IO event source should not be removed.
+ */
 static gboolean
 gimp_plug_in_recv_message (GIOChannel   *channel,
                            GIOCondition  cond,
@@ -196,6 +200,18 @@ gimp_plug_in_recv_message (GIOChannel   *channel,
     return TRUE;
 #endif
 
+#ifdef __APPLE__
+  /* Workaround for #12711:
+   * sometimes we get an IO event of type G_IO_IN when the pipe is empty.
+   *
+   * There must be at least 4 bytes of message type
+   * else reads will hang, and the app appear non-responsive.
+   */
+
+  if (gimp_wire_count_bytes_ready (channel) < 4)
+    return TRUE;
+#endif
+
   if (plug_in->my_read == NULL)
     return TRUE;
 

libgimpbase/gimpbase.def

@@ -273,6 +273,7 @@ EXPORTS
 	gimp_value_take_double_array
 	gimp_value_take_int32_array
 	gimp_wire_clear_error
+  gimp_wire_count_bytes_ready
 	gimp_wire_destroy
 	gimp_wire_error
 	gimp_wire_flush

libgimpbase/gimpwire.c

@@ -21,6 +21,7 @@
 #include <string.h>
 
 #include <glib-object.h>
+#include <sys/ioctl.h>
 
 #include <libgimpcolor/gimpcolortypes.h>
 
@@ -310,6 +311,41 @@ gimp_wire_destroy (GimpWireMessage *msg)
   (* handler->destroy_func) (msg);
 }
 
+/* Returns the count of bytes in the channel.
+ * Bytes that can be read without blocking.
+ *
+ * Returns zero on an IO error.
+ * Also may return zero if the channel is empty.
+ *
+ * Requires channel is a pipe open for reading.
+ *
+ * This should only be used in extraordinary situations.
+ * It is only for UNIX-like platforms; might not be portable to MSWindows.
+ * It can also be used for debugging the protocol, to know message lengths.
+ *
+ * Used on MacOS for a seeming bug in IO events.
+ * Usually, on an IO event on condition G_IO_IN,
+ * you can assume the pipe is not empty and a read will not block.
+ */
+guint
+gimp_wire_count_bytes_ready (GIOChannel *channel)
+{
+  int   err = 0;
+  guint result;
+  int   fd;
+
+  fd = g_io_channel_unix_get_fd (channel);
+  err = ioctl (fd, FIONREAD, &result);
+  if (err < 0)
+    {
+      g_warning ("%s ioctl failed.", G_STRFUNC);
+      result = 0;
+    }
+
+  g_debug ("%s bytes ready: %d", G_STRFUNC, result);
+  return result;
+}
+
 gboolean
 _gimp_wire_read_int64 (GIOChannel *channel,
                        guint64    *data,

libgimpbase/gimpwire.h

@@ -80,6 +80,8 @@ gboolean  gimp_wire_write_msg     (GIOChannel          *channel,
 
 void      gimp_wire_destroy       (GimpWireMessage     *msg);
 
+guint     gimp_wire_count_bytes_ready (GIOChannel      *channel);
+
 
 /*  for internal use in libgimpbase  */