libobs: Fix Wayland push to talk (sort of)

libobs/obs-cocoa.m

@@ -683,29 +683,29 @@ obs_key_t obs_key_from_virtual_key(int keyCode)
     return OBS_KEY_NONE;
 }
 
-bool obs_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *platform, obs_key_t key)
+enum obs_hotkey_platform_pressed_state obs_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *platform, obs_key_t key)
 {
     if (key >= OBS_KEY_MOUSE1 && key <= OBS_KEY_MOUSE29) {
         int button = key - 1;
 
         NSUInteger buttons = [NSEvent pressedMouseButtons];
 
         if ((buttons & (1 << button)) != 0) {
-            return true;
+            return OBS_HOTKEY_PLATFORM_PRESSED;
         } else {
-            return false;
+            return OBS_HOTKEY_PLATFORM_NOT_PRESSED;
         }
     }
 
-    if (!platform) {
-        return false;
+    if (!platform || !platform->eventTap) {
+        return OBS_HOTKEY_PLATFORM_UNKNOWN;
     }
 
     if (key >= OBS_KEY_LAST_VALUE) {
-        return false;
+        return OBS_HOTKEY_PLATFORM_NOT_PRESSED;
     }
 
-    return platform->is_key_down[key];
+    return platform->is_key_down[key] ? OBS_HOTKEY_PLATFORM_PRESSED : OBS_HOTKEY_PLATFORM_NOT_PRESSED;
 }
 
 static void unichar_to_utf8(const UniChar *character, char *buffer)

libobs/obs-hotkey.c

@@ -999,7 +999,7 @@ static inline bool modifiers_match(obs_hotkey_binding_t *binding, uint32_t modif
 		return modifiers == modifiers_;
 }
 
-static inline bool is_pressed(obs_key_t key)
+static inline enum obs_hotkey_platform_pressed_state get_pressed_state(obs_key_t key)
 {
 	return obs_hotkeys_platform_is_pressed(obs->hotkeys.platform_context, key);
 }
@@ -1050,9 +1050,17 @@ static inline void handle_binding(obs_hotkey_binding_t *binding, uint32_t modifi
 	if ((!binding->modifiers_match && !modifiers_only) || !modifiers_match_)
 		goto reset;
 
-	if ((pressed && !*pressed) || (!pressed && !is_pressed(binding->key.key)))
+	if (pressed && !*pressed)
 		goto reset;
 
+	if (!pressed) {
+		enum obs_hotkey_platform_pressed_state pressed_state = get_pressed_state(binding->key.key);
+		if (pressed_state == OBS_HOTKEY_PLATFORM_UNKNOWN)
+			return;
+		if (pressed_state == OBS_HOTKEY_PLATFORM_NOT_PRESSED)
+			goto reset;
+	}
+
 	if (binding->pressed || no_press)
 		return;
 
@@ -1077,16 +1085,36 @@ static inline bool inject_hotkey(void *data, size_t idx, obs_hotkey_binding_t *b
 {
 	UNUSED_PARAMETER(idx);
 	struct obs_hotkey_internal_inject *event = data;
+	bool modifiers_match_ = modifiers_match(binding, event->hotkey.modifiers, event->strict_modifiers);
+	bool modifiers_only = binding->key.key == OBS_KEY_NONE;
+	bool key_matches = binding->key.key == event->hotkey.key;
+
+	if (event->hotkey.key == OBS_KEY_NONE && binding->pressed && !modifiers_match_) {
+		release_pressed_binding(binding);
+		return true;
+	}
+
+	if (modifiers_only) {
+		if (event->pressed && modifiers_match_) {
+			binding->modifiers_match = true;
+			if (!binding->pressed)
+				press_released_binding(binding);
+		}
+
+		return true;
+	}
 
-	if (modifiers_match(binding, event->hotkey.modifiers, event->strict_modifiers)) {
-		bool pressed = binding->key.key == event->hotkey.key && event->pressed;
-		if (binding->key.key == OBS_KEY_NONE)
-			pressed = true;
+	if (key_matches) {
+		if (event->pressed) {
+			if (!modifiers_match_)
+				return true;
 
-		if (pressed) {
 			binding->modifiers_match = true;
 			if (!binding->pressed)
 				press_released_binding(binding);
+
+		} else if (binding->pressed) {
+			release_pressed_binding(binding);
 		}
 	}
 
@@ -1118,6 +1146,7 @@ void obs_hotkey_enable_background_press(bool enable)
 
 struct obs_query_hotkeys_helper {
 	uint32_t modifiers;
+	bool modifiers_known;
 	bool no_press;
 	bool strict_modifiers;
 };
@@ -1127,25 +1156,36 @@ static inline bool query_hotkey(void *data, size_t idx, obs_hotkey_binding_t *bi
 	UNUSED_PARAMETER(idx);
 
 	struct obs_query_hotkeys_helper *param = (struct obs_query_hotkeys_helper *)data;
+	if (!param->modifiers_known)
+		return true;
+
 	handle_binding(binding, param->modifiers, param->no_press, param->strict_modifiers, NULL);
 
 	return true;
 }
 
+static inline bool query_modifier(obs_key_t key, uint32_t modifier, uint32_t *modifiers)
+{
+	enum obs_hotkey_platform_pressed_state pressed_state = get_pressed_state(key);
+	if (pressed_state == OBS_HOTKEY_PLATFORM_UNKNOWN)
+		return false;
+	if (pressed_state == OBS_HOTKEY_PLATFORM_PRESSED)
+		*modifiers |= modifier;
+
+	return true;
+}
+
 static inline void query_hotkeys()
 {
 	uint32_t modifiers = 0;
-	if (is_pressed(OBS_KEY_SHIFT))
-		modifiers |= INTERACT_SHIFT_KEY;
-	if (is_pressed(OBS_KEY_CONTROL))
-		modifiers |= INTERACT_CONTROL_KEY;
-	if (is_pressed(OBS_KEY_ALT))
-		modifiers |= INTERACT_ALT_KEY;
-	if (is_pressed(OBS_KEY_META))
-		modifiers |= INTERACT_COMMAND_KEY;
+	bool modifiers_known = query_modifier(OBS_KEY_SHIFT, INTERACT_SHIFT_KEY, &modifiers) &&
+			       query_modifier(OBS_KEY_CONTROL, INTERACT_CONTROL_KEY, &modifiers) &&
+			       query_modifier(OBS_KEY_ALT, INTERACT_ALT_KEY, &modifiers) &&
+			       query_modifier(OBS_KEY_META, INTERACT_COMMAND_KEY, &modifiers);
 
 	struct obs_query_hotkeys_helper param = {
 		modifiers,
+		modifiers_known,
 		obs->hotkeys.thread_disable_press,
 		obs->hotkeys.strict_modifiers,
 	};

libobs/obs-internal.h

@@ -242,10 +242,16 @@ typedef struct obs_hotkeys_platform obs_hotkeys_platform_t;
 
 void *obs_hotkey_thread(void *param);
 
+enum obs_hotkey_platform_pressed_state {
+	OBS_HOTKEY_PLATFORM_PRESSED,
+	OBS_HOTKEY_PLATFORM_NOT_PRESSED,
+	OBS_HOTKEY_PLATFORM_UNKNOWN,
+};
+
 struct obs_core_hotkeys;
 bool obs_hotkeys_platform_init(struct obs_core_hotkeys *hotkeys);
 void obs_hotkeys_platform_free(struct obs_core_hotkeys *hotkeys);
-bool obs_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *context, obs_key_t key);
+enum obs_hotkey_platform_pressed_state obs_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *context, obs_key_t key);
 
 const char *obs_get_hotkey_translation(obs_key_t key, const char *def);
 

libobs/obs-nix-wayland.c

@@ -258,14 +258,15 @@ static void obs_nix_wayland_hotkeys_platform_free(struct obs_core_hotkeys *hotke
 	bfree(plat);
 }
 
-static bool obs_nix_wayland_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *context, obs_key_t key)
+static enum obs_hotkey_platform_pressed_state
+obs_nix_wayland_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *context, obs_key_t key)
 {
 	UNUSED_PARAMETER(context);
 	UNUSED_PARAMETER(key);
 	// This function is only used by the hotkey thread for capturing out of
 	// focus hotkey triggers. Since wayland never delivers key events when out
-	// of focus we leave this blank intentionally.
-	return false;
+	// of focus we report that the state cannot be queried.
+	return OBS_HOTKEY_PLATFORM_UNKNOWN;
 }
 
 static void obs_nix_wayland_key_to_str(obs_key_t key, struct dstr *dstr)

libobs/obs-nix-x11.c

@@ -1077,15 +1077,19 @@ static bool key_pressed(xcb_connection_t *connection, obs_hotkeys_platform_t *co
 	return pressed;
 }
 
-static bool obs_nix_x11_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *context, obs_key_t key)
+static enum obs_hotkey_platform_pressed_state obs_nix_x11_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *context,
+										      obs_key_t key)
 {
 	xcb_connection_t *conn = XGetXCBConnection(context->display);
+	bool pressed;
 
 	if (key >= OBS_KEY_MOUSE1 && key <= OBS_KEY_MOUSE29) {
-		return mouse_button_pressed(conn, context, key);
+		pressed = mouse_button_pressed(conn, context, key);
 	} else {
-		return key_pressed(conn, context, key);
+		pressed = key_pressed(conn, context, key);
 	}
+
+	return pressed ? OBS_HOTKEY_PLATFORM_PRESSED : OBS_HOTKEY_PLATFORM_NOT_PRESSED;
 }
 
 static bool get_key_translation(struct dstr *dstr, xcb_keycode_t keycode)

libobs/obs-nix.c

@@ -443,7 +443,7 @@ void obs_hotkeys_platform_free(struct obs_core_hotkeys *hotkeys)
 	hotkeys_vtable = NULL;
 }
 
-bool obs_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *context, obs_key_t key)
+enum obs_hotkey_platform_pressed_state obs_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *context, obs_key_t key)
 {
 	return hotkeys_vtable->is_pressed(context, key);
 }

libobs/obs-nix.h

@@ -28,7 +28,7 @@ struct obs_nix_hotkeys_vtable {
 
 	void (*free)(struct obs_core_hotkeys *hotkeys);
 
-	bool (*is_pressed)(obs_hotkeys_platform_t *context, obs_key_t key);
+	enum obs_hotkey_platform_pressed_state (*is_pressed)(obs_hotkeys_platform_t *context, obs_key_t key);
 
 	void (*key_to_str)(obs_key_t key, struct dstr *dstr);
 

libobs/obs-windows.c

@@ -991,14 +991,18 @@ static bool vk_down(DWORD vk)
 	return down;
 }
 
-bool obs_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *context, obs_key_t key)
+enum obs_hotkey_platform_pressed_state obs_hotkeys_platform_is_pressed(obs_hotkeys_platform_t *context, obs_key_t key)
 {
+	bool pressed;
+
 	if (key == OBS_KEY_META) {
-		return vk_down(VK_LWIN) || vk_down(VK_RWIN);
+		pressed = vk_down(VK_LWIN) || vk_down(VK_RWIN);
+		return pressed ? OBS_HOTKEY_PLATFORM_PRESSED : OBS_HOTKEY_PLATFORM_NOT_PRESSED;
 	}
 
 	UNUSED_PARAMETER(context);
-	return vk_down(obs_key_to_virtual_key(key));
+	pressed = vk_down(obs_key_to_virtual_key(key));
+	return pressed ? OBS_HOTKEY_PLATFORM_PRESSED : OBS_HOTKEY_PLATFORM_NOT_PRESSED;
 }
 
 void obs_key_to_str(obs_key_t key, struct dstr *str)