-
Notifications
You must be signed in to change notification settings - Fork 33
Bugfix: Server-initiated room disconnect shutdown fix #205
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
8f5433c
d5dda8c
0b9ea06
9effbcb
738693b
0672226
7810fa3
9f56644
83c403b
a2e6a9e
e8569e3
7534b22
55299c3
8416551
a6a8d20
5edbf42
8bc9a2a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -212,7 +212,10 @@ bool Room::connect(const std::string& url, const std::string& token, const RoomO | |
|
|
||
| bool Room::disconnect(DisconnectReason reason) { | ||
| TRACE_EVENT0("livekit", "Room::disconnect"); | ||
| return shutdown(true, reason, true); | ||
| } | ||
|
|
||
| bool Room::shutdown(bool disconnect_ffi, DisconnectReason reason, bool notify_delegate) { | ||
| std::shared_ptr<FfiHandle> handle; | ||
| RoomDelegate* delegate_snapshot = nullptr; | ||
| std::shared_ptr<LocalParticipant> local_participant_to_cleanup; | ||
|
|
@@ -224,58 +227,82 @@ bool Room::disconnect(DisconnectReason reason) { | |
|
|
||
| { | ||
| const std::scoped_lock<std::mutex> g(lock_); | ||
| if (connection_state_ == ConnectionState::Disconnected) { | ||
| // Already torn down (or never connected). Nothing to do. | ||
| const bool has_room_state = connection_state_ != ConnectionState::Disconnected || listener_id_ != 0 || | ||
| room_handle_ || local_participant_ || !remote_participants_.empty(); | ||
| // Return false for a no-op so callers can tell whether this call claimed the | ||
| // room state and performed cleanup. Matches disconnect()'s documented contract. | ||
| if (!has_room_state) { | ||
| return false; | ||
| } | ||
| handle = room_handle_; | ||
| handle = std::move(room_handle_); | ||
| delegate_snapshot = delegate_; | ||
| // Take ownership of everything under the lock so the kEos handler (which | ||
| // also tries to move it out) loses any race here — only one teardown | ||
| // path operates on this state. | ||
| local_participant_to_cleanup = std::move(local_participant_); | ||
| remote_participants_to_clear = std::move(remote_participants_); | ||
| e2ee_manager_to_clear = std::move(e2ee_manager_); | ||
| text_stream_readers_to_clear = std::move(text_stream_readers_); | ||
| byte_stream_readers_to_clear = std::move(byte_stream_readers_); | ||
| listener_to_remove = listener_id_; | ||
| listener_id_ = 0; | ||
| room_handle_.reset(); | ||
| // Flip state immediately so the in-flight Disconnected room-event we'll | ||
| // get back doesn't double-fire onDisconnected. Mirrors Python's | ||
| // Room.disconnect() | ||
| connection_state_ = ConnectionState::Disconnected; | ||
| } | ||
|
|
||
| // Drain in-flight RPC handlers BEFORE telling Rust to tear down the room. | ||
| // Mirrors client-sdk-python's Room.disconnect() ordering | ||
| bool shutdown_ok = true; | ||
| if (local_participant_to_cleanup) { | ||
| local_participant_to_cleanup->shutdown(); | ||
| try { | ||
| local_participant_to_cleanup->shutdown(); | ||
| } catch (const std::exception& e) { | ||
| LK_LOG_ERROR("Room shutdown: local participant shutdown failed: {}", e.what()); | ||
| shutdown_ok = false; | ||
| } catch (...) { | ||
| LK_LOG_ERROR("Room shutdown: local participant shutdown failed: unknown exception"); | ||
| shutdown_ok = false; | ||
| } | ||
| } | ||
|
|
||
| // Tell the FFI to close the room and wait for the callback. If this fails | ||
| // we still complete local-side teardown below | ||
| bool ffi_ok = true; | ||
| if (handle) { | ||
| if (disconnect_ffi && handle && handle->valid()) { | ||
| try { | ||
| FfiClient::instance().disconnectAsync(handle->get(), reason).get(); | ||
| } catch (const std::exception& e) { | ||
| LK_LOG_ERROR("Room::disconnect: FFI disconnect failed (continuing local teardown): {}", e.what()); | ||
| ffi_ok = false; | ||
| LK_LOG_ERROR("Room shutdown: FFI disconnect failed (continuing local shutdown): {}", e.what()); | ||
| shutdown_ok = false; | ||
| } catch (...) { | ||
| LK_LOG_ERROR("Room shutdown: FFI disconnect failed (continuing local shutdown): unknown exception"); | ||
| shutdown_ok = false; | ||
| } | ||
| } | ||
|
|
||
| // Stop dispatcher so no track callbacks fire mid-teardown. | ||
| if (subscription_thread_dispatcher_) { | ||
| subscription_thread_dispatcher_->stopAll(); | ||
| try { | ||
| subscription_thread_dispatcher_->stopAll(); | ||
| } catch (const std::exception& e) { | ||
| LK_LOG_ERROR("Room shutdown: subscription shutdown failed: {}", e.what()); | ||
| shutdown_ok = false; | ||
| } catch (...) { | ||
| LK_LOG_ERROR("Room shutdown: subscription shutdown failed: unknown exception"); | ||
| shutdown_ok = false; | ||
| } | ||
| } | ||
|
|
||
| if (listener_to_remove != 0) { | ||
| FfiClient::instance().removeListener(listener_to_remove); | ||
| try { | ||
| FfiClient::instance().removeListener(listener_to_remove); | ||
| } catch (const std::exception& e) { | ||
| LK_LOG_ERROR("Room shutdown: listener removal failed: {}", e.what()); | ||
| shutdown_ok = false; | ||
| } catch (...) { | ||
| LK_LOG_ERROR("Room shutdown: listener removal failed: unknown exception"); | ||
| shutdown_ok = false; | ||
| } | ||
| } | ||
|
|
||
| // Fire onDisconnected exactly once, with the reason the caller passed. | ||
| if (delegate_snapshot) { | ||
| local_participant_to_cleanup.reset(); | ||
| remote_participants_to_clear.clear(); | ||
| e2ee_manager_to_clear.reset(); | ||
| text_stream_readers_to_clear.clear(); | ||
| byte_stream_readers_to_clear.clear(); | ||
| handle.reset(); | ||
|
|
||
| if (notify_delegate && delegate_snapshot) { | ||
| DisconnectedEvent ev; | ||
| ev.reason = reason; | ||
| try { | ||
|
|
@@ -287,9 +314,7 @@ bool Room::disconnect(DisconnectReason reason) { | |
| } | ||
| } | ||
|
|
||
| // Moved-out state (local participant, remote participants, e2ee manager, | ||
| // stream readers) destructs here, releasing FFI handles. | ||
| return ffi_ok; | ||
| return shutdown_ok; | ||
| } | ||
|
|
||
| RoomInfoData Room::roomInfo() const { | ||
|
|
@@ -1168,20 +1193,17 @@ void Room::onEvent(const FfiEvent& event) { | |
| break; | ||
| } | ||
| case proto::RoomEvent::kDisconnected: { | ||
| // If disconnect() was driven from our side, it already flipped state | ||
| // to Disconnected and fired the delegate; skip the duplicate here. | ||
| bool already_disconnected = false; | ||
| bool should_notify = false; | ||
| { | ||
| const std::scoped_lock<std::mutex> guard(lock_); | ||
| already_disconnected = (connection_state_ == ConnectionState::Disconnected); | ||
| connection_state_ = ConnectionState::Disconnected; | ||
| } | ||
| if (already_disconnected) { | ||
| break; | ||
| } | ||
| DisconnectedEvent ev; | ||
| ev.reason = toDisconnectReason(re.disconnected().reason()); | ||
| if (delegate_snapshot) { | ||
| // Local shutdown marks the state before awaiting the FFI response | ||
| // and notifies the delegate itself. Suppress that duplicate while | ||
| // passing server-initiated disconnects through unchanged. | ||
| should_notify = connection_state_ != ConnectionState::Disconnected; | ||
| } | ||
| if (should_notify && delegate_snapshot) { | ||
| DisconnectedEvent ev; | ||
| ev.reason = toDisconnectReason(re.disconnected().reason()); | ||
| delegate_snapshot->onDisconnected(*this, ev); | ||
| } | ||
| break; | ||
|
devin-ai-integration[bot] marked this conversation as resolved.
Comment on lines
+1196
to
1209
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔴 Server disconnect notification can fire twice when a user disconnect races with a server disconnect event The disconnect notification is decided ( Impact: The application's disconnect handler runs twice with different reasons, which can cause double-cleanup, incorrect state transitions, or crashes in user code that assumes exactly-one semantics. Race window between kDisconnected handler and user disconnect()The old code at the base commit always set The new code at bool should_notify = false;
{
const std::scoped_lock<std::mutex> guard(lock_);
should_notify = connection_state_ != ConnectionState::Disconnected;
// ← state is NOT updated here
}
if (should_notify && delegate_snapshot) {
delegate_snapshot->onDisconnected(*this, ev);
}Race sequence:
Result: delegate receives two Prompt for agentsWas this helpful? React with 👍 or 👎 to provide feedback. |
||
|
|
@@ -1208,56 +1230,7 @@ void Room::onEvent(const FfiEvent& event) { | |
| break; | ||
| } | ||
| case proto::RoomEvent::kEos: { | ||
| if (subscription_thread_dispatcher_) { | ||
| subscription_thread_dispatcher_->stopAll(); | ||
| } | ||
|
|
||
| int listener_to_remove = 0; | ||
|
|
||
| // Move state out of lock scope before destroying to avoid holding lock | ||
| // during potentially long destructors | ||
| std::shared_ptr<LocalParticipant> old_local_participant; | ||
| std::unordered_map<std::string, std::shared_ptr<RemoteParticipant>> old_remote_participants; | ||
| std::shared_ptr<FfiHandle> old_room_handle; | ||
| std::shared_ptr<E2EEManager> old_e2ee_manager; | ||
| std::unordered_map<std::string, std::shared_ptr<TextStreamReader>> old_text_readers; | ||
| std::unordered_map<std::string, std::shared_ptr<ByteStreamReader>> old_byte_readers; | ||
|
|
||
| { | ||
| const std::scoped_lock<std::mutex> guard(lock_); | ||
| listener_to_remove = listener_id_; | ||
| listener_id_ = 0; | ||
|
|
||
| // Reset connection state | ||
| connection_state_ = ConnectionState::Disconnected; | ||
|
|
||
| // Move state out for cleanup outside lock | ||
| old_local_participant = std::move(local_participant_); | ||
| old_remote_participants = std::move(remote_participants_); | ||
| old_room_handle = std::move(room_handle_); | ||
| old_e2ee_manager = std::move(e2ee_manager_); | ||
| old_text_readers = std::move(text_stream_readers_); | ||
| old_byte_readers = std::move(byte_stream_readers_); | ||
| } | ||
|
|
||
| // Drain in-flight RPC invocations before destroying the local | ||
| // participant's FFI handle. Mirrors the ordering in disconnect(); | ||
| // without this, a listener-thread RPC handler can race with handle | ||
| // disposal and send to a dead handle → INVALID_HANDLE → terminate. | ||
| if (old_local_participant) { | ||
| old_local_participant->shutdown(); | ||
| } | ||
|
|
||
| // Remove listener outside lock | ||
| if (listener_to_remove != 0) { | ||
| FfiClient::instance().removeListener(listener_to_remove); | ||
| } | ||
|
|
||
| if (old_local_participant) { | ||
| old_local_participant->shutdown(); | ||
| } | ||
|
|
||
| // old_* state is destroyed here when going out of scope | ||
| (void)shutdown(false, DisconnectReason::Unknown, false); | ||
|
|
||
| const RoomEosEvent ev; | ||
| if (delegate_snapshot) { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| /* | ||
| * Copyright 2026 LiveKit | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <gtest/gtest.h> | ||
|
|
||
| #include <cstdint> | ||
| #include <string> | ||
|
|
||
| #include "ffi.pb.h" | ||
| #include "ffi_client.h" | ||
|
|
||
| namespace livekit::test { | ||
|
|
||
| /// Serializes and dispatches a synthetic FFI event through the real callback entry point. | ||
| /// Defined in this header for use across different tests. | ||
| inline void emitFfiEvent(const proto::FfiEvent& event) { | ||
| std::string bytes; | ||
| ASSERT_TRUE(event.SerializeToString(&bytes)); | ||
| ffiEventCallback(reinterpret_cast<const std::uint8_t*>(bytes.data()), bytes.size()); | ||
| } | ||
|
|
||
| } // namespace livekit::test |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should we return true if nothing to do or it is being torndown ?
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm open to changing this, right now our public API for
disconnect()states:Which this matches,
false== noop. I get how that could be confusing, but I think the idea istrueis when the disconnect actually happenedThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That is fair. We can keep the current behavior to avoid breaking changes.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
FWIW we have a public
connectionStateso apps can check the state before trying to disconnect