/* * Copyright (C) 2020 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "host/frontend/webrtc/lib/streamer.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "host/frontend/webrtc/lib/audio_device.h" #include "host/frontend/webrtc/lib/audio_track_source_impl.h" #include "host/frontend/webrtc/lib/client_handler.h" #include "host/frontend/webrtc/lib/port_range_socket_factory.h" #include "host/frontend/webrtc/lib/video_track_source_impl.h" #include "host/frontend/webrtc/lib/vp8only_encoder_factory.h" #include "host/frontend/webrtc_operator/constants/signaling_constants.h" namespace cuttlefish { namespace webrtc_streaming { namespace { constexpr auto kStreamIdField = "stream_id"; constexpr auto kXResField = "x_res"; constexpr auto kYResField = "y_res"; constexpr auto kDpiField = "dpi"; constexpr auto kIsTouchField = "is_touch"; constexpr auto kDisplaysField = "displays"; constexpr auto kAudioStreamsField = "audio_streams"; constexpr auto kHardwareField = "hardware"; constexpr auto kControlPanelButtonCommand = "command"; constexpr auto kControlPanelButtonTitle = "title"; constexpr auto kControlPanelButtonIconName = "icon_name"; constexpr auto kControlPanelButtonShellCommand = "shell_command"; constexpr auto kControlPanelButtonDeviceStates = "device_states"; constexpr auto kControlPanelButtonLidSwitchOpen = "lid_switch_open"; constexpr auto kControlPanelButtonHingeAngleValue = "hinge_angle_value"; constexpr auto kCustomControlPanelButtonsField = "custom_control_panel_buttons"; void SendJson(WsConnection* ws_conn, const Json::Value& data) { Json::StreamWriterBuilder factory; auto data_str = Json::writeString(factory, data); ws_conn->Send(reinterpret_cast(data_str.c_str()), data_str.size()); } bool ParseMessage(const uint8_t* data, size_t length, Json::Value* msg_out) { auto str = reinterpret_cast(data); Json::CharReaderBuilder builder; std::unique_ptr json_reader(builder.newCharReader()); std::string errorMessage; return json_reader->parse(str, str + length, msg_out, &errorMessage); } std::unique_ptr CreateAndStartThread(const std::string& name) { auto thread = rtc::Thread::CreateWithSocketServer(); if (!thread) { LOG(ERROR) << "Failed to create " << name << " thread"; return nullptr; } thread->SetName(name, nullptr); if (!thread->Start()) { LOG(ERROR) << "Failed to start " << name << " thread"; return nullptr; } return thread; } struct DisplayDescriptor { int width; int height; int dpi; bool touch_enabled; rtc::scoped_refptr source; }; struct ControlPanelButtonDescriptor { std::string command; std::string title; std::string icon_name; std::optional shell_command; std::vector device_states; }; // TODO (jemoreira): move to a place in common with the signaling server struct OperatorServerConfig { std::vector servers; }; // Wraps a scoped_refptr pointer to an audio device module class AudioDeviceModuleWrapper : public AudioSource { public: AudioDeviceModuleWrapper( rtc::scoped_refptr device_module) : device_module_(device_module) {} int GetMoreAudioData(void* data, int bytes_per_sample, int samples_per_channel, int num_channels, int sample_rate, bool& muted) override { return device_module_->GetMoreAudioData(data, bytes_per_sample, samples_per_channel, num_channels, sample_rate, muted); } rtc::scoped_refptr device_module() { return device_module_; } private: rtc::scoped_refptr device_module_; }; } // namespace class Streamer::Impl : public WsConnectionObserver { public: std::shared_ptr CreateClientHandler(int client_id); void SendMessageToClient(int client_id, const Json::Value& msg); void DestroyClientHandler(int client_id); // WsObserver void OnOpen() override; void OnClose() override; void OnError(const std::string& error) override; void OnReceive(const uint8_t* msg, size_t length, bool is_binary) override; void HandleConfigMessage(const Json::Value& msg); void HandleClientMessage(const Json::Value& server_message); // All accesses to these variables happen from the signal_thread, so there is // no need for extra synchronization mechanisms (mutex) StreamerConfig config_; OperatorServerConfig operator_config_; std::shared_ptr server_connection_; std::shared_ptr connection_observer_factory_; rtc::scoped_refptr peer_connection_factory_; std::unique_ptr network_thread_; std::unique_ptr worker_thread_; std::unique_ptr signal_thread_; std::map displays_; std::map> audio_sources_; std::map> clients_; std::weak_ptr operator_observer_; std::map hardware_; std::vector custom_control_panel_buttons_; std::shared_ptr audio_device_module_; }; Streamer::Streamer(std::unique_ptr impl) : impl_(std::move(impl)) {} /* static */ std::unique_ptr Streamer::Create( const StreamerConfig& cfg, std::shared_ptr connection_observer_factory) { rtc::LogMessage::LogToDebug(rtc::LS_ERROR); std::unique_ptr impl(new Streamer::Impl()); impl->config_ = cfg; impl->connection_observer_factory_ = connection_observer_factory; impl->network_thread_ = CreateAndStartThread("network-thread"); impl->worker_thread_ = CreateAndStartThread("work-thread"); impl->signal_thread_ = CreateAndStartThread("signal-thread"); if (!impl->network_thread_ || !impl->worker_thread_ || !impl->signal_thread_) { return nullptr; } impl->audio_device_module_ = std::make_shared( rtc::scoped_refptr( new rtc::RefCountedObject())); impl->peer_connection_factory_ = webrtc::CreatePeerConnectionFactory( impl->network_thread_.get(), impl->worker_thread_.get(), impl->signal_thread_.get(), impl->audio_device_module_->device_module(), webrtc::CreateBuiltinAudioEncoderFactory(), webrtc::CreateBuiltinAudioDecoderFactory(), std::make_unique( webrtc::CreateBuiltinVideoEncoderFactory()), webrtc::CreateBuiltinVideoDecoderFactory(), nullptr /* audio_mixer */, nullptr /* audio_processing */); if (!impl->peer_connection_factory_) { LOG(ERROR) << "Failed to create peer connection factory"; return nullptr; } webrtc::PeerConnectionFactoryInterface::Options options; // By default the loopback network is ignored, but generating candidates for // it is useful when using TCP port forwarding. options.network_ignore_mask = 0; impl->peer_connection_factory_->SetOptions(options); return std::unique_ptr(new Streamer(std::move(impl))); } std::shared_ptr Streamer::AddDisplay(const std::string& label, int width, int height, int dpi, bool touch_enabled) { // Usually called from an application thread return impl_->signal_thread_->Invoke>( RTC_FROM_HERE, [this, &label, width, height, dpi, touch_enabled]() -> std::shared_ptr { if (impl_->displays_.count(label)) { LOG(ERROR) << "Display with same label already exists: " << label; return nullptr; } rtc::scoped_refptr source( new rtc::RefCountedObject(width, height)); impl_->displays_[label] = {width, height, dpi, touch_enabled, source}; return std::shared_ptr( new VideoTrackSourceImplSinkWrapper(source)); }); } std::shared_ptr Streamer::AddAudioStream(const std::string& label) { // Usually called from an application thread return impl_->signal_thread_->Invoke>( RTC_FROM_HERE, [this, &label]() -> std::shared_ptr { if (impl_->audio_sources_.count(label)) { LOG(ERROR) << "Audio stream with same label already exists: " << label; return nullptr; } rtc::scoped_refptr source( new rtc::RefCountedObject()); impl_->audio_sources_[label] = source; return std::shared_ptr( new AudioTrackSourceImplSinkWrapper(source)); }); } std::shared_ptr Streamer::GetAudioSource() { return impl_->audio_device_module_; } void Streamer::SetHardwareSpec(std::string key, std::string value) { impl_->hardware_.emplace(key, value); } void Streamer::AddCustomControlPanelButton(const std::string& command, const std::string& title, const std::string& icon_name) { ControlPanelButtonDescriptor button = { .command = command, .title = title, .icon_name = icon_name}; impl_->custom_control_panel_buttons_.push_back(button); } void Streamer::AddCustomControlPanelButtonWithShellCommand( const std::string& command, const std::string& title, const std::string& icon_name, const std::string& shell_command) { ControlPanelButtonDescriptor button = { .command = command, .title = title, .icon_name = icon_name}; button.shell_command = shell_command; impl_->custom_control_panel_buttons_.push_back(button); } void Streamer::AddCustomControlPanelButtonWithDeviceStates( const std::string& command, const std::string& title, const std::string& icon_name, const std::vector& device_states) { ControlPanelButtonDescriptor button = { .command = command, .title = title, .icon_name = icon_name}; button.device_states = device_states; impl_->custom_control_panel_buttons_.push_back(button); } void Streamer::Register(std::weak_ptr observer) { // Usually called from an application thread // No need to block the calling thread on this, the observer will be notified // when the connection is established. impl_->signal_thread_->PostTask(RTC_FROM_HERE, [this, observer]() { impl_->operator_observer_ = observer; // This can be a local variable since the connection object will keep a // reference to it. auto ws_context = WsConnectionContext::Create(); CHECK(ws_context) << "Failed to create websocket context"; impl_->server_connection_ = ws_context->CreateConnection( impl_->config_.operator_server.port, impl_->config_.operator_server.addr, impl_->config_.operator_server.path, impl_->config_.operator_server.security, impl_, impl_->config_.operator_server.http_headers); CHECK(impl_->server_connection_) << "Unable to create websocket connection object"; impl_->server_connection_->Connect(); }); } void Streamer::Unregister() { // Usually called from an application thread. impl_->signal_thread_->PostTask( RTC_FROM_HERE, [this]() { impl_->server_connection_.reset(); }); } void Streamer::RecordDisplays(LocalRecorder& recorder) { for (auto& [key, display] : impl_->displays_) { rtc::scoped_refptr source = display.source; auto deleter = [](webrtc::VideoTrackSourceInterface* source) { source->Release(); }; std::shared_ptr source_shared( source.release(), deleter); recorder.AddDisplay(display.width, display.height, source_shared); } } void Streamer::Impl::OnOpen() { // Called from the websocket thread. // Connected to operator. signal_thread_->PostTask(RTC_FROM_HERE, [this]() { Json::Value register_obj; register_obj[cuttlefish::webrtc_signaling::kTypeField] = cuttlefish::webrtc_signaling::kRegisterType; register_obj[cuttlefish::webrtc_signaling::kDeviceIdField] = config_.device_id; Json::Value device_info; Json::Value displays(Json::ValueType::arrayValue); // No need to synchronize with other accesses to display_ because all // happens on signal_thread. for (auto& entry : displays_) { Json::Value display; display[kStreamIdField] = entry.first; display[kXResField] = entry.second.width; display[kYResField] = entry.second.height; display[kDpiField] = entry.second.dpi; display[kIsTouchField] = true; displays.append(display); } device_info[kDisplaysField] = displays; Json::Value audio_streams(Json::ValueType::arrayValue); for (auto& entry : audio_sources_) { Json::Value audio; audio[kStreamIdField] = entry.first; audio_streams.append(audio); } device_info[kAudioStreamsField] = audio_streams; Json::Value hardware; for (const auto& [k, v] : hardware_) { hardware[k] = v; } device_info[kHardwareField] = hardware; Json::Value custom_control_panel_buttons(Json::arrayValue); for (const auto& button : custom_control_panel_buttons_) { Json::Value button_entry; button_entry[kControlPanelButtonCommand] = button.command; button_entry[kControlPanelButtonTitle] = button.title; button_entry[kControlPanelButtonIconName] = button.icon_name; if (button.shell_command) { button_entry[kControlPanelButtonShellCommand] = *(button.shell_command); } else if (!button.device_states.empty()) { Json::Value device_states(Json::arrayValue); for (const DeviceState& device_state : button.device_states) { Json::Value device_state_entry; if (device_state.lid_switch_open) { device_state_entry[kControlPanelButtonLidSwitchOpen] = *device_state.lid_switch_open; } if (device_state.hinge_angle_value) { device_state_entry[kControlPanelButtonHingeAngleValue] = *device_state.hinge_angle_value; } device_states.append(device_state_entry); } button_entry[kControlPanelButtonDeviceStates] = device_states; } custom_control_panel_buttons.append(button_entry); } device_info[kCustomControlPanelButtonsField] = custom_control_panel_buttons; register_obj[cuttlefish::webrtc_signaling::kDeviceInfoField] = device_info; SendJson(server_connection_.get(), register_obj); // Do this last as OnRegistered() is user code and may take some time to // complete (although it shouldn't...) auto observer = operator_observer_.lock(); if (observer) { observer->OnRegistered(); } }); } void Streamer::Impl::OnClose() { // Called from websocket thread // The operator shouldn't close the connection with the client, it's up to the // device to decide when to disconnect. LOG(WARNING) << "Websocket closed unexpectedly"; signal_thread_->PostTask(RTC_FROM_HERE, [this]() { auto observer = operator_observer_.lock(); if (observer) { observer->OnClose(); } }); } void Streamer::Impl::OnError(const std::string& error) { // Called from websocket thread. LOG(ERROR) << "Error on connection with the operator: " << error; signal_thread_->PostTask(RTC_FROM_HERE, [this]() { auto observer = operator_observer_.lock(); if (observer) { observer->OnError(); } }); } void Streamer::Impl::HandleConfigMessage(const Json::Value& server_message) { CHECK(signal_thread_->IsCurrent()) << __FUNCTION__ << " called from the wrong thread"; if (server_message.isMember("ice_servers") && server_message["ice_servers"].isArray()) { auto servers = server_message["ice_servers"]; operator_config_.servers.clear(); for (int server_idx = 0; server_idx < servers.size(); server_idx++) { auto server = servers[server_idx]; webrtc::PeerConnectionInterface::IceServer ice_server; if (!server.isMember("urls") || !server["urls"].isArray()) { // The urls field is required LOG(WARNING) << "Invalid ICE server specification obtained from server: " << server.toStyledString(); continue; } auto urls = server["urls"]; for (int url_idx = 0; url_idx < urls.size(); url_idx++) { auto url = urls[url_idx]; if (!url.isString()) { LOG(WARNING) << "Non string 'urls' field in ice server: " << url.toStyledString(); continue; } ice_server.urls.push_back(url.asString()); if (server.isMember("credential") && server["credential"].isString()) { ice_server.password = server["credential"].asString(); } if (server.isMember("username") && server["username"].isString()) { ice_server.username = server["username"].asString(); } operator_config_.servers.push_back(ice_server); } } } } void Streamer::Impl::HandleClientMessage(const Json::Value& server_message) { CHECK(signal_thread_->IsCurrent()) << __FUNCTION__ << " called from the wrong thread"; if (!server_message.isMember(cuttlefish::webrtc_signaling::kClientIdField) || !server_message[cuttlefish::webrtc_signaling::kClientIdField].isInt()) { LOG(ERROR) << "Client message received without valid client id"; return; } auto client_id = server_message[cuttlefish::webrtc_signaling::kClientIdField].asInt(); if (!server_message.isMember(cuttlefish::webrtc_signaling::kPayloadField)) { LOG(WARNING) << "Received empty client message"; return; } auto client_message = server_message[cuttlefish::webrtc_signaling::kPayloadField]; if (clients_.count(client_id) == 0) { auto client_handler = CreateClientHandler(client_id); if (!client_handler) { LOG(ERROR) << "Failed to create a new client handler"; return; } clients_.emplace(client_id, client_handler); } auto client_handler = clients_[client_id]; client_handler->HandleMessage(client_message); } void Streamer::Impl::OnReceive(const uint8_t* msg, size_t length, bool is_binary) { // Usually called from websocket thread. Json::Value server_message; // Once OnReceive returns the buffer can be destroyed/recycled at any time, so // parse the data into a JSON object while still on the websocket thread. if (is_binary || !ParseMessage(msg, length, &server_message)) { LOG(ERROR) << "Received invalid JSON from server: '" << (is_binary ? std::string("(binary_data)") : std::string(msg, msg + length)) << "'"; return; } // Transition to the signal thread before member variables are accessed. signal_thread_->PostTask(RTC_FROM_HERE, [this, server_message]() { if (!server_message.isMember(cuttlefish::webrtc_signaling::kTypeField) || !server_message[cuttlefish::webrtc_signaling::kTypeField].isString()) { LOG(ERROR) << "No message_type field from server"; // Notify the caller OnError( "Invalid message received from operator: no message type field " "present"); return; } auto type = server_message[cuttlefish::webrtc_signaling::kTypeField].asString(); if (type == cuttlefish::webrtc_signaling::kConfigType) { HandleConfigMessage(server_message); } else if (type == cuttlefish::webrtc_signaling::kClientDisconnectType) { if (!server_message.isMember( cuttlefish::webrtc_signaling::kClientIdField) || !server_message.isMember( cuttlefish::webrtc_signaling::kClientIdField)) { LOG(ERROR) << "Invalid disconnect message received from server"; // Notify the caller OnError("Invalid disconnect message: client_id is required"); return; } auto client_id = server_message[cuttlefish::webrtc_signaling::kClientIdField].asInt(); LOG(INFO) << "Client " << client_id << " has disconnected."; DestroyClientHandler(client_id); } else if (type == cuttlefish::webrtc_signaling::kClientMessageType) { HandleClientMessage(server_message); } else { LOG(ERROR) << "Unknown message type: " << type; // Notify the caller OnError("Invalid message received from operator: unknown message type"); return; } }); } std::shared_ptr Streamer::Impl::CreateClientHandler( int client_id) { CHECK(signal_thread_->IsCurrent()) << __FUNCTION__ << " called from the wrong thread"; auto observer = connection_observer_factory_->CreateObserver(); auto client_handler = ClientHandler::Create( client_id, observer, [this, client_id](const Json::Value& msg) { SendMessageToClient(client_id, msg); }, [this, client_id] { DestroyClientHandler(client_id); }); webrtc::PeerConnectionInterface::RTCConfiguration config; config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; config.enable_dtls_srtp = true; config.servers.insert(config.servers.end(), operator_config_.servers.begin(), operator_config_.servers.end()); webrtc::PeerConnectionDependencies dependencies(client_handler.get()); // PortRangeSocketFactory's super class' constructor needs to be called on the // network thread or have it as a parameter dependencies.packet_socket_factory.reset(new PortRangeSocketFactory( network_thread_.get(), config_.udp_port_range, config_.tcp_port_range)); auto peer_connection = peer_connection_factory_->CreatePeerConnection( config, std::move(dependencies)); if (!peer_connection) { LOG(ERROR) << "Failed to create peer connection"; return nullptr; } if (!client_handler->SetPeerConnection(std::move(peer_connection))) { return nullptr; } for (auto& entry : displays_) { auto& label = entry.first; auto& video_source = entry.second.source; auto video_track = peer_connection_factory_->CreateVideoTrack(label, video_source.get()); client_handler->AddDisplay(video_track, label); } for (auto& entry : audio_sources_) { auto& label = entry.first; auto& audio_stream = entry.second; auto audio_track = peer_connection_factory_->CreateAudioTrack(label, audio_stream.get()); client_handler->AddAudio(audio_track, label); } return client_handler; } void Streamer::Impl::SendMessageToClient(int client_id, const Json::Value& msg) { LOG(VERBOSE) << "Sending to client: " << msg.toStyledString(); CHECK(signal_thread_->IsCurrent()) << __FUNCTION__ << " called from the wrong thread"; Json::Value wrapper; wrapper[cuttlefish::webrtc_signaling::kPayloadField] = msg; wrapper[cuttlefish::webrtc_signaling::kTypeField] = cuttlefish::webrtc_signaling::kForwardType; wrapper[cuttlefish::webrtc_signaling::kClientIdField] = client_id; // This is safe to call from the webrtc threads because // WsConnection is thread safe SendJson(server_connection_.get(), wrapper); } void Streamer::Impl::DestroyClientHandler(int client_id) { // Usually called from signal thread, could be called from websocket thread or // an application thread. signal_thread_->PostTask(RTC_FROM_HERE, [this, client_id]() { // This needs to be 'posted' to the thread instead of 'invoked' // immediately for two reasons: // * The client handler is destroyed by this code, it's generally a // bad idea (though not necessarily wrong) to return to a member // function of a destroyed object. // * The client handler may call this from within a peer connection // observer callback, destroying the client handler there leads to a // deadlock. clients_.erase(client_id); }); } } // namespace webrtc_streaming } // namespace cuttlefish