mirror
/
nheko
mirror of https://github.com/Nheko-Reborn/nheko
2
0
Fork 1
Code Issues Releases Wiki Activity

Merge pull request #320 from trilene/webrtc-video

Browse Source 
Video calls: add local webcam view
pull/349/head
DeepBlueV7.X 4 years ago committed by GitHub
parent a2332934af d391404b79
commit 27bf654d92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 168 additions and 57 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. BIN
      resources/icons/ui/toggle-camera-view.png
  2. 16
      resources/qml/ActiveCallBar.qml
  3. 1
      resources/res.qrc
  4. 200
      src/WebRTCSession.cpp
  5. 1
      src/WebRTCSession.h
  6. 6
      src/timeline/TimelineViewManager.cpp
  7. 1
      src/timeline/TimelineViewManager.h

BIN
resources/icons/ui/toggle-camera-view.png
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 374 B

16
resources/qml/ActiveCallBar.qml
Unescape View File

@ -103,6 +103,22 @@ Rectangle {
Layout.fillWidth: true
}
ImageButton {
visible: TimelineManager.onVideoCall
width: 24
height: 24
buttonTextColor: "#000000"
image: ":/icons/icons/ui/toggle-camera-view.png"
hoverEnabled: true
ToolTip.visible: hovered
ToolTip.text: "Toggle camera view"
onClicked: TimelineManager.toggleCameraView()
}
Item {
implicitWidth: 8
}
ImageButton {
width: 24
height: 24

1
resources/res.qrc
Unescape View File

@ -74,6 +74,7 @@
<file>icons/ui/end-call.png</file>
<file>icons/ui/microphone-mute.png</file>
<file>icons/ui/microphone-unmute.png</file>
<file>icons/ui/toggle-camera-view.png</file>
<file>icons/ui/video-call.png</file>
<file>icons/emoji-categories/people.png</file>

200
src/WebRTCSession.cpp
Unescape View File

@ -103,6 +103,7 @@ bool haveAudioStream_;
bool haveVideoStream_;
std::vector<AudioSource> audioSources_;
std::vector<VideoSource> videoSources_;
GstPad *insetSinkPad_ = nullptr;
using FrameRate = std::pair<int, int>;
std::optional<FrameRate>
@ -496,6 +497,92 @@ setWaitForKeyFrame(GstBin *decodebin G_GNUC_UNUSED, GstElement *element, gpointe
}
#endif
GstElement *
newAudioSinkChain(GstElement *pipe)
{
GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *convert = gst_element_factory_make("audioconvert", nullptr);
GstElement *resample = gst_element_factory_make("audioresample", nullptr);
GstElement *sink = gst_element_factory_make("autoaudiosink", nullptr);
gst_bin_add_many(GST_BIN(pipe), queue, convert, resample, sink, nullptr);
gst_element_link_many(queue, convert, resample, sink, nullptr);
gst_element_sync_state_with_parent(queue);
gst_element_sync_state_with_parent(convert);
gst_element_sync_state_with_parent(resample);
gst_element_sync_state_with_parent(sink);
return queue;
}
GstElement *
newVideoSinkChain(GstElement *pipe)
{
// use compositor for now; acceleration needs investigation
GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *compositor = gst_element_factory_make("compositor", "compositor");
GstElement *glupload = gst_element_factory_make("glupload", nullptr);
GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr);
GstElement *qmlglsink = gst_element_factory_make("qmlglsink", nullptr);
GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr);
g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr);
g_object_set(glsinkbin, "sink", qmlglsink, nullptr);
gst_bin_add_many(
GST_BIN(pipe), queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr);
gst_element_link_many(queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr);
gst_element_sync_state_with_parent(queue);
gst_element_sync_state_with_parent(compositor);
gst_element_sync_state_with_parent(glupload);
gst_element_sync_state_with_parent(glcolorconvert);
gst_element_sync_state_with_parent(glsinkbin);
return queue;
}
std::pair<int, int>
getResolution(GstPad *pad)
{
std::pair<int, int> ret;
GstCaps *caps = gst_pad_get_current_caps(pad);
const GstStructure *s = gst_caps_get_structure(caps, 0);
gst_structure_get_int(s, "width", &ret.first);
gst_structure_get_int(s, "height", &ret.second);
gst_caps_unref(caps);
return ret;
}
void
addCameraView(GstElement *pipe, const std::pair<int, int> &videoCallSize)
{
GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee");
GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *videorate = gst_element_factory_make("videorate", nullptr);
gst_bin_add_many(GST_BIN(pipe), queue, videorate, nullptr);
gst_element_link_many(tee, queue, videorate, nullptr);
gst_element_sync_state_with_parent(queue);
gst_element_sync_state_with_parent(videorate);
gst_object_unref(tee);
GstElement *camerafilter = gst_bin_get_by_name(GST_BIN(pipe), "camerafilter");
GstPad *filtersinkpad = gst_element_get_static_pad(camerafilter, "sink");
auto cameraResolution = getResolution(filtersinkpad);
int insetWidth = videoCallSize.first / 4;
int insetHeight =
static_cast<double>(cameraResolution.second) / cameraResolution.first * insetWidth;
nhlog::ui()->debug("WebRTC: picture-in-picture size: {}x{}", insetWidth, insetHeight);
gst_object_unref(filtersinkpad);
gst_object_unref(camerafilter);
GstPad *camerapad = gst_element_get_static_pad(videorate, "src");
GstElement *compositor = gst_bin_get_by_name(GST_BIN(pipe), "compositor");
insetSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u");
g_object_set(insetSinkPad_, "zorder", 2, nullptr);
g_object_set(insetSinkPad_, "width", insetWidth, "height", insetHeight, nullptr);
gint offset = videoCallSize.first / 80;
g_object_set(insetSinkPad_, "xpos", offset, "ypos", offset, nullptr);
if (GST_PAD_LINK_FAILED(gst_pad_link(camerapad, insetSinkPad_)))
nhlog::ui()->error("WebRTC: failed to link camera view chain");
gst_object_unref(camerapad);
gst_object_unref(compositor);
}
void
linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
{
@ -511,51 +598,29 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
gst_object_unref(sinkpad);
WebRTCSession *session = &WebRTCSession::instance();
GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *queue = nullptr;
if (!std::strcmp(mediaType, "audio")) {
nhlog::ui()->debug("WebRTC: received incoming audio stream");
haveAudioStream_ = true;
GstElement *convert = gst_element_factory_make("audioconvert", nullptr);
GstElement *resample = gst_element_factory_make("audioresample", nullptr);
GstElement *sink = gst_element_factory_make("autoaudiosink", nullptr);
gst_bin_add_many(GST_BIN(pipe), queue, convert, resample, sink, nullptr);
gst_element_link_many(queue, convert, resample, sink, nullptr);
gst_element_sync_state_with_parent(queue);
gst_element_sync_state_with_parent(convert);
gst_element_sync_state_with_parent(resample);
gst_element_sync_state_with_parent(sink);
haveAudioStream_ = true;
queue = newAudioSinkChain(pipe);
} else if (!std::strcmp(mediaType, "video")) {
nhlog::ui()->debug("WebRTC: received incoming video stream");
if (!session->getVideoItem()) {
g_free(mediaType);
gst_object_unref(queue);
nhlog::ui()->error("WebRTC: video call item not set");
return;
}
haveVideoStream_ = true;
keyFrameRequestData_.statsField =
std::string("rtp-inbound-stream-stats_") + std::to_string(ssrc);
GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr);
GstElement *glupload = gst_element_factory_make("glupload", nullptr);
GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr);
GstElement *qmlglsink = gst_element_factory_make("qmlglsink", nullptr);
GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr);
g_object_set(qmlglsink, "widget", session->getVideoItem(), nullptr);
g_object_set(glsinkbin, "sink", qmlglsink, nullptr);
gst_bin_add_many(
GST_BIN(pipe), queue, videoconvert, glupload, glcolorconvert, glsinkbin, nullptr);
gst_element_link_many(
queue, videoconvert, glupload, glcolorconvert, glsinkbin, nullptr);
gst_element_sync_state_with_parent(queue);
gst_element_sync_state_with_parent(videoconvert);
gst_element_sync_state_with_parent(glupload);
gst_element_sync_state_with_parent(glcolorconvert);
gst_element_sync_state_with_parent(glsinkbin);
queue = newVideoSinkChain(pipe);
auto videoCallSize = getResolution(newpad);
nhlog::ui()->info("WebRTC: incoming video resolution: {}x{}",
videoCallSize.first,
videoCallSize.second);
addCameraView(pipe, videoCallSize);
} else {
g_free(mediaType);
gst_object_unref(queue);
nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad));
return;
}
@ -600,7 +665,7 @@ addDecodeBin(GstElement *webrtc G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe)
gst_element_sync_state_with_parent(decodebin);
GstPad *sinkpad = gst_element_get_static_pad(decodebin, "sink");
if (GST_PAD_LINK_FAILED(gst_pad_link(newpad, sinkpad)))
nhlog::ui()->error("WebRTC: unable to link new pad");
nhlog::ui()->error("WebRTC: unable to link decodebin");
gst_object_unref(sinkpad);
}
@ -689,7 +754,8 @@ WebRTCSession::havePlugins(bool isVideo, std::string *errorMessage)
"webrtc",
nullptr};
const gchar *videoPlugins[] = {"opengl", "qmlgl", "rtp", "videoconvert", "vpx", nullptr};
const gchar *videoPlugins[] = {
"compositor", "opengl", "qmlgl", "rtp", "videoconvert", "vpx", nullptr};
std::string strError("Missing GStreamer plugins: ");
const gchar **needed = isVideo ? videoPlugins : voicePlugins;
@ -729,6 +795,7 @@ WebRTCSession::createOffer(bool isVideo)
videoItem_ = nullptr;
haveAudioStream_ = false;
haveVideoStream_ = false;
insetSinkPad_ = nullptr;
localsdp_.clear();
localcandidates_.clear();
@ -752,6 +819,7 @@ WebRTCSession::acceptOffer(const std::string &sdp)
videoItem_ = nullptr;
haveAudioStream_ = false;
haveVideoStream_ = false;
insetSinkPad_ = nullptr;
localsdp_.clear();
localcandidates_.clear();
@ -974,6 +1042,7 @@ WebRTCSession::createPipeline(int opusPayloadType, int vp8PayloadType)
nhlog::ui()->error("WebRTC: failed to link audio pipeline elements");
return false;
}
return isVideo_ ? addVideoPipeline(vp8PayloadType) : true;
}
@ -984,8 +1053,9 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
if (videoSources_.empty())
return !isOffering_;
std::string cameraSetting = ChatPage::instance()->userSettings()->camera().toStdString();
auto it = std::find_if(videoSources_.cbegin(),
QSharedPointer<UserSettings> settings = ChatPage::instance()->userSettings();
std::string cameraSetting = settings->camera().toStdString();
auto it = std::find_if(videoSources_.cbegin(),
videoSources_.cend(),
[&cameraSetting](const auto &s) { return s.name == cameraSetting; });
if (it == videoSources_.cend()) {
@ -993,11 +1063,9 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
return false;
}
std::string resSetting =
ChatPage::instance()->userSettings()->cameraResolution().toStdString();
std::string resSetting = settings->cameraResolution().toStdString();
const std::string &res = resSetting.empty() ? it->caps.front().resolution : resSetting;
std::string frSetting =
ChatPage::instance()->userSettings()->cameraFrameRate().toStdString();
std::string frSetting = settings->cameraFrameRate().toStdString();
const std::string &fr = frSetting.empty() ? it->caps.front().frameRates.front() : frSetting;
auto resolution = tokenise(res, 'x');
auto frameRate = tokenise(fr, '/');
@ -1005,9 +1073,10 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
nhlog::ui()->debug("WebRTC: camera resolution: {}x{}", resolution.first, resolution.second);
nhlog::ui()->debug("WebRTC: camera frame rate: {}/{}", frameRate.first, frameRate.second);
GstElement *source = gst_device_create_element(it->device, nullptr);
GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr);
GstCaps *caps = gst_caps_new_simple("video/x-raw",
GstElement *source = gst_device_create_element(it->device, nullptr);
GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr);
GstElement *capsfilter = gst_element_factory_make("capsfilter", "camerafilter");
GstCaps *caps = gst_caps_new_simple("video/x-raw",
"width",
G_TYPE_INT,
resolution.first,
@ -1021,15 +1090,13 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
nullptr);
g_object_set(capsfilter, "caps", caps, nullptr);
gst_caps_unref(caps);
GstElement *convert = gst_element_factory_make("videoconvert", nullptr);
GstElement *queue1 = gst_element_factory_make("queue", nullptr);
GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr);
GstElement *tee = gst_element_factory_make("tee", "videosrctee");
GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr);
g_object_set(vp8enc, "deadline", 1, nullptr);
g_object_set(vp8enc, "error-resilient", 1, nullptr);
GstElement *rtp = gst_element_factory_make("rtpvp8pay", nullptr);
GstElement *queue2 = gst_element_factory_make("queue", nullptr);
GstElement *rtpvp8pay = gst_element_factory_make("rtpvp8pay", nullptr);
GstElement *rtpqueue = gst_element_factory_make("queue", nullptr);
GstElement *rtpcapsfilter = gst_element_factory_make("capsfilter", nullptr);
GstCaps *rtpcaps = gst_caps_new_simple("application/x-rtp",
"media",
@ -1047,27 +1114,30 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
gst_bin_add_many(GST_BIN(pipe_),
source,
videoconvert,
capsfilter,
convert,
queue1,
tee,
queue,
vp8enc,
rtp,
queue2,
rtpvp8pay,
rtpqueue,
rtpcapsfilter,
nullptr);
GstElement *webrtcbin = gst_bin_get_by_name(GST_BIN(pipe_), "webrtcbin");
if (!gst_element_link_many(source,
videoconvert,
capsfilter,
convert,
queue1,
tee,
queue,
vp8enc,
rtp,
queue2,
rtpvp8pay,
rtpqueue,
rtpcapsfilter,
webrtcbin,
nullptr)) {
nhlog::ui()->error("WebRTC: failed to link video pipeline elements");
gst_object_unref(webrtcbin);
return false;
}
gst_object_unref(webrtcbin);
@ -1101,6 +1171,16 @@ WebRTCSession::toggleMicMute()
return !muted;
}
void
WebRTCSession::toggleCameraView()
{
if (insetSinkPad_) {
guint zorder;
g_object_get(insetSinkPad_, "zorder", &zorder, nullptr);
g_object_set(insetSinkPad_, "zorder", zorder ? 0 : 2, nullptr);
}
}
void
WebRTCSession::end()
{
@ -1115,11 +1195,13 @@ WebRTCSession::end()
busWatchId_ = 0;
}
}
webrtc_ = nullptr;
isVideo_ = false;
isOffering_ = false;
isRemoteVideoRecvOnly_ = false;
videoItem_ = nullptr;
insetSinkPad_ = nullptr;
if (state_ != State::DISCONNECTED)
emit stateChanged(State::DISCONNECTED);
}
@ -1270,6 +1352,10 @@ WebRTCSession::toggleMicMute()
return false;
}
void
WebRTCSession::toggleCameraView()
{}
void
WebRTCSession::end()
{}

1
src/WebRTCSession.h
Unescape View File

@ -53,6 +53,7 @@ public:
bool isMicMuted() const;
bool toggleMicMute();
void toggleCameraView();
void end();
void setTurnServers(const std::vector<std::string> &uris) { turnServers_ = uris; }

6
src/timeline/TimelineViewManager.cpp
Unescape View File

@ -330,6 +330,12 @@ TimelineViewManager::toggleMicMute()
emit micMuteChanged();
}
void
TimelineViewManager::toggleCameraView()
{
WebRTCSession::instance().toggleCameraView();
}
void
TimelineViewManager::openImageOverlay(QString mxcUrl, QString eventId) const
{

1
src/timeline/TimelineViewManager.h
Unescape View File

@ -61,6 +61,7 @@ public:
QString callPartyAvatarUrl() const { return callManager_->callPartyAvatarUrl(); }
bool isMicMuted() const { return WebRTCSession::instance().isMicMuted(); }
Q_INVOKABLE void toggleMicMute();
Q_INVOKABLE void toggleCameraView();
Q_INVOKABLE void openImageOverlay(QString mxcUrl, QString eventId) const;
Q_INVOKABLE QColor userColor(QString id, QColor background);
Q_INVOKABLE QString escapeEmoji(QString str) const;

Write Preview
Loading…
Cancel
Save