Added Direct Play and websocket improvements

* [backend]: Websocket now automatically tries to reconnect if connection was lost, up to 3 times.
* [backend]: Move more playback and resume logic to the backend, to avoid having it in multiple places within the QML. Regression: pausing playback sometimes halts the video player for an unknown reason.
* [playback]: Sailfin will try to play without the server transcoding, if possible.
* [ui]: added a debug page in the settings

core/include/JellyfinQt/jellyfinapiclient.h

@@ -82,6 +82,7 @@ public:
     Q_PROPERTY(QString userId READ userId NOTIFY userIdChanged)
     Q_PROPERTY(QJsonObject deviceProfile READ deviceProfile NOTIFY deviceProfileChanged)
     Q_PROPERTY(QString version READ version)
+    Q_PROPERTY(WebSocket *websocket READ websocket NOTIFY websocketChanged)
 
     /*QNetworkReply *handleRequest(QString path, QStringList sort, Pagination *pagination,
                                  QVariantMap filters, QStringList fields, QStringList expand, QString id);*/
@@ -111,6 +112,7 @@ public:
     QJsonObject &deviceProfile() { return m_deviceProfile; }
     QJsonObject &playbackDeviceProfile() { return m_playbackDeviceProfile; }
     QString version() const;
+    WebSocket *websocket() const { return m_webSocket; }
 
     /**
      * @brief Sets the error handler of a reply to this classes default error handler
@@ -146,10 +148,8 @@ signals:
 
     void userIdChanged(QString userId);
 
-    void itemFetched(const QString &itemId, const QJsonObject &result);
-    void itemFetchFailed(const QString &itemId, const QNetworkReply::NetworkError error);
-
     void deviceProfileChanged();
+    void websocketChanged(WebSocket *newWebsocket);
 
     /**
      * @brief onUserDataChanged Emitted when the user data of an item is changed on the server.
@@ -179,8 +179,6 @@ public slots:
      */
     void deleteSession();
 
-    void fetchItem(const QString &id);
-
     /**
      * @brief Shares the capabilities of this device to the server.
      */

core/include/JellyfinQt/jellyfinitem.h

@@ -367,6 +367,7 @@ public:
     Q_PROPERTY(QJsonObject imageTags MEMBER m_imageTags NOTIFY imageTagsChanged)
     Q_PROPERTY(QStringList backdropImageTags MEMBER m_backdropImageTags NOTIFY backdropImageTagsChanged)
     Q_PROPERTY(QJsonObject imageBlurHashes MEMBER m_imageBlurHashes NOTIFY imageBlurHashesChanged)
+    Q_PROPERTY(QString mediaType MEMBER m_mediaType READ mediaType NOTIFY mediaTypeChanged)
     Q_PROPERTY(int width MEMBER m_width NOTIFY widthChanged)
     Q_PROPERTY(int height MEMBER m_height NOTIFY heightChanged)
 
@@ -406,6 +407,7 @@ public:
     void setRecursiveItemCount(int newRecursiveItemCount) { m_recursiveItemCount = newRecursiveItemCount; emit recursiveItemCountChanged(newRecursiveItemCount); }
     int childCount() const { return m_childCount.value_or(-1); }
     void setChildCount(int newChildCount) { m_childCount = newChildCount; emit childCountChanged(newChildCount); }
+    QString mediaType() const { return m_mediaType; }
 
     //QQmlListProperty<MediaStream> mediaStreams() { return toReadOnlyQmlListProperty<MediaStream>(m_mediaStreams); }
     //QList<QObject *> mediaStreams() { return *reinterpret_cast<QList<QObject *> *>(&m_mediaStreams); }
@@ -460,6 +462,7 @@ signals:
     void imageTagsChanged();
     void backdropImageTagsChanged();
     void imageBlurHashesChanged();
+    void mediaTypeChanged(const QString &newMediaType);
     void widthChanged(int newWidth);
     void heightChanged(int newHeight);
 
@@ -520,6 +523,7 @@ protected:
     QJsonObject m_imageTags;
     QStringList m_backdropImageTags;
     QJsonObject m_imageBlurHashes;
+    QString m_mediaType;
     int m_width;
     int m_height;
 

core/include/JellyfinQt/jellyfinplaybackmanager.h

@@ -65,6 +65,7 @@ public:
     Q_PROPERTY(int subtitleIndex MEMBER m_subtitleIndex NOTIFY subtitleIndexChanged)
     Q_PROPERTY(bool resumePlayback MEMBER m_resumePlayback NOTIFY resumePlaybackChanged)
     Q_PROPERTY(QObject* mediaPlayer READ mediaPlayer WRITE setMediaPlayer NOTIFY mediaPlayerChanged)
+    Q_PROPERTY(PlayMethod playMethod READ playMethod NOTIFY playMethodChanged)
 
     Item *item() const { return m_item; }
     void setItem(Item *newItem);
@@ -75,6 +76,7 @@ public:
     void setMediaPlayer(QObject *qmlMediaPlayer);
 
     QString streamUrl() const { return m_streamUrl; }
+    PlayMethod playMethod() const { return m_playMethod; }
 signals:
     void itemChanged(Item *newItemId);
     void streamUrlChanged(const QString &newStreamUrl);
@@ -83,6 +85,7 @@ signals:
     void subtitleIndexChanged(int subtitleIndex);
     void mediaPlayerChanged(QObject *newMediaPlayer);
     void resumePlaybackChanged(bool newResumePlayback);
+    void playMethodChanged(PlayMethod newPlayMethod);
 
 public slots:
     void updatePlaybackInfo();
@@ -103,7 +106,7 @@ private:
     qint64 m_oldPosition = 0;
     qint64 m_stopPosition = 0;
     QMediaPlayer::State m_oldState = QMediaPlayer::StoppedState;
-    PlayMethod m_playMethod;
+    PlayMethod m_playMethod = Transcode;
     QObject *m_qmlMediaPlayer = nullptr;
     QMediaPlayer * m_mediaPlayer = nullptr;
     bool m_resumePlayback = true;

core/include/JellyfinQt/jellyfinwebsocket.h

@@ -58,7 +58,12 @@ public:
         KeepAlive,
         UserDataChanged
     };
+    Q_PROPERTY(QAbstractSocket::SocketState state READ state NOTIFY stateChanged)
     Q_ENUM(MessageType)
+
+    QAbstractSocket::SocketState state() const {
+        return m_webSocket.state();
+    }
 public slots:
     void open();
 private slots:
@@ -67,14 +72,18 @@ private slots:
     void onDisconnected();
 
     void sendKeepAlive();
+    void onWebsocketStateChanged(QAbstractSocket::SocketState newState) { emit stateChanged(newState); }
 signals:
     void commandReceived(QString arts, QVariantMap args);
+    void stateChanged(QAbstractSocket::SocketState newState);
 
 protected:
     ApiClient *m_apiClient;
     QWebSocket m_webSocket;
 
     QTimer m_keepAliveTimer;
+    QTimer m_retryTimer;
+    int m_reconnectAttempt = 0;
 
 
     void setupKeepAlive(int data);

core/src/jellyfinapiclient.cpp

@@ -25,6 +25,8 @@ ApiClient::ApiClient(QObject *parent)
     : QObject(parent),
       m_webSocket(new WebSocket(this)) {
     m_deviceName = QHostInfo::localHostName();
+    uint uuid1 = qHash(m_deviceName);
+    uint uuid2 = qHash(QSysInfo::productVersion());
     m_deviceId = QUuid::createUuid().toString(); // TODO: make this not random?
     m_credManager = CredentialsManager::newInstance(this);
 
@@ -218,23 +220,6 @@ void ApiClient::deleteSession() {
     });
 }
 
-void ApiClient::fetchItem(const QString &id) {
-    QNetworkReply *rep = get("/Users/" + m_userId + "/Items/" + id);
-    connect(rep, &QNetworkReply::finished, this, [rep, id, this]() {
-        int status = statusCode(rep);
-        if (status >= 200 && status < 300) {
-            QJsonObject data = QJsonDocument::fromJson(rep->readAll()).object();
-            emit this->itemFetched(id, data);
-        }
-        rep->deleteLater();
-    });
-    connect(rep, static_cast<void (QNetworkReply::*)(QNetworkReply::NetworkError)>(&QNetworkReply::error),
-            this, [id, rep, this](QNetworkReply::NetworkError error) {
-        emit this->itemFetchFailed(id, error);
-        rep->deleteLater();
-    });
-}
-
 void ApiClient::postCapabilities() {
     QJsonObject capabilities;
     capabilities["SupportsPersistentIdentifier"] = false; // Technically untrue, but not implemented yet.

core/src/jellyfindeviceprofile.cpp

@@ -46,6 +46,18 @@ QJsonObject DeviceProfile::generateProfile() {
     using JsonPair = QPair<QString, QJsonValue>;
     QJsonObject profile;
 
+    QStringList audioCodes = {
+        "aac",
+        "flac",
+        "mp2",
+        "mp3",
+        "oga"
+        "ogg",
+        "wav",
+        "webma",
+        "wma"
+    };
+
     QStringList videoAudioCodecs;
     QStringList mp4VideoCodecs;
     QStringList hlsVideoCodecs;
@@ -171,6 +183,7 @@ QJsonObject DeviceProfile::generateProfile() {
                             }));
 
     // Direct play profiles
+    // Video
     QJsonArray directPlayProfiles;
     directPlayProfiles.append(QJsonObject {
                                   JsonPair("Container", "mp4,m4v"),
@@ -178,6 +191,39 @@ QJsonObject DeviceProfile::generateProfile() {
                                   JsonPair("VideoCodec", mp4VideoCodecs.join(',')),
                                   JsonPair("AudioCodec", videoAudioCodecs.join(','))
                               });
+    directPlayProfiles.append(QJsonObject {
+                                  JsonPair("Container", "mkv"),
+                                  JsonPair("Type", "Video"),
+                                  JsonPair("VideoCodec", mp4VideoCodecs.join(',')),
+                                  JsonPair("AudioCodec", videoAudioCodecs.join(','))
+                              });
+
+    // Audio
+    for (auto it = audioCodes.begin(); it != audioCodes.end(); it++) {
+        if (*it == "mp2") {
+            directPlayProfiles.append(QJsonObject {
+                                          JsonPair("Container", "mp2,mp3"),
+                                          JsonPair("Type", "Audio"),
+                                          JsonPair("AudioCodec", "mp2")
+                                      });
+        } else if(*it == "mp3") {
+            directPlayProfiles.append(QJsonObject {
+                                          JsonPair("Container", "mp3"),
+                                          JsonPair("Type", "Audio"),
+                                          JsonPair("AudioCodec", "mp3")
+                                      });
+        } else if (*it == "webma") {
+            directPlayProfiles.append(QJsonObject {
+                                          JsonPair("Container", "webma,webm"),
+                                          JsonPair("Type", "Audio"),
+                                      });
+        } else {
+            directPlayProfiles.append(QJsonObject {
+                                          JsonPair("Container", *it),
+                                          JsonPair("Type", "Audio")
+                                      });
+        }
+    }
 
     profile["CodecProfiles"] = codecProfiles;
     profile["ContainerProfiles"] = QJsonArray();

core/src/jellyfinplaybackmanager.cpp

@@ -29,7 +29,10 @@ PlaybackManager::PlaybackManager(QObject *parent)
 }
 
 void PlaybackManager::fetchStreamUrl() {
-    if (m_item == nullptr || m_apiClient == nullptr) return;
+    if (m_item == nullptr || m_apiClient == nullptr) {
+        qDebug() << "Item or apiClient not set";
+        return;
+    }
     m_resumePosition = 0;
     if (m_resumePlayback && !m_item->property("userData").isNull()) {
         UserData* userData = qvariant_cast<UserData *>(m_item->property("userData"));
@@ -57,12 +60,36 @@ void PlaybackManager::fetchStreamUrl() {
 
         if (this->m_autoOpen) {
             QJsonArray mediaSources = root["MediaSources"].toArray();
+            QJsonObject firstMediaSource = mediaSources[0].toObject();
             //FIXME: relies on the fact that the returned transcode url always has a query!
-            QString streamUrl = this->m_apiClient->baseUrl()
+            if (firstMediaSource.isEmpty()) {
-                    + mediaSources[0].toObject()["TranscodingUrl"].toString();
+                qWarning() << "No media source found";
+            } else if (firstMediaSource["SupportsDirectStream"].toBool()) {
+                QUrlQuery query;
+                query.addQueryItem("mediaSourceId", firstMediaSource["Id"].toString());
+                query.addQueryItem("deviceId", m_apiClient->m_deviceId);
+                query.addQueryItem("api_key", m_apiClient->token());
+                query.addQueryItem("Static", "True");
+                QString mediaType = "unknown";
+                if (m_item->mediaType() == "Audio") {
+                    mediaType = "Audio";
+                } else if (m_item->mediaType() == "Video") {
+                    mediaType = "Videos";
+                }
+                QString streamUrl = this->m_apiClient->baseUrl() + "/" + mediaType + "/" + m_item->jellyfinId() + "/stream."
+                        + firstMediaSource["Container"].toString() + "?" + query.toString(QUrl::EncodeReserved);
+                setStreamUrl(streamUrl);
+                this->m_playMethod = DirectPlay;
+            } else if (firstMediaSource["SupportsTranscoding"].toBool() && !firstMediaSource["TranscodingUrl"].isNull()) {
+                QString streamUrl = this->m_apiClient->baseUrl()
+                        + firstMediaSource["TranscodingUrl"].toString();
 
-            this->m_playMethod = Transcode;
+                this->m_playMethod = Transcode;
-            setStreamUrl(streamUrl);
+                setStreamUrl(streamUrl);
+            } else {
+                qDebug() << "No stream url found";
+                return;
+            }
             qDebug() << "Found stream url: " << this->m_streamUrl;
         }
 
@@ -71,6 +98,7 @@ void PlaybackManager::fetchStreamUrl() {
 }
 
 void PlaybackManager::setItem(Item *newItem) {
+    if (m_mediaPlayer != nullptr) m_mediaPlayer->stop();
     this->m_item = newItem;
     // Don't try to start fetching when we're not completely parsed yet.
     if (m_qmlIsParsingComponent) return;
@@ -81,8 +109,14 @@ void PlaybackManager::setItem(Item *newItem) {
     }
     // Deinitialize the streamUrl
     setStreamUrl("");
-    if (newItem != nullptr && !newItem->jellyfinId().isEmpty()) {
+    if (newItem != nullptr) {
-        fetchStreamUrl();
+        if (m_item->status() == RemoteData::Ready) {
+            fetchStreamUrl();
+        } else {
+            connect(m_item, &RemoteData::ready, [this]() -> void {
+                fetchStreamUrl();
+            });
+        }
     }
 }
 
@@ -161,6 +195,10 @@ void PlaybackManager::updatePlaybackInfo() {
 void PlaybackManager::postPlaybackInfo(PlaybackInfoType type) {
     QJsonObject root;
 
+    if (m_item == nullptr) {
+        qWarning() << "Item is null. Not posting playback info";
+        return;
+    }
     root["ItemId"] = m_item->jellyfinId();
     root["SessionId"] = m_playSessionId;
 
@@ -204,6 +242,7 @@ void PlaybackManager::postPlaybackInfo(PlaybackInfoType type) {
 
 void PlaybackManager::componentComplete() {
     if (m_apiClient == nullptr) qWarning() << "No ApiClient set for PlaybackManager";
+    m_qmlIsParsingComponent = false;
     if (m_item != nullptr) {
         if (m_item->status() == RemoteData::Ready) {
             fetchStreamUrl();

core/src/jellyfinwebsocket.cpp

@@ -28,6 +28,9 @@ WebSocket::WebSocket(ApiClient *client)
         Q_UNUSED(error)
         qDebug() << "Connection error: " <<  m_webSocket.errorString();
     });
+    connect(&m_webSocket, &QWebSocket::stateChanged, this, &WebSocket::onWebsocketStateChanged);
+    connect(&m_keepAliveTimer, &QTimer::timeout, this, &WebSocket::sendKeepAlive);
+    connect(&m_retryTimer, &QTimer::timeout, this, &WebSocket::open);
 }
 
 void WebSocket::open() {
@@ -39,16 +42,22 @@ void WebSocket::open() {
     connectionUrl.setPath("/socket");
     connectionUrl.setQuery(query);
     m_webSocket.open(connectionUrl);
-    qDebug() << "Opening WebSocket connection to " << m_webSocket.requestUrl();
+    m_reconnectAttempt++;
+    qDebug() << "Opening WebSocket connection to " << m_webSocket.requestUrl() << ", connect attempt " << m_reconnectAttempt;
 }
 
 void WebSocket::onConnected() {
     connect(&m_webSocket, &QWebSocket::textMessageReceived, this, &WebSocket::textMessageReceived);
+    m_reconnectAttempt = 0;
 }
 
 void WebSocket::onDisconnected() {
     disconnect(&m_webSocket, &QWebSocket::textMessageReceived, this, &WebSocket::textMessageReceived);
     m_keepAliveTimer.stop();
+    if (m_reconnectAttempt <= 3) {
+        // 500, 2500, 12500
+        m_retryTimer.setInterval(100 * static_cast<int>(std::pow(5., m_reconnectAttempt)));
+    }
 }
 
 void WebSocket::textMessageReceived(const QString &message) {
@@ -112,7 +121,6 @@ void WebSocket::setupKeepAlive(int data) {
     // Data is timeout in seconds, we want to send a keepalive at half the timeout
     m_keepAliveTimer.setInterval(data * 500);
     m_keepAliveTimer.setSingleShot(false);
-    connect(&m_keepAliveTimer, &QTimer::timeout, this, &WebSocket::sendKeepAlive);
     m_keepAliveTimer.start();
     sendKeepAlive();
 }

sailfish/qml/components/VideoPlayer.qml

@@ -31,15 +31,15 @@ import "../"
 
 SilicaItem {
     id: playerRoot
-    property alias item : mediaSource.item
+    property JellyfinItem item
     property string title: item.name
-    property alias resume: mediaSource.resumePlayback
+    property bool resume
     property int progress
     readonly property bool landscape: videoOutput.contentRect.width > videoOutput.contentRect.height
     property MediaPlayer player
     readonly property bool hudVisible: !hud.hidden || player.error !== MediaPlayer.NoError
-    property alias audioTrack: mediaSource.audioIndex
+    property int audioTrack: 0
-    property alias subtitleTrack: mediaSource.subtitleIndex
+    property int subtitleTrack: 0
 
     // Blackground to prevent the ambience from leaking through
     Rectangle {
@@ -47,13 +47,6 @@ SilicaItem {
         color: Theme.overlayBackgroundColor
     }
 
-    PlaybackManager {
-        id: mediaSource
-        apiClient: ApiClient
-        mediaPlayer: player
-        autoOpen: true
-    }
-
     VideoOutput {
         id: videoOutput
         source: player
@@ -69,7 +62,8 @@ SilicaItem {
         Label {
             anchors.fill: parent
             anchors.margins: Theme.horizontalPageMargin
-            text: item.jellyfinId + "\n" + mediaSource.streamUrl + "\n"
+            text: item.jellyfinId + "\n" + appWindow.playbackManager.streamUrl + "\n"
+                  + (appWindow.playbackManager.playMethod == PlaybackManager.DirectPlay ? "Direct Play" : "Transcoding") + "\n"
                   + player.position + "\n"
                   + player.status + "\n"
                   + player.bufferProgress + "\n"
@@ -78,7 +72,7 @@ SilicaItem {
                   + player.errorString + "\n"
             font.pixelSize: Theme.fontSizeExtraSmall
             wrapMode: "WordWrap"
-            visible: true
+            visible: appWindow.showDebugInfo
         }
     }
 
@@ -87,6 +81,13 @@ SilicaItem {
         player: playerRoot.player
     }
 
+    function start() {
+        appWindow.playbackManager.audioIndex = audioTrack
+        appWindow.playbackManager.subtitleIndex = subtitleTrack
+        appWindow.playbackManager.resumePlayback = resume
+        appWindow.playbackManager.item = item
+    }
+
     function stop() {
         player.stop()
     }

sailfish/qml/harbour-sailfin.qml

@@ -33,12 +33,16 @@ ApplicationWindow {
     property bool _hasInitialized: false
     // The global mediaPlayer instance
     readonly property MediaPlayer mediaPlayer: _mediaPlayer
+    readonly property PlaybackManager playbackManager: _playbackManager
 
     // Data of the currently selected item. For use on the cover.
     property JellyfinItem itemData
     // Id of the collection currently browsing. For use on the cover.
     property string collectionId
 
+    // Bad way to implement settings, but it'll do for now.
+    property bool showDebugInfo: false
+
     //FIXME: proper error handling
     Connections {
         target: ApiClient
@@ -98,6 +102,14 @@ ApplicationWindow {
         autoPlay: true
     }
 
+    PlaybackManager {
+        id: _playbackManager
+        apiClient: ApiClient
+        mediaPlayer: _mediaPlayer
+        audioIndex: 0
+        autoOpen: true
+    }
+
     // Keep the sytem alive while playing media
     KeepAlive {
         enabled: _mediaPlayer.playbackState == MediaPlayer.PlayingState

sailfish/qml/pages/VideoPage.qml

@@ -54,6 +54,7 @@ Page {
             //appWindow.orientation = landscape ? Orientation.Landscape : Orientation.Portrait
             videoPage.allowedOrientations = landscape ? Orientation.LandscapeMask : Orientation.PortraitMask
         }
+
     }
 
     onStatusChanged: {
@@ -62,6 +63,7 @@ Page {
             videoPlayer.stop()
             break;
         case PageStatus.Active:
+            videoPlayer.start()
             appWindow.itemData = videoPage.itemData
             break;
         }

sailfish/qml/pages/itemdetails/MusicAlbumPage.qml

@@ -73,10 +73,13 @@ BaseDetailPage {
                 }
             }
             delegate: SongDelegate {
+                id: songDelegate
                 name: model.name
                 artists: model.artists
                 duration: model.runTimeTicks
                 indexNumber: model.indexNumber
+                onClicked: window.playbackManager.item = Qt.createQmlObject("import nl.netsoj.chris.Jellyfin 1.0;"
+                    + "JellyfinItem { jellyfinId: \"" + model.id + "\"; apiClient: ApiClient; }", songDelegate, "nonexistent.qml");
             }
 
             VerticalScrollDecorator {}

sailfish/qml/pages/settings/DebugPage.qml

@@ -39,6 +39,55 @@ Page {
                 title: qsTr("Debug information")
             }
 
+            TextSwitch {
+                text: qsTr("Show debug information")
+                checked: appWindow.showDebugInfo
+                onCheckedChanged: appWindow.showDebugInfo = checked
+            }
+
+            SectionHeader {
+                text: qsTr("Websocket")
+            }
+
+            DetailItem {
+                label: qsTr("Connection state")
+                value: {
+                    var stateText
+                    switch( ApiClient.websocket.state) {
+                    case 0:
+                        //- Socket state
+                        stateText = qsTr("Unconnected");
+                        break;
+                    case 1:
+                        //- Socket state
+                        stateText = "Looking up host";
+                        break;
+                    case 2:
+                        //- Socket state
+                        stateText = "Connecting";
+                        break;
+                    case 3:
+                        //- Socket state
+                        stateText = "Connected";
+                        break;
+                    case 4:
+                        //- Socket state
+                        stateText = "Bound";
+                        break;
+                    case 5:
+                        //- Socket state
+                        stateText = "Closing";
+                        break;
+                    case 6:
+                        //- Socket state
+                        stateText = "Listening";
+                        break;
+                    }
+                    //- Socket state: "state no (state description)"
+                    qsTr("%1 (%2)").arg(ApiClient.websocket.state).arg(stateText)
+                }
+            }
+
             SectionHeader {
                 text: qsTr("Device profile")
             }