Merge pull request #240 from Stivius/feat/improved_proof_of_play

Improved Proof Of Play

.github/workflows/appimage.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Installing main dependencies
         run: |
           sudo apt-get update
-          sudo apt-get -y install software-properties-common apt-transport-https ca-certificates libglibmm-2.4-dev libssl-dev
+          sudo apt-get -y install software-properties-common apt-transport-https ca-certificates libglibmm-2.4-dev libssl-dev libsqlite3-dev
           sudo apt-get -y install gnupg curl libcurl4-gnutls-dev wget unzip libgtkmm-3.0-dev libwebkitgtk-3.0-dev libxss-dev
       - name: Installing newer boost
         run: |
@@ -112,6 +112,14 @@ jobs:
           cmake . -DBUILD_TZ_LIB=ON -DBUILD_SHARED_LIBS=ON -DUSE_SYSTEM_TZ_DB=ON
           make -j4
           sudo make install
+      - name: Installing sqlite-orm
+        run: |
+          curl -o sqlite-orm.tar.gz -SL https://github.com/fnc12/sqlite_orm/archive/refs/tags/1.6.tar.gz
+          tar -zxvf sqlite-orm.tar.gz 
+          cd sqlite_orm-1.6
+          cmake . -DBUILD_TESTING=OFF
+          make -j4
+          sudo make install
       - name: Update ldconfig cache
         run: |
           sudo ldconfig

.gitignore

@@ -10,3 +10,5 @@ pch.py
 *.json
 commits_config
 snapcraft.login
+.history
+third_party

player/XiboApp.cpp

@@ -13,7 +13,7 @@
 #include "control/screenshot/ScreenShotInterval.hpp"
 
 #include "schedule/Scheduler.hpp"
-#include "stat/StatsRecorder.hpp"
+#include "stat/Recorder.hpp"
 #include "xmr/XmrManager.hpp"
 
 #include "cms/CollectionInterval.hpp"
@@ -47,7 +47,7 @@ XiboApp::XiboApp(const std::string& name) :
     mainLoop_(std::make_unique<MainLoop>(name)),
     fileCache_(std::make_unique<FileCacheImpl>()),
     scheduler_(std::make_unique<Scheduler>(*fileCache_)),
-    statsRecorder_(std::make_unique<StatsRecorder>()),
+    statsRecorder_(std::make_unique<Stats::Recorder>()),
     webserver_(std::make_shared<LocalWebServer>())
 {
     if (!FileSystem::exists(AppConfig::cmsSettingsPath()))

player/XiboApp.hpp

@@ -21,7 +21,10 @@ class PlayerError;
 class XmrManager;
 using ApplicationWindowGtk = ApplicationWindow<WindowGtk>;
 class LocalWebServer;
-class StatsRecorder;
+namespace Stats
+{
+    class Recorder;
+}
 class LayoutsManager;
 class XiboApp;
 struct PlayerSettings;
@@ -59,7 +62,7 @@ class XiboApp : boost::noncopyable
     std::unique_ptr<CollectionInterval> collectionInterval_;
     std::unique_ptr<ScreenShotInterval> screenShotInterval_;
     std::unique_ptr<XmdsRequestSender> xmdsManager_;
-    std::unique_ptr<StatsRecorder> statsRecorder_;
+    std::unique_ptr<Stats::Recorder> statsRecorder_;
     std::unique_ptr<XmrManager> xmrManager_;
     std::shared_ptr<ApplicationWindowGtk> mainWindow_;
     std::shared_ptr<LocalWebServer> webserver_;

player/cms/CollectionInterval.cpp

@@ -3,6 +3,7 @@
 #include "MainLoop.hpp"
 
 #include "NotifyStatusInfo.hpp"
+#include "common/PlayerRuntimeError.hpp"
 #include "common/dt/DateTime.hpp"
 #include "common/dt/Timer.hpp"
 #include "common/fs/FileSystem.hpp"
@@ -14,13 +15,13 @@
 #include "config/AppConfig.hpp"
 
 #include "cms/xmds/XmdsRequestSender.hpp"
-#include "stat/StatsFormatter.hpp"
-#include "stat/StatsRecorder.hpp"
+#include "stat/Recorder.hpp"
+#include "stat/records/XmlFormatter.hpp"
 
 namespace ph = std::placeholders;
 
 CollectionInterval::CollectionInterval(XmdsRequestSender& xmdsSender,
-                                       StatsRecorder& statsRecorder,
+                                       Stats::Recorder& statsRecorder,
                                        FileCache& fileCache,
                                        const FilePath& resourceDirectory) :
     xmdsSender_{xmdsSender},
@@ -96,25 +97,9 @@ void CollectionInterval::onDisplayRegistered(const ResponseResult<RegisterDispla
             onSchedule(scheduleResult);
             onRequiredFiles(requiredFilesResult);
 
-            XmlLogsRetriever logsRetriever;
-            auto submitLogsResult = xmdsSender_.submitLogs(logsRetriever.retrieveLogs()).get();
-            onSubmitted("SubmitLogs", submitLogsResult);
-
-            if (!statsRecorder_.empty())
-            {
-                auto submitStatsResult = xmdsSender_.submitStats(statsRecorder_.records().string()).get();
-                statsRecorder_.clear();
-                onSubmitted("SubmitStats", submitStatsResult);
-            }
-
-            NotifyStatusInfo notifyInfo;
-            // FIXME: store it in collection interval until XMDS refactoring
-            notifyInfo.currentLayoutId = currentLayoutId_;
-            notifyInfo.deviceName = System::hostname();
-            notifyInfo.spaceUsageInfo = FileSystem::storageUsageFor(resourceDirectory_);
-            notifyInfo.timezone = DateTime::currentTimezone();
-            auto notifyStatusResult = xmdsSender_.notifyStatus(notifyInfo.string()).get();
-            onSubmitted("NotifyStatus", notifyStatusResult);
+            submitLogs();
+            submitStats();
+            notifyStatus();
         }
         sessionFinished(displayError);
     }
@@ -237,6 +222,57 @@ void CollectionInterval::onSchedule(const ResponseResult<Schedule::Result>& sche
     }
 }
 
+void CollectionInterval::submitLogs()
+{
+    XmlLogsRetriever logsRetriever;
+    auto submitLogsResult = xmdsSender_.submitLogs(logsRetriever.retrieveLogs()).get();
+    onSubmitted("SubmitLogs", submitLogsResult);
+}
+
+void CollectionInterval::submitStats()
+{
+    try
+    {
+        const auto recordsCount = statsRecorder_.recordsCount();
+        if (recordsCount > 0)
+        {
+            const auto RecordsToSend = [recordsCount]() -> size_t {
+                if (recordsCount > 500)
+                    return 300;
+                else
+                    return recordsCount > 50 ? 50 : recordsCount;
+            }();
+
+            Log::debug("[CollectionInterval] Total records: {} Records to send {}", recordsCount, RecordsToSend);
+
+            auto records = statsRecorder_.records(RecordsToSend);
+            statsRecorder_.removeFromQueue(RecordsToSend);
+
+            Stats::XmlFormatter formatter;
+            auto submitStatsResult = xmdsSender_.submitStats(formatter.format(records)).get();
+            onSubmitted("SubmitStats", submitStatsResult);
+        }
+    }
+    catch (const std::exception& e)
+    {
+        Log::error(e.what());
+        Log::error("[CollectionInterval] Failed to submit stats");
+    }
+}
+
+void CollectionInterval::notifyStatus()
+{
+    NotifyStatusInfo notifyInfo;
+    // FIXME: store it in collection interval until XMDS refactoring
+    notifyInfo.currentLayoutId = currentLayoutId_;
+    notifyInfo.deviceName = System::hostname();
+    notifyInfo.spaceUsageInfo = FileSystem::storageUsageFor(resourceDirectory_);
+    notifyInfo.timezone = DateTime::currentTimezone();
+
+    auto notifyStatusResult = xmdsSender_.notifyStatus(notifyInfo.string()).get();
+    onSubmitted("NotifyStatus", notifyStatusResult);
+}
+
 template <typename Result>
 void CollectionInterval::onSubmitted(std::string_view requestName, const ResponseResult<Result>& submitResult)
 {

player/cms/CollectionInterval.hpp

@@ -25,7 +25,10 @@ using SignalCollectionFinished = boost::signals2::signal<void(const PlayerError&
 using SignalFilesDownloaded = boost::signals2::signal<void()>;
 
 class XmdsRequestSender;
-class StatsRecorder;
+namespace Stats
+{
+    class Recorder;
+}
 class FileCache;
 
 class CollectionInterval
@@ -34,7 +37,7 @@ class CollectionInterval
 
 public:
     CollectionInterval(XmdsRequestSender& xmdsSender,
-                       StatsRecorder& statsRecorder,
+                       Stats::Recorder& statsRecorder,
                        FileCache& fileCache,
                        const FilePath& resourceDirectory);
 
@@ -60,13 +63,15 @@ class CollectionInterval
     void onRequiredFiles(const ResponseResult<RequiredFiles::Result>& requiredFiles);
     void updateMediaInventory(const RequiredFiles::Result& requiredFilesResult);
     void onSchedule(const ResponseResult<Schedule::Result>& schedule);
-    void onSubmitStats(const ResponseResult<SubmitStats::Result>& statsResult);
+    void submitLogs();
+    void submitStats();
+    void notifyStatus();
     template <typename Result>
     void onSubmitted(std::string_view requestName, const ResponseResult<Result>& submitResult);
 
 private:
     XmdsRequestSender& xmdsSender_;
-    StatsRecorder& statsRecorder_;
+    Stats::Recorder& statsRecorder_;
     FileCache& fileCache_;
     std::unique_ptr<JoinableThread> workerThread_;
     std::unique_ptr<Timer> intervalTimer_;

player/config/AppConfig.cpp

@@ -94,6 +94,11 @@ FilePath AppConfig::cachePath()
     return configDirectory() / "cacheFile.xml";
 }
 
+FilePath AppConfig::statsCache()
+{
+    return configDirectory() / "stats.sqlite";
+}
+
 FilePath AppConfig::additionalResourcesDirectory()
 {
 #if defined(SNAP_ENABLED)

player/config/AppConfig.hpp

@@ -32,6 +32,7 @@ class AppConfig
     static FilePath playerSettingsPath();
     static FilePath schedulePath();
     static FilePath cachePath();
+    static FilePath statsCache();
 
     static std::string playerBinary();
     static std::string optionsBinary();

player/control/layout/CMakeLists.txt

@@ -15,4 +15,5 @@ add_library(${PROJECT_NAME}
 
 target_link_libraries(${PROJECT_NAME}
     widgets
+    stat
 )

player/control/layout/LayoutsManager.cpp

@@ -7,18 +7,15 @@
 #include "common/logger/Logging.hpp"
 #include "common/storage/FileCache.hpp"
 #include "schedule/Scheduler.hpp"
-#include "stat/StatsRecorder.hpp"
+#include "stat/Recorder.hpp"
 
 #include "config/AppConfig.hpp"
 
 LayoutsManager::LayoutsManager(Scheduler& scheduler,
-                               StatsRecorder& statsRecorder,
+                               Stats::Recorder& statsRecorder,
                                FileCache& fileCache,
                                bool statsEnabled) :
-    scheduler_(scheduler),
-    statsRecorder_(statsRecorder),
-    fileCache_(fileCache),
-    statsEnabled_{statsEnabled}
+    scheduler_(scheduler), statsRecorder_(statsRecorder), fileCache_(fileCache), statsEnabled_{statsEnabled}
 {
     scheduler_.layoutUpdated().connect(std::bind(&LayoutsManager::fetchMainLayout, this));
     scheduler_.overlaysUpdated().connect(std::bind(&LayoutsManager::fetchOverlays, this));
@@ -96,12 +93,29 @@ std::unique_ptr<Xibo::MainLayout> LayoutsManager::createLayout(int layoutId)
         auto layout = parser.parseBy(layoutId);
         auto scheduleId = scheduler_.scheduleIdBy(layoutId);
 
-        layout->statReady().connect(
-            [=](const PlayingStat& stat) { statsRecorder_.addLayoutStat(scheduleId, layoutId, stat); });
-        layout->mediaStatsReady().connect([=](const MediaPlayingStats& stats) {
-            for (auto&& [mediaId, interval] : stats)
+        layout->statReady().connect([this, layoutId, scheduleId](const Stats::PlayingTime& interval) {
+            try
             {
-                statsRecorder_.addMediaStat(scheduleId, layoutId, mediaId, interval);
+                statsRecorder_.addLayoutRecord(Stats::LayoutRecord::create(scheduleId, layoutId, interval));
+            }
+            catch (const std::exception& e)
+            {
+                Log::error(e.what());
+            }
+        });
+        layout->mediaStatsReady().connect([this, layoutId, scheduleId](const MediaPlayingTime& intervals) {
+            try
+            {
+                Stats::MediaRecords records;
+                for (auto&& [mediaId, interval] : intervals)
+                {
+                    records.add(Stats::MediaRecord::create(scheduleId, layoutId, mediaId, interval));
+                }
+                statsRecorder_.addMediaRecords(std::move(records));
+            }
+            catch (const std::exception& e)
+            {
+                Log::error(e.what());
             }
         });
 

player/control/layout/LayoutsManager.hpp

@@ -8,15 +8,18 @@
 
 class Scheduler;
 class FileCache;
-class StatsRecorder;
+namespace Stats
+{
+    class Recorder;
+}
 
 using MainLayoutLoaded = boost::signals2::signal<void(const std::shared_ptr<Xibo::Widget>&)>;
 using OverlaysLoaded = boost::signals2::signal<void(const std::vector<std::shared_ptr<Xibo::Widget>>&)>;
 
 class LayoutsManager
 {
 public:
-    LayoutsManager(Scheduler& scheduler, StatsRecorder& statsRecorder, FileCache& fileCache, bool statsEnabled);
+    LayoutsManager(Scheduler& scheduler, Stats::Recorder& statsRecorder, FileCache& fileCache, bool statsEnabled);
 
     void fetchMainLayout();
     void fetchOverlays();
@@ -31,7 +34,7 @@ class LayoutsManager
 
 private:
     Scheduler& scheduler_;
-    StatsRecorder& statsRecorder_;
+    Stats::Recorder& statsRecorder_;
     FileCache& fileCache_;
     bool statsEnabled_;
 

player/control/layout/MainLayout.hpp

@@ -2,15 +2,15 @@
 
 #include "control/region/Region.hpp"
 #include "control/widgets/Image.hpp"
-#include "stat/PlayingStat.hpp"
+#include "stat/PlayingTime.hpp"
 
 #include <boost/signals2/signal.hpp>
 #include <memory>
 
 using SignalLayoutExpired = boost::signals2::signal<void()>;
-using SignalLayoutStatReady = boost::signals2::signal<void(const PlayingStat&)>;
-using MediaPlayingStats = std::multimap<int, PlayingStat>;
-using SignalLayoutMediaStatsReady = boost::signals2::signal<void(const MediaPlayingStats&)>;
+using SignalLayoutStatReady = boost::signals2::signal<void(const Stats::PlayingTime&)>;
+using MediaPlayingTime = std::multimap<int, Stats::PlayingTime>;
+using SignalLayoutMediaStatsReady = boost::signals2::signal<void(const MediaPlayingTime&)>;
 
 namespace Xibo
 {

player/control/layout/MainLayoutImpl.cpp

@@ -8,8 +8,7 @@
 namespace ph = std::placeholders;
 
 MainLayoutImpl::MainLayoutImpl(const MainLayoutOptions& options) :
-    options_(options),
-    view_(OverlayContainerFactory::create(options.width, options.height))
+    options_(options), view_(OverlayContainerFactory::create(options.width, options.height))
 {
     view_->shown().connect(std::bind(&MainLayoutImpl::startLayout, this));
 }
@@ -39,7 +38,7 @@ void MainLayoutImpl::monitorMediaStats(Xibo::Region& region)
     for (auto&& media : region.mediaList())
     {
         media->statReady().connect(
-            [id = media->id(), this](const PlayingStat& interval) { mediaIntervals_.emplace(id, interval); });
+            [id = media->id(), this](const Stats::PlayingTime& interval) { mediaIntervals_.emplace(id, interval); });
     }
 }
 

player/control/layout/MainLayoutImpl.hpp

@@ -37,8 +37,8 @@ class MainLayoutImpl : public Xibo::MainLayout, private boost::noncopyable
     MainLayoutOptions options_;
     std::shared_ptr<Xibo::OverlayContainer> view_;
 
-    PlayingStat interval_;
-    MediaPlayingStats mediaIntervals_;
+    Stats::PlayingTime interval_;
+    MediaPlayingTime mediaIntervals_;
     SignalLayoutStatReady statsReady_;
     SignalLayoutMediaStatsReady mediaStatsReady_;
 

player/control/media/Media.hpp

@@ -2,14 +2,14 @@
 
 #include "control/media/MediaOptions.hpp"
 #include "control/widgets/Widget.hpp"
-#include "stat/PlayingStat.hpp"
+#include "stat/PlayingTime.hpp"
 
 #include <boost/signals2/signal.hpp>
 
 class TransitionExecutor;
 
 using SignalMediaFinished = boost::signals2::signal<void()>;
-using SignalMediaStatReady = boost::signals2::signal<void(PlayingStat)>;
+using SignalMediaStatReady = boost::signals2::signal<void(Stats::PlayingTime)>;
 
 namespace Xibo
 {