From 3a4925816f3268230640525ad7af507aab8eecb9 Mon Sep 17 00:00:00 2001 From: Benjamin Bellamy Date: Thu, 8 Oct 2020 14:45:46 +0000 Subject: [PATCH] feat: add unique listeners analytics - add unique listener - add some charts - correct minor bugs --- app/Config/Routes.php | 8 + app/Controllers/Admin/AnalyticsData.php | 5 +- ...20-06-08-120000_add_analytics_podcasts.php | 9 +- ...0000_add_analytics_podcasts_by_episode.php | 4 +- ...00_add_analytics_website_by_entry_page.php | 4 +- ...dd_analytics_podcasts_stored_procedure.php | 7 +- .../Seeds/FakePodcastsAnalyticsSeeder.php | 1 + app/Entities/AnalyticsPodcasts.php | 1 + app/Helpers/analytics_helper.php | 54 +++++-- app/Language/en/Charts.php | 20 +++ .../AnalyticsEpisodesByCountryModel.php | 26 ---- app/Models/AnalyticsEpisodesByPlayerModel.php | 26 ---- ...php => AnalyticsPodcastByCountryModel.php} | 29 ++-- ...php => AnalyticsPodcastByEpisodeModel.php} | 8 +- ....php => AnalyticsPodcastByPlayerModel.php} | 24 +-- app/Models/AnalyticsPodcastByRegionModel.php | 58 +++++++ app/Models/AnalyticsPodcastModel.php | 146 ++++++++++++++++++ .../AnalyticsPodcastsByCountryModel.php | 25 --- app/Models/AnalyticsPodcastsByRegionModel.php | 25 --- app/Models/AnalyticsWebsiteByBrowserModel.php | 29 ++++ app/Models/AnalyticsWebsiteByCountryModel.php | 26 ---- .../AnalyticsWebsiteByEntryPageModel.php | 29 ++++ app/Models/AnalyticsWebsiteByRefererModel.php | 58 +++++++ app/Views/admin/podcast/analytics.php | 68 +++++++- 24 files changed, 501 insertions(+), 189 deletions(-) create mode 100644 app/Language/en/Charts.php delete mode 100644 app/Models/AnalyticsEpisodesByCountryModel.php delete mode 100644 app/Models/AnalyticsEpisodesByPlayerModel.php rename app/Models/{AnalyticsPodcastsModel.php => AnalyticsPodcastByCountryModel.php} (51%) rename app/Models/{AnalyticsPodcastsByEpisodeModel.php => AnalyticsPodcastByEpisodeModel.php} (96%) rename app/Models/{AnalyticsPodcastsByPlayerModel.php => AnalyticsPodcastByPlayerModel.php} (88%) create mode 100644 app/Models/AnalyticsPodcastByRegionModel.php create mode 100644 app/Models/AnalyticsPodcastModel.php delete mode 100644 app/Models/AnalyticsPodcastsByCountryModel.php delete mode 100644 app/Models/AnalyticsPodcastsByRegionModel.php delete mode 100644 app/Models/AnalyticsWebsiteByCountryModel.php diff --git a/app/Config/Routes.php b/app/Config/Routes.php index 5d2c45f8..159afc47 100644 --- a/app/Config/Routes.php +++ b/app/Config/Routes.php @@ -121,6 +121,14 @@ $routes->group( 'as' => 'podcast-analytics', 'filter' => 'permission:podcasts-view,podcast-view', ]); + $routes->get( + 'analytics-data/(:segment)', + 'AnalyticsData::getData/$1/$2', + [ + 'as' => 'analytics-full-data', + 'filter' => 'permission:podcasts-view,podcast-view', + ] + ); $routes->get( 'analytics-data/(:segment)/(:segment)', 'AnalyticsData::getData/$1/$2/$3', diff --git a/app/Controllers/Admin/AnalyticsData.php b/app/Controllers/Admin/AnalyticsData.php index a57960df..ba5e1673 100644 --- a/app/Controllers/Admin/AnalyticsData.php +++ b/app/Controllers/Admin/AnalyticsData.php @@ -23,14 +23,15 @@ class AnalyticsData extends BaseController public function _remap($method, ...$params) { - if (count($params) > 2) { + if (count($params) > 1) { if (!($this->podcast = (new PodcastModel())->find($params[0]))) { throw \CodeIgniter\Exceptions\PageNotFoundException::forPageNotFound( 'Podcast not found: ' . $params[0] ); } $this->className = '\App\Models\Analytics' . $params[1] . 'Model'; - $this->methodName = 'getData' . $params[2]; + $this->methodName = + 'getData' . (empty($params[2]) ? '' : $params[2]); if (count($params) > 3) { if ( !($this->episode = (new EpisodeModel()) diff --git a/app/Database/Migrations/2020-06-08-120000_add_analytics_podcasts.php b/app/Database/Migrations/2020-06-08-120000_add_analytics_podcasts.php index e31d932d..041a195a 100644 --- a/app/Database/Migrations/2020-06-08-120000_add_analytics_podcasts.php +++ b/app/Database/Migrations/2020-06-08-120000_add_analytics_podcasts.php @@ -1,8 +1,8 @@ 10, 'default' => 1, ], + 'unique_listeners' => [ + 'type' => 'INT', + 'constraint' => 10, + 'default' => 1, + ], ]); $this->forge->addPrimaryKey(['podcast_id', 'date']); $this->forge->addField( diff --git a/app/Database/Migrations/2020-06-08-130000_add_analytics_podcasts_by_episode.php b/app/Database/Migrations/2020-06-08-130000_add_analytics_podcasts_by_episode.php index f26977ac..c9c50221 100644 --- a/app/Database/Migrations/2020-06-08-130000_add_analytics_podcasts_by_episode.php +++ b/app/Database/Migrations/2020-06-08-130000_add_analytics_podcasts_by_episode.php @@ -1,8 +1,8 @@ db->getPrefix(); $createQuery = << $podcast->id, 'date' => date('Y-m-d', $date), 'hits' => $hits, + 'unique_listeners' => $hits, ]; $analytics_podcasts_by_country[] = [ 'podcast_id' => $podcast->id, diff --git a/app/Entities/AnalyticsPodcasts.php b/app/Entities/AnalyticsPodcasts.php index 7f0f169e..b15e94e3 100644 --- a/app/Entities/AnalyticsPodcasts.php +++ b/app/Entities/AnalyticsPodcasts.php @@ -18,5 +18,6 @@ class AnalyticsPodcasts extends Entity 'podcast_id' => 'integer', 'date' => 'datetime', 'hits' => 'integer', + 'unique_listeners' => 'integer', ]; } diff --git a/app/Helpers/analytics_helper.php b/app/Helpers/analytics_helper.php index cef4e0ac..5c242d27 100644 --- a/app/Helpers/analytics_helper.php +++ b/app/Helpers/analytics_helper.php @@ -199,7 +199,7 @@ function webpage_hit($podcast_id) $referer = $session->get('referer'); $domain = empty(parse_url($referer, PHP_URL_HOST)) - ? null + ? '- Direct -' : parse_url($referer, PHP_URL_HOST); parse_str(parse_url($referer, PHP_URL_QUERY), $queries); $keywords = empty($queries['q']) ? null : $queries['q']; @@ -248,9 +248,13 @@ function podcast_hit($podcastId, $episodeId, $bytesThreshold, $fileSize) if ($session->get('denyListIp')) { $session->get('player')['bot'] = true; } - $httpRange = $_SERVER['HTTP_RANGE']; - // We create a sha1 hash for this IP_Address+User_Agent+Episode_ID: - $hashID = + //We get the HTTP header field `Range`: + $httpRange = isset($_SERVER['HTTP_RANGE']) + ? $_SERVER['HTTP_RANGE'] + : null; + + // We create a sha1 hash for this IP_Address+User_Agent+Episode_ID (used to count only once multiple episode downloads): + $episodeHashId = '_IpUaEp_' . sha1( $_SERVER['REMOTE_ADDR'] . @@ -260,12 +264,13 @@ function podcast_hit($podcastId, $episodeId, $bytesThreshold, $fileSize) $episodeId ); // Was this episode downloaded in the past 24h: - $downloadedBytes = cache($hashID); + $downloadedBytes = cache($episodeHashId); // Rolling window is 24 hours (86400 seconds): - $ttl = 86400; + $rollingTTL = 86400; if ($downloadedBytes) { // In case it was already downloaded, TTL should be adjusted (rolling window is 24h since 1st download): - $ttl = cache()->getMetadata($hashID)['expire'] - time(); + $rollingTTL = + cache()->getMetadata($episodeHashId)['expire'] - time(); } else { // If it was never downloaded that means that zero byte were downloaded: $downloadedBytes = 0; @@ -274,7 +279,7 @@ function podcast_hit($podcastId, $episodeId, $bytesThreshold, $fileSize) // (Otherwise it means that this was already counted, therefore we don't do anything) if ($downloadedBytes < $bytesThreshold) { // If HTTP_RANGE is null we are downloading the complete file: - if (!isset($httpRange)) { + if (!$httpRange) { $downloadedBytes = $fileSize; } else { // [0-1] bytes range requests are used (by Apple) to check that file exists and that 206 partial content is working. @@ -291,19 +296,44 @@ function podcast_hit($podcastId, $episodeId, $bytesThreshold, $fileSize) } } // We save the number of downloaded bytes for this user and this episode: - cache()->save($hashID, $downloadedBytes, $ttl); + cache()->save($episodeHashId, $downloadedBytes, $rollingTTL); - // If more that 1mn was downloaded, we send that to the database: + // If more that 1mn was downloaded, that's a hit, we send that to the database: if ($downloadedBytes >= $bytesThreshold) { $db = \Config\Database::connect(); $procedureName = $db->prefixTable('analytics_podcasts'); + // We create a sha1 hash for this IP_Address+User_Agent+Podcast_ID (used to count unique listeners): + $listenerHashId = + '_IpUaPo_' . + sha1( + $_SERVER['REMOTE_ADDR'] . + '_' . + $_SERVER['HTTP_USER_AGENT'] . + '_' . + $podcastId + ); + $newListener = 1; + // Has this listener already downloaded an episode today: + $downloadsByUser = cache($listenerHashId); + // We add one download + if ($downloadsByUser) { + $newListener = 0; + $downloadsByUser++; + } else { + $downloadsByUser = 1; + } + // Listener count is calculated from 00h00 to 23h59: + $midnightTTL = strtotime('tomorrow') - time(); + // We save the download count for this user until midnight: + cache()->save($listenerHashId, $downloadsByUser, $midnightTTL); + $app = $session->get('player')['app']; $device = $session->get('player')['device']; $os = $session->get('player')['os']; $bot = $session->get('player')['bot']; - $db->query("CALL $procedureName(?,?,?,?,?,?,?,?,?,?);", [ + $db->query("CALL $procedureName(?,?,?,?,?,?,?,?,?,?,?);", [ $podcastId, $episodeId, $session->get('location')['countryCode'], @@ -314,10 +344,12 @@ function podcast_hit($podcastId, $episodeId, $bytesThreshold, $fileSize) $device == null ? '' : $device, $os == null ? '' : $os, $bot == null ? 0 : $bot, + $newListener, ]); } } } catch (\Exception $e) { // If things go wrong the show must go on and the user must be able to download the file + log_message('critical', $e); } } diff --git a/app/Language/en/Charts.php b/app/Language/en/Charts.php new file mode 100644 index 00000000..c654e32a --- /dev/null +++ b/app/Language/en/Charts.php @@ -0,0 +1,20 @@ + 'Podcast downloads by player (for the past week)', + 'unique_daily_listeners' => 'Daily unique listeners', + 'unique_monthly_listeners' => 'Monthly unique listeners', + 'by_browser' => 'Website usage by browser (for the past week)', + 'podcast_by_day' => 'Podcast daily downloads', + 'podcast_by_month' => 'Podcast monthly downloads', + 'episodes_by_day' => + '5 latest episodes downloads (during their first 60 days)', + 'by_country' => 'Podcast downloads by country (for the past week)', + 'by_domain' => 'Website visits by origin (for the past week)', +]; diff --git a/app/Models/AnalyticsEpisodesByCountryModel.php b/app/Models/AnalyticsEpisodesByCountryModel.php deleted file mode 100644 index 25644114..00000000 --- a/app/Models/AnalyticsEpisodesByCountryModel.php +++ /dev/null @@ -1,26 +0,0 @@ -select('`date` as `labels`') + if (!($found = cache("{$podcastId}_analytics_podcast_by_country"))) { + $found = $this->select('`country_code` as `labels`') ->selectSum('`hits`', '`values`') + ->groupBy('`country_code`') ->where([ '`podcast_id`' => $podcastId, - '`date` >' => date('Y-m-d', strtotime('-1 year')), + '`date` >' => date('Y-m-d', strtotime('-1 week')), ]) - ->groupBy('`labels`') - ->orderBy('`labels``', 'ASC') + ->orderBy('`labels`', 'ASC') ->findAll(); cache()->save( - "{$podcastId}_analytics_podcast_by_day", + "{$podcastId}_analytics_podcast_by_country", $found, - 14400 + 600 ); } - return $found; } } diff --git a/app/Models/AnalyticsPodcastsByEpisodeModel.php b/app/Models/AnalyticsPodcastByEpisodeModel.php similarity index 96% rename from app/Models/AnalyticsPodcastsByEpisodeModel.php rename to app/Models/AnalyticsPodcastByEpisodeModel.php index 59c82360..2dc3e0ee 100644 --- a/app/Models/AnalyticsPodcastsByEpisodeModel.php +++ b/app/Models/AnalyticsPodcastByEpisodeModel.php @@ -1,7 +1,7 @@ save( "{$podcastId}_analytics_podcast_by_episode_by_day", $found, - 14400 + 600 ); } return $found; @@ -104,7 +104,7 @@ class AnalyticsPodcastsByEpisodeModel extends Model cache()->save( "{$podcastId}_{$episodeId}_analytics_podcast_by_episode_by_day", $found, - 14400 + 600 ); } return $found; diff --git a/app/Models/AnalyticsPodcastsByPlayerModel.php b/app/Models/AnalyticsPodcastByPlayerModel.php similarity index 88% rename from app/Models/AnalyticsPodcastsByPlayerModel.php rename to app/Models/AnalyticsPodcastByPlayerModel.php index 900b44fa..dfe3e493 100644 --- a/app/Models/AnalyticsPodcastsByPlayerModel.php +++ b/app/Models/AnalyticsPodcastByPlayerModel.php @@ -1,7 +1,7 @@ selectSum('`hits`', '`values`') ->where([ '`podcast_id`' => $podcastId, - '`app` !=' => null, + '`app` !=' => '', '`bot`' => 0, '`date` >' => date('Y-m-d', strtotime('-1 week')), ]) ->groupBy('`labels`') - ->orderBy('`values``', 'DESC') + ->orderBy('`values`', 'DESC') ->findAll(10); cache()->save( "{$podcastId}_analytics_podcasts_by_player_by_app", $found, - 14400 + 600 ); } @@ -60,7 +60,7 @@ class AnalyticsPodcastsByPlayerModel extends Model } /** - * Gets all data for a podcast + * Gets device data for a podcast * * @param int $podcastId * @@ -84,7 +84,7 @@ class AnalyticsPodcastsByPlayerModel extends Model '`date` >' => date('Y-m-d', strtotime('-1 week')), ]) ->groupBy('`ids`') - ->orderBy('`values``', 'DESC') + ->orderBy('`values`', 'DESC') ->findAll(); $foundOs = $this->select( @@ -98,7 +98,7 @@ class AnalyticsPodcastsByPlayerModel extends Model '`date` >' => date('Y-m-d', strtotime('-1 week')), ]) ->groupBy('`ids`') - ->orderBy('`values``', 'DESC') + ->orderBy('`values`', 'DESC') ->findAll(); $foundDevice = $this->select( @@ -112,7 +112,7 @@ class AnalyticsPodcastsByPlayerModel extends Model '`date` >' => date('Y-m-d', strtotime('-1 week')), ]) ->groupBy('`ids`') - ->orderBy('`values``', 'DESC') + ->orderBy('`values`', 'DESC') ->findAll(); $foundBot = $this->select( @@ -125,14 +125,14 @@ class AnalyticsPodcastsByPlayerModel extends Model '`date` >' => date('Y-m-d', strtotime('-1 week')), ]) ->groupBy('`ids`') - ->orderBy('`values``', 'DESC') + ->orderBy('`values`', 'DESC') ->findAll(); $found = array_merge($foundApp, $foundOs, $foundDevice, $foundBot); cache()->save( "{$podcastId}_analytics_podcasts_by_player_by_device", $found, - 14400 + 600 ); } diff --git a/app/Models/AnalyticsPodcastByRegionModel.php b/app/Models/AnalyticsPodcastByRegionModel.php new file mode 100644 index 00000000..9953fbe2 --- /dev/null +++ b/app/Models/AnalyticsPodcastByRegionModel.php @@ -0,0 +1,58 @@ +select( + '`country_code`, `region_code`, `latitude`, `longitude`' + ) + ->selectSum('`hits`', '`values`') + ->groupBy( + '`country_code`, `region_code`, `latitude`, `longitude`' + ) + ->where([ + '`podcast_id`' => $podcastId, + '`date` >' => date('Y-m-d', strtotime('-1 week')), + ]) + ->orderBy('`country_code`, `region_code`', 'ASC') + ->findAll(); + + cache()->save( + "{$podcastId}_analytics_podcast_by_region", + $found, + 600 + ); + } + return $found; + } +} diff --git a/app/Models/AnalyticsPodcastModel.php b/app/Models/AnalyticsPodcastModel.php new file mode 100644 index 00000000..82e57113 --- /dev/null +++ b/app/Models/AnalyticsPodcastModel.php @@ -0,0 +1,146 @@ +select('`date` as `labels`, `hits` as `values`') + ->where([ + '`podcast_id`' => $podcastId, + '`date` >' => date('Y-m-d', strtotime('-1 year')), + ]) + ->orderBy('`labels`', 'ASC') + ->findAll(); + + cache()->save("{$podcastId}_analytics_podcast_by_day", $found, 600); + } + return $found; + } + + /** + * Gets hits data for a podcast + * + * @param int $podcastId + * + * @return array + */ + public function getDataByMonth(int $podcastId): array + { + if (!($found = cache("{$podcastId}_analytics_podcast_by_month"))) { + $found = $this->select( + 'concat(year(`date`),"-",month(`date`),"-01") as `labels`' + ) + ->selectSum('`hits`', '`values`') + ->where([ + '`podcast_id`' => $podcastId, + '`date` >' => date('Y-m-d', strtotime('-1 year')), + ]) + ->orderBy('`date`', 'ASC') + ->groupBy('concat(month(`date`),"-",year(`date`))') + ->findAll(); + + cache()->save( + "{$podcastId}_analytics_podcast_by_month", + $found, + 600 + ); + } + return $found; + } + + /** + * Gets unique listeners data for a podcast + * + * @param int $podcastId + * + * @return array + */ + public function getDataUniqueListenersByDay(int $podcastId): array + { + if ( + !($found = cache( + "{$podcastId}_analytics_podcast_unique_listeners_by_day" + )) + ) { + $found = $this->select( + '`date` as `labels`, `unique_listeners` as `values`' + ) + ->where([ + '`podcast_id`' => $podcastId, + '`date` >' => date('Y-m-d', strtotime('-1 year')), + ]) + ->orderBy('`labels`', 'ASC') + ->findAll(); + + cache()->save( + "{$podcastId}_analytics_podcast_unique_listeners_by_day", + $found, + 600 + ); + } + return $found; + } + + /** + * Gets unique listeners data for a podcast + * + * @param int $podcastId + * + * @return array + */ + public function getDataUniqueListenersByMonth(int $podcastId): array + { + if ( + !($found = cache( + "{$podcastId}_analytics_podcast_unique_listeners_by_month" + )) + ) { + $found = $this->select( + 'concat(year(`date`),"-",month(`date`),"-01") as `labels`' + ) + ->selectSum('`unique_listeners`', '`values`') + ->where([ + '`podcast_id`' => $podcastId, + ]) + ->groupBy('concat(month(`date`),"-",year(`date`))') + ->orderBy('`date`', 'ASC') + ->findAll(); + + cache()->save( + "{$podcastId}_analytics_podcast_unique_listeners_by_month", + $found, + 600 + ); + } + return $found; + } +} diff --git a/app/Models/AnalyticsPodcastsByCountryModel.php b/app/Models/AnalyticsPodcastsByCountryModel.php deleted file mode 100644 index 4f209453..00000000 --- a/app/Models/AnalyticsPodcastsByCountryModel.php +++ /dev/null @@ -1,25 +0,0 @@ -select('`browser` as `labels`') + ->selectSum('`hits`', '`values`') + ->groupBy('`browser`') + ->where([ + '`podcast_id`' => $podcastId, + '`date` >' => date('Y-m-d', strtotime('-1 week')), + ]) + ->orderBy('`browser`', 'ASC') + ->findAll(); + + cache()->save( + "{$podcastId}_analytics_website_by_browser", + $found, + 600 + ); + } + return $found; + } } diff --git a/app/Models/AnalyticsWebsiteByCountryModel.php b/app/Models/AnalyticsWebsiteByCountryModel.php deleted file mode 100644 index 0163b65b..00000000 --- a/app/Models/AnalyticsWebsiteByCountryModel.php +++ /dev/null @@ -1,26 +0,0 @@ -select('`entry_page` as `labels`') + ->selectSum('`hits`', '`values`') + ->groupBy('`entry_page`') + ->where([ + '`podcast_id`' => $podcastId, + '`date` >' => date('Y-m-d', strtotime('-1 week')), + ]) + ->orderBy('`entry_page`', 'ASC') + ->findAll(); + + cache()->save( + "{$podcastId}_analytics_website_by_entry_page", + $found, + 600 + ); + } + return $found; + } } diff --git a/app/Models/AnalyticsWebsiteByRefererModel.php b/app/Models/AnalyticsWebsiteByRefererModel.php index 5d60298c..e45ea20b 100644 --- a/app/Models/AnalyticsWebsiteByRefererModel.php +++ b/app/Models/AnalyticsWebsiteByRefererModel.php @@ -22,4 +22,62 @@ class AnalyticsWebsiteByRefererModel extends Model protected $useSoftDeletes = false; protected $useTimestamps = false; + + /** + * Gets referer data for a podcast + * + * @param int $podcastId + * + * @return array + */ + public function getData(int $podcastId): array + { + if (!($found = cache("{$podcastId}_analytics_website_by_referer"))) { + $found = $this->select('`referer` as `labels`') + ->selectSum('`hits`', '`values`') + ->groupBy('`referer`') + ->where([ + '`podcast_id`' => $podcastId, + '`date` >' => date('Y-m-d', strtotime('-1 week')), + ]) + ->orderBy('`referer`', 'ASC') + ->findAll(); + + cache()->save( + "{$podcastId}_analytics_website_by_referer", + $found, + 600 + ); + } + return $found; + } + + /** + * Gets domain data for a podcast + * + * @param int $podcastId + * + * @return array + */ + public function getDataByDomain(int $podcastId): array + { + if (!($found = cache("{$podcastId}_analytics_website_by_domain"))) { + $found = $this->select('`domain` as `labels`') + ->selectSum('`hits`', '`values`') + ->groupBy('`domain`') + ->where([ + '`podcast_id`' => $podcastId, + '`date` >' => date('Y-m-d', strtotime('-1 week')), + ]) + ->orderBy('`domain`', 'ASC') + ->findAll(); + + cache()->save( + "{$podcastId}_analytics_website_by_domain", + $found, + 600 + ); + } + return $found; + } } diff --git a/app/Views/admin/podcast/analytics.php b/app/Views/admin/podcast/analytics.php index c57559b6..c09619f2 100644 --- a/app/Views/admin/podcast/analytics.php +++ b/app/Views/admin/podcast/analytics.php @@ -9,24 +9,76 @@ endSection() ?> section('content') ?> -
+ +

+ +

+
+ +

+
+ +

+
+ +

+

+
+ +

+
+ +

+
+ +

+
+ endSection() ?>