rdelaage
/
castopod
mirror of https://code.castopod.org/adaures/castopod.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(analytics): add service name from rss user-agent 

BREAKING CHANGE: analytics_podcasts_by_player table and analytics_podcasts procedure were updated
This commit is contained in:
Benjamin Bellamy 2020-10-21 16:04:18 +00:00
parent 8ca5b33b60
commit 7202b9867b
15 changed files with 200 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/Controllers/Analytics.php
View File

@ -55,7 +55,15 @@ class Analytics extends Controller
) {
helper('media');
podcast_hit($podcastId, $episodeId, $bytesThreshold, $fileSize);
$serviceName = isset($_GET['s']) ? $_GET['s'] : '';
podcast_hit(
$podcastId,
$episodeId,
$bytesThreshold,
$fileSize,
$serviceName
);
return redirect()->to(media_url(implode('/', $filename)));
}
}

27
app/Controllers/Feed.php
View File

@ -15,13 +15,32 @@ class Feed extends Controller
{
public function index($podcastName)
{
// The page cache is set to a decade so it is deleted manually upon podcast update
$this->cachePage(DECADE);
helper('rss');
$podcast = (new PodcastModel())->where('name', $podcastName)->first();
if (!$podcast) {
throw \CodeIgniter\Exceptions\PageNotFoundException::forPageNotFound();
}
return $this->response->setXML(get_rss_feed($podcast));
$service = null;
try {
$service = \Opawg\UserAgentsPhp\UserAgentsRSS::find(
$_SERVER['HTTP_USER_AGENT']
);
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
log_message('critical', $e);
}
$cacheName =
"podcast{$podcast->id}_feed" .
($service ? "_{$service['slug']}" : '');
if (!($found = cache($cacheName))) {
$found = get_rss_feed(
$podcast,
$service ? '?s=' . urlencode($service['name']) : ''
);
cache()->save($cacheName, $found, DECADE);
}
return $this->response->setXML($found);
}
}

5
app/Database/Migrations/2020-06-08-140000_add_analytics_podcasts_by_player.php
View File

@ -25,6 +25,10 @@ class AddAnalyticsPodcastsByPlayer extends Migration
'date' => [
'type' => 'date',
],
'service' => [
'type' => 'VARCHAR',
'constraint' => 128,
],
'app' => [
'type' => 'VARCHAR',
'constraint' => 128,
@ -51,6 +55,7 @@ class AddAnalyticsPodcastsByPlayer extends Migration
$this->forge->addPrimaryKey([
'podcast_id',
'date',
'service',
'app',
'device',
'os',

5
app/Database/Migrations/2020-06-11-210000_add_analytics_podcasts_stored_procedure.php
View File

@ -28,6 +28,7 @@ CREATE PROCEDURE `{$prefix}analytics_podcasts` (
IN `p_region_code` VARCHAR(3) CHARSET utf8mb4,
IN `p_latitude` FLOAT,
IN `p_longitude` FLOAT,
IN `p_service` VARCHAR(128) CHARSET utf8mb4,
IN `p_app` VARCHAR(128) CHARSET utf8mb4,
IN `p_device` VARCHAR(32) CHARSET utf8mb4,
IN `p_os` VARCHAR(32) CHARSET utf8mb4,
@ -52,8 +53,8 @@ IF NOT `p_bot` THEN
VALUES (p_podcast_id, p_country_code, p_region_code, p_latitude, p_longitude, DATE(NOW()))
ON DUPLICATE KEY UPDATE `hits`=`hits`+1;
END IF;
INSERT INTO `{$prefix}analytics_podcasts_by_player`(`podcast_id`, `app`, `device`, `os`, `bot`, `date`)
VALUES (p_podcast_id, p_app, p_device, p_os, p_bot, DATE(NOW()))
INSERT INTO `{$prefix}analytics_podcasts_by_player`(`podcast_id`, `service`, `app`, `device`, `os`, `bot`, `date`)
VALUES (p_podcast_id, p_service, p_app, p_device, p_os, p_bot, DATE(NOW()))
ON DUPLICATE KEY UPDATE `hits`=`hits`+1;
END
EOD;

29
app/Helpers/analytics_helper.php
View File

@ -102,7 +102,7 @@ function set_user_session_player()
$userAgent = $_SERVER['HTTP_USER_AGENT'];
try {
$playerFound = \Podlibre\UserAgentsPhp\UserAgents::find($userAgent);
$playerFound = \Opawg\UserAgentsPhp\UserAgents::find($userAgent);
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
}
@ -227,6 +227,7 @@ function webpage_hit($podcast_id)
* ✅ Castopod does not do pre-load
* ✅ IP deny list https://github.com/client9/ipcat
* ✅ User-agent Filtering https://github.com/opawg/user-agents
* ✅ RSS User-agent https://github.com/opawg/podcast-rss-useragents
* ✅ Ignores 2 bytes range "Range: 0-1" (performed by official Apple iOS Podcast app)
* ✅ In case of partial content, adds up all requests to check >1mn was downloaded
* ✅ Identifying Uniques is done with a combination of IP Address and User Agent
@ -234,11 +235,17 @@ function webpage_hit($podcast_id)
* @param int $episodeId The Episode ID
* @param int $bytesThreshold The minimum total number of bytes that must be downloaded so that an episode is counted (>1mn)
* @param int $fileSize The podcast complete file size
* @param string $serviceName The name of the service that had fetched the RSS feed
*
* @return void
*/
function podcast_hit($podcastId, $episodeId, $bytesThreshold, $fileSize)
{
function podcast_hit(
$podcastId,
$episodeId,
$bytesThreshold,
$fileSize,
$serviceName
) {
$session = \Config\Services::session();
$session->start();
@ -328,22 +335,18 @@ function podcast_hit($podcastId, $episodeId, $bytesThreshold, $fileSize)
// We save the download count for this user until midnight:
cache()->save($listenerHashId, $downloadsByUser, $midnightTTL);
$app = $session->get('player')['app'];
$device = $session->get('player')['device'];
$os = $session->get('player')['os'];
$bot = $session->get('player')['bot'];
$db->query("CALL $procedureName(?,?,?,?,?,?,?,?,?,?,?);", [
$db->query("CALL $procedureName(?,?,?,?,?,?,?,?,?,?,?,?);", [
$podcastId,
$episodeId,
$session->get('location')['countryCode'],
$session->get('location')['regionCode'],
$session->get('location')['latitude'],
$session->get('location')['longitude'],
$app == null ? '' : $app,
$device == null ? '' : $device,
$os == null ? '' : $os,
$bot == null ? 0 : $bot,
$serviceName,
$session->get('player')['app'],
$session->get('player')['device'],
$session->get('player')['os'],
$session->get('player')['bot'],
$newListener,
]);
}

5
app/Helpers/rss_helper.php
View File

@ -13,9 +13,10 @@ use CodeIgniter\I18n\Time;
* Generates the rss feed for a given podcast entity
*
* @param App\Entities\Podcast $podcast
* @param string $service The name of the service that fetches the RSS feed for future reference when the audio file is eventually downloaded
* @return string rss feed as xml
*/
function get_rss_feed($podcast)
function get_rss_feed($podcast, $serviceName = '')
{
$episodes = $podcast->episodes;
@ -102,7 +103,7 @@ function get_rss_feed($podcast)
$item->addChild('title', $episode->title);
$enclosure = $item->addChild('enclosure');
$enclosure->addAttribute('url', $episode->enclosure_url);
$enclosure->addAttribute('url', $episode->enclosure_url . $serviceName);
$enclosure->addAttribute('length', $episode->enclosure_filesize);
$enclosure->addAttribute('type', $episode->enclosure_mimetype);

19
app/Language/en/Charts.php
View File

@ -7,22 +7,23 @@
*/
return [
'by_player_weekly' => 'Podcast downloads by player (for the past week)',
'by_player_yearly' => 'Podcast downloads by player (for the past year)',
'by_device_weekly' => 'Podcast downloads by device (for the past week)',
'by_os_weekly' => 'Podcast downloads by O.S. (for the past week)',
'podcast_by_region' => 'Podcast downloads by region (for the past week)',
'by_service_weekly' => 'Episode downloads by service (for the past week)',
'by_player_weekly' => 'Episode downloads by player (for the past week)',
'by_player_yearly' => 'Episode downloads by player (for the past year)',
'by_device_weekly' => 'Episode downloads by device (for the past week)',
'by_os_weekly' => 'Episode downloads by O.S. (for the past week)',
'podcast_by_region' => 'Episode downloads by region (for the past week)',
'unique_daily_listeners' => 'Daily unique listeners',
'unique_monthly_listeners' => 'Monthly unique listeners',
'by_browser' => 'Web pages usage by browser (for the past week)',
'podcast_by_day' => 'Podcast daily downloads',
'podcast_by_month' => 'Podcast monthly downloads',
'podcast_by_day' => 'Episode daily downloads',
'podcast_by_month' => 'Episode monthly downloads',
'episode_by_day' => 'Episode daily downloads (first 60 days)',
'episode_by_month' => 'Episode monthly downloads',
'episodes_by_day' =>
'5 latest episodes downloads (during their first 60 days)',
'by_country_weekly' => 'Podcast downloads by country (for the past week)',
'by_country_yearly' => 'Podcast downloads by country (for the past year)',
'by_country_weekly' => 'Episode downloads by country (for the past week)',
'by_country_yearly' => 'Episode downloads by country (for the past year)',
'by_domain_weekly' => 'Web pages visits by source (for the past week)',
'by_domain_yearly' => 'Web pages visits by source (for the past year)',
'by_entry_page' => 'Web pages visits by landing page (for the past week)',

20
app/Language/fr/Charts.php
View File

@ -7,31 +7,33 @@
*/
return [
'by_service_weekly' =>
'Téléchargements d’épisodes par service (sur la dernière semaine)',
'by_player_weekly' =>
'Téléchargements de Podcast par lecteur (sur la dernière semaine)',
'Téléchargements d’épisodes par lecteur (sur la dernière semaine)',
'by_player_yearly' =>
'Téléchargements de Podcast par lecteur (sur la dernière année)',
'Téléchargements d’épisodes par lecteur (sur la dernière année)',
'by_device_weekly' =>
'Téléchargements de Podcast par appareil (sur la dernière semaine)',
'Téléchargements d’épisodes par appareil (sur la dernière semaine)',
'by_os_weekly' =>
'Téléchargements de Podcast par OS (sur la dernière semaine)',
'Téléchargements d’épisodes par OS (sur la dernière semaine)',
'podcast_by_region' =>
'Téléchargements de Podcast par région (sur la dernière semaine)',
'Téléchargements d’épisodes par région (sur la dernière semaine)',
'unique_daily_listeners' => 'Auditeurs uniques quotidiens',
'unique_monthly_listeners' => 'Auditeurs uniques mensuels',
'by_browser' =>
'Fréquentation des pages web par navigateur (sur la dernière semaine)',
'podcast_by_day' => 'Téléchargements quotidiens de podcasts',
'podcast_by_month' => 'Téléchargements mensuels de podcasts',
'podcast_by_day' => 'Téléchargements quotidiens d’épisodes',
'podcast_by_month' => 'Téléchargements mensuels d’épisodes',
'episode_by_day' =>
'Téléchargements quotidiens de l’épisode (sur les 60 premiers jours)',
'episode_by_month' => 'Téléchargements mensuels de l’épisode',
'episodes_by_day' =>
'Téléchargements des 5 derniers épisodes (sur les 60 premiers jours)',
'by_country_weekly' =>
'Téléchargement de podcasts par pays (sur la dernière semaine)',
'Téléchargement d’épisodes par pays (sur la dernière semaine)',
'by_country_yearly' =>
'Téléchargement de podcasts par pays (sur la dernière année)',
'Téléchargement d’épisodes par pays (sur la dernière année)',
'by_domain_weekly' =>
'Fréquentation des pages web par origine (sur la dernière semaine)',
'by_domain_yearly' =>

35
app/Models/AnalyticsPodcastByPlayerModel.php
View File

@ -23,6 +23,41 @@ class AnalyticsPodcastByPlayerModel extends Model
protected $useTimestamps = false;
/**
* Gets service data for a podcast
*
* @param int $podcastId
*
* @return array
*/
public function getDataByServiceWeekly(int $podcastId): array
{
if (
!($found = cache(
"{$podcastId}_analytics_podcasts_by_player_by_service_weekly"
))
) {
$found = $this->select('`service` as `labels`')
->selectSum('`hits`', '`values`')
->where([
'`podcast_id`' => $podcastId,
'`service` !=' => '',
'`bot`' => 0,
'`date` >' => date('Y-m-d', strtotime('-1 week')),
])
->groupBy('`labels`')
->orderBy('`values`', 'DESC')
->findAll(10);
cache()->save(
"{$podcastId}_analytics_podcasts_by_player_by_service_weekly",
$found,
600
);
}
return $found;
}
/**
* Gets player data for a podcast
*

7
app/Models/EpisodeModel.php
View File

@ -272,7 +272,12 @@ class EpisodeModel extends Model
);
// delete cache for rss feed
cache()->delete(md5($episode->podcast->feed_url));
cache()->delete("podcast{$episode->podcast_id}_feed");
foreach (\Opawg\UserAgentsPhp\UserAgentsRSS::$db as $service) {
cache()->delete(
"podcast{$episode->podcast_id}_feed_{$service['slug']}"
);
}
// delete model requests cache
cache()->delete("podcast{$episode->podcast_id}_episodes");

5
app/Models/PodcastModel.php
View File

@ -173,7 +173,10 @@ class PodcastModel extends Model
$supportedLocales = config('App')->supportedLocales;
// delete cache for rss feed and podcast pages
cache()->delete(md5($podcast->feed_url));
cache()->delete("podcast{$podcast->id}_feed");
foreach (\Opawg\UserAgentsPhp\UserAgentsRSS::$db as $service) {
cache()->delete("podcast{$podcast->id}_feed_{$service['slug']}");
}
// delete model requests cache
cache()->delete("podcast{$podcast->id}");

7
app/Views/admin/podcast/analytics/players.php
View File

@ -21,14 +21,13 @@
) ?>"></div>
</div>
<div class="mb-12 mr-6 text-center">
<h2><?= lang('Charts.by_player_yearly') ?></h2>
<div class="chart-pie" id="by-app-yearly-pie" data-chart-type="pie-chart" data-chart-url="<?= route_to(
<h2><?= lang('Charts.by_service_weekly') ?></h2>
<div class="chart-pie" id="by-service-weekly-pie" data-chart-type="pie-chart" data-chart-url="<?= route_to(
'analytics-data',
$podcast->id,
'PodcastByPlayer',
'ByAppYearly'
'ByServiceWeekly'
) ?>"></div>
</div>

20
app/Views/admin/podcast/create.php
View File

@ -27,6 +27,7 @@
'id' => 'image',
'name' => 'image',
'class' => 'form-input',
'required' => 'required',
'type' => 'file',
'accept' => '.jpg,.jpeg,.png',
@ -58,21 +59,27 @@
'required' => 'required',
]) ?>
<?= form_fieldset('', [
'class' => 'mb-4',
]) ?>
<?= form_fieldset('', ['class' => 'mb-4']) ?>
<legend>
<?= lang('Podcast.form.type.label') .
hint_tooltip(lang('Podcast.form.type.hint'), 'ml-1') ?>
</legend>
<?= form_radio(
['id' => 'episodic', 'name' => 'type', 'class' => 'form-radio-btn'],
[
'id' => 'episodic',
'name' => 'type',
'class' => 'form-radio-btn',
],
'episodic',
old('type') ? old('type') == 'episodic' : true
) ?>
<label for="episodic"><?= lang('Podcast.form.type.episodic') ?></label>
<?= form_radio(
['id' => 'serial', 'name' => 'type', 'class' => 'form-radio-btn'],
[
'id' => 'serial',
'name' => 'type',
'class' => 'form-radio-btn',
],
'serial',
old('type') ? old('type') == 'serial' : false
) ?>
@ -252,6 +259,7 @@
['id' => 'block', 'name' => 'block'],
'yes',
old('block', false),
'mb-2'
) ?>
@ -266,7 +274,7 @@
lang('Podcast.form.lock'),
['id' => 'lock', 'name' => 'lock'],
'yes',
old('lock', $podcast->lock)
old('lock', true)
) ?>
<?= form_section_close() ?>

8
composer.json
View File

@ -15,7 +15,7 @@
"league/commonmark": "^1.5",
"vlucas/phpdotenv": "^5.2",
"league/html-to-markdown": "^4.10",
"podlibre/user-agents-php": "*",
"opawg/user-agents-php": "*",
"podlibre/ipcat": "*"
},
"require-dev": {
@ -31,12 +31,14 @@
"scripts": {
"test": "phpunit",
"post-install-cmd": [
"@php vendor/podlibre/user-agents-php/src/UserAgentsGenerate.php > vendor/podlibre/user-agents-php/src/UserAgents.php",
"@php vendor/opawg/user-agents-php/src/UserAgentsGenerate.php > vendor/opawg/user-agents-php/src/UserAgents.php",
"@php vendor/opawg/user-agents-php/src/UserAgentsRSSGenerate.php > vendor/opawg/user-agents-php/src/UserAgentsRSS.php",
"@php vendor/podlibre/ipcat/IpDbGenerate.php > vendor/podlibre/ipcat/IpDb.php"
],
"post-update-cmd": [
"@composer dump-autoload",
"@php vendor/podlibre/user-agents-php/src/UserAgentsGenerate.php > vendor/podlibre/user-agents-php/src/UserAgents.php",
"@php vendor/opawg/user-agents-php/src/UserAgentsGenerate.php > vendor/opawg/user-agents-php/src/UserAgents.php",
"@php vendor/opawg/user-agents-php/src/UserAgentsRSSGenerate.php > vendor/opawg/user-agents-php/src/UserAgentsRSS.php",
"@php vendor/podlibre/ipcat/IpDbGenerate.php > vendor/podlibre/ipcat/IpDb.php"
]
},

103
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "47b9f628f03f8c494a9339b054359ec8",
"content-hash": "37551523e4097a9341bc00dd317f573d",
"packages": [
{
"name": "codeigniter4/codeigniter4",
@ -12,12 +12,12 @@
"source": {
"type": "git",
"url": "https://github.com/codeigniter4/CodeIgniter4.git",
"reference": "f5545aa7274575c397efae4ebcf6c18779dcf895"
"reference": "13ff147fa4cd9db15888b041ef35bc22ed94252a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/codeigniter4/CodeIgniter4/zipball/f5545aa7274575c397efae4ebcf6c18779dcf895",
"reference": "f5545aa7274575c397efae4ebcf6c18779dcf895",
"url": "https://api.github.com/repos/codeigniter4/CodeIgniter4/zipball/13ff147fa4cd9db15888b041ef35bc22ed94252a",
"reference": "13ff147fa4cd9db15888b041ef35bc22ed94252a",
"shasum": ""
},
"require": {
@ -75,7 +75,7 @@
"slack": "https://codeigniterchat.slack.com",
"issues": "https://github.com/codeigniter4/CodeIgniter4/issues"
},
"time": "2020-10-06T06:38:58+00:00"
"time": "2020-10-20T18:13:11+00:00"
},
{
"name": "composer/ca-bundle",
@ -518,16 +518,16 @@
},
{
"name": "league/commonmark",
"version": "1.5.5",
"version": "1.5.6",
"source": {
"type": "git",
"url": "https://github.com/thephpleague/commonmark.git",
"reference": "45832dfed6007b984c0d40addfac48d403dc6432"
"reference": "a56e91e0fa1f6d0049153a9c34f63488f6b7ce61"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/thephpleague/commonmark/zipball/45832dfed6007b984c0d40addfac48d403dc6432",
"reference": "45832dfed6007b984c0d40addfac48d403dc6432",
"url": "https://api.github.com/repos/thephpleague/commonmark/zipball/a56e91e0fa1f6d0049153a9c34f63488f6b7ce61",
"reference": "a56e91e0fa1f6d0049153a9c34f63488f6b7ce61",
"shasum": ""
},
"require": {
@ -609,7 +609,7 @@
"type": "tidelift"
}
],
"time": "2020-09-13T14:44:46+00:00"
"time": "2020-10-17T21:33:03+00:00"
},
{
"name": "league/html-to-markdown",
@ -805,12 +805,12 @@
"source": {
"type": "git",
"url": "https://github.com/lonnieezell/myth-auth.git",
"reference": "e838cb8de6ffa118caf2b9909e71776a866c8973"
"reference": "e9d6a2f557bd275158e0b84624534b2abeeb539c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/lonnieezell/myth-auth/zipball/e838cb8de6ffa118caf2b9909e71776a866c8973",
"reference": "e838cb8de6ffa118caf2b9909e71776a866c8973",
"url": "https://api.github.com/repos/lonnieezell/myth-auth/zipball/e9d6a2f557bd275158e0b84624534b2abeeb539c",
"reference": "e9d6a2f557bd275158e0b84624534b2abeeb539c",
"shasum": ""
},
"require": {
@ -818,9 +818,12 @@
},
"require-dev": {
"codeigniter4/codeigniter4": "dev-develop",
"codeigniter4/codeigniter4-standard": "^1.0",
"fzaninotto/faker": "^1.9@dev",
"mockery/mockery": "^1.0",
"phpunit/phpunit": "8.5.*"
"phpstan/phpstan": "^0.12",
"phpunit/phpunit": "^8.5",
"squizlabs/php_codesniffer": "^3.5"
},
"type": "library",
"autoload": {
@ -857,7 +860,42 @@
"type": "patreon"
}
],
"time": "2020-09-07T03:37:26+00:00"
"time": "2020-10-16T18:51:37+00:00"
},
{
"name": "opawg/user-agents-php",
"version": "dev-main",
"source": {
"type": "git",
"url": "https://github.com/opawg/user-agents-php.git",
"reference": "3b71eeed2c3216f1c1c361c62d4d3a7002be0481"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/opawg/user-agents-php/zipball/3b71eeed2c3216f1c1c361c62d4d3a7002be0481",
"reference": "3b71eeed2c3216f1c1c361c62d4d3a7002be0481",
"shasum": ""
},
"type": "library",
"autoload": {
"psr-4": {
"Opawg\\UserAgentsPhp\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Benjamin Bellamy",
"email": "ben@podlibre.org",
"homepage": "https://podlibre.org/"
}
],
"description": "PHP implementation for opawg/user-agents.",
"homepage": "https://github.com/opawg/user-agents-php",
"time": "2020-10-20T23:22:20+00:00"
},
{
"name": "phpoption/phpoption",
@ -959,41 +997,6 @@
"homepage": "https://github.com/podlibre/ipcat",
"time": "2020-10-05T17:15:07+00:00"
},
{
"name": "podlibre/user-agents-php",
"version": "dev-main",
"source": {
"type": "git",
"url": "https://github.com/podlibre/user-agents-php.git",
"reference": "891066bae6b4881a8b7a57eb72a67fca1fcf67c0"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/podlibre/user-agents-php/zipball/891066bae6b4881a8b7a57eb72a67fca1fcf67c0",
"reference": "891066bae6b4881a8b7a57eb72a67fca1fcf67c0",
"shasum": ""
},
"type": "library",
"autoload": {
"psr-4": {
"Podlibre\\UserAgentsPhp\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Benjamin Bellamy",
"email": "ben@podlibre.org",
"homepage": "https://podlibre.org/"
}
],
"description": "PHP implementation for opawg/user-agents.",
"homepage": "https://github.com/podlibre/user-agents-php",
"time": "2020-10-05T16:58:13+00:00"
},
{
"name": "psr/cache",
"version": "1.0.1",