rdelaage
/
castopod
mirror of https://code.castopod.org/adaures/castopod.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(analytics): move all analytics files to a new Libraries/Analytics folder 

- add page hit on podcast activity page
- update development docs
This commit is contained in:
Yassine Doghri 2021-04-14 15:58:40 +00:00
parent 1c0d6cee44
commit 247ae1824f
59 changed files with 865 additions and 752 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Dockerfile
View File

@ -20,9 +20,9 @@ RUN docker-php-ext-configure gd --with-jpeg-dir=/usr/include/ \
RUN docker-php-ext-install mysqli && docker-php-ext-enable mysqli
RUN echo "file_uploads = On\n" \
"memory_limit = 100M\n" \
"upload_max_filesize = 100M\n" \
"post_max_size = 120M\n" \
"memory_limit = 512M\n" \
"upload_max_filesize = 500M\n" \
"post_max_size = 512M\n" \
"max_execution_time = 300\n" \
> /usr/local/etc/php/conf.d/uploads.ini

35
app/Config/Analytics.php Normal file
View File

@ -0,0 +1,35 @@
<?php
namespace Config;
use Analytics\Config\Analytics as AnalyticsBase;
class Analytics extends AnalyticsBase
{
/**
* --------------------------------------------------------------------
* Route filters options
* --------------------------------------------------------------------
*/
public $routeFilters = [
'analytics-full-data' => 'permission:podcasts-view,podcast-view',
'analytics-data' => 'permission:podcasts-view,podcast-view',
'analytics-filtered-data' => 'permission:podcasts-view,podcast-view',
];
public function __construct()
{
parent::__construct();
// set the analytics gateway behind the admin gateway.
// Only logged in users should be able to view analytics
$this->gateway = config('App')->adminGateway . '/analytics';
}
public function getEnclosureUrl($enclosureUri)
{
helper('media');
return media_base_url($enclosureUri);
}
}

1
app/Config/Autoload.php
View File

@ -43,6 +43,7 @@ class Autoload extends AutoloadConfig
APP_NAMESPACE => APPPATH, // For custom app namespace
'Config' => APPPATH . 'Config',
'ActivityPub' => APPPATH . 'Libraries/ActivityPub',
'Analytics' => APPPATH . 'Libraries/Analytics',
];
/**

37
app/Config/Routes.php
View File

@ -70,18 +70,6 @@ $routes->group(config('App')->installGateway, function ($routes) {
]);
});
// Route for podcast audio file analytics (/audio/pack(podcast_id,episode_id,bytes_threshold,filesize,duration,date)/podcast_folder/filename.mp3)
$routes->head('audio/(:base64)/(:any)', 'Analytics::hit/$1/$2', [
'as' => 'analytics_hit',
]);
$routes->get('audio/(:base64)/(:any)', 'Analytics::hit/$1/$2', [
'as' => 'analytics_hit',
]);
// Show the Unknown UserAgents
$routes->get('.well-known/unknown-useragents', 'UnknownUserAgents');
$routes->get('.well-known/unknown-useragents/(:num)', 'UnknownUserAgents/$1');
$routes->get('.well-known/platforms', 'Platform');
// Admin area
@ -237,31 +225,6 @@ $routes->group(
);
});
$routes->get(
'analytics-data/(:segment)',
'AnalyticsData::getData/$1/$2',
[
'as' => 'analytics-full-data',
'filter' => 'permission:podcasts-view,podcast-view',
],
);
$routes->get(
'analytics-data/(:segment)/(:segment)',
'AnalyticsData::getData/$1/$2/$3',
[
'as' => 'analytics-data',
'filter' => 'permission:podcasts-view,podcast-view',
],
);
$routes->get(
'analytics-data/(:segment)/(:segment)/(:num)',
'AnalyticsData::getData/$1/$2/$3/$4',
[
'as' => 'analytics-filtered-data',
'filter' => 'permission:podcasts-view,podcast-view',
],
);
// Podcast episodes
$routes->group('episodes', function ($routes) {
$routes->get('/', 'Episode::list/$1', [

70
app/Controllers/Admin/AnalyticsData.php
View File

@ -1,70 +0,0 @@
<?php
/**
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
namespace App\Controllers\Admin;
use App\Models\PodcastModel;
use App\Models\EpisodeModel;
class AnalyticsData extends BaseController
{
/**
* @var \App\Entities\Podcast|null
*/
protected $podcast;
protected $className;
protected $methodName;
protected $episode;
public function _remap($method, ...$params)
{
if (count($params) > 1) {
if (!($this->podcast = (new PodcastModel())->find($params[0]))) {
throw \CodeIgniter\Exceptions\PageNotFoundException::forPageNotFound(
'Podcast not found: ' . $params[0]
);
}
$this->className = '\App\Models\Analytics' . $params[1] . 'Model';
$this->methodName =
'getData' . (empty($params[2]) ? '' : $params[2]);
if (count($params) > 3) {
if (
!($this->episode = (new EpisodeModel())
->where([
'podcast_id' => $this->podcast->id,
'id' => $params[3],
])
->first())
) {
throw \CodeIgniter\Exceptions\PageNotFoundException::forPageNotFound(
'Episode not found: ' . $params[3]
);
}
}
}
return $this->$method();
}
public function getData()
{
$analytics_model = new $this->className();
$methodName = $this->methodName;
if ($this->episode) {
return $this->response->setJSON(
$analytics_model->$methodName(
$this->podcast->id,
$this->episode->id
)
);
} else {
return $this->response->setJSON(
$analytics_model->$methodName($this->podcast->id)
);
}
}
}

7
app/Controllers/Episode.php
View File

@ -44,8 +44,6 @@ class Episode extends BaseController
public function index()
{
$episodeModel = new EpisodeModel();
self::triggerWebpageHit($this->podcast->id);
$locale = service('request')->getLocale();
@ -65,7 +63,7 @@ class Episode extends BaseController
'persons' => $podcastPersons,
];
$secondsToNextUnpublishedEpisode = $episodeModel->getSecondsToNextUnpublishedEpisode(
$secondsToNextUnpublishedEpisode = (new EpisodeModel())->getSecondsToNextUnpublishedEpisode(
$this->podcast->id,
);
@ -112,7 +110,6 @@ class Episode extends BaseController
$cacheName = "page_podcast{$this->episode->podcast_id}_episode{$this->episode->id}_embeddable_player_{$theme}_{$locale}";
if (!($cachedView = cache($cacheName))) {
$episodeModel = new EpisodeModel();
$theme = EpisodeModel::$themes[$theme];
$data = [
@ -121,7 +118,7 @@ class Episode extends BaseController
'theme' => $theme,
];
$secondsToNextUnpublishedEpisode = $episodeModel->getSecondsToNextUnpublishedEpisode(
$secondsToNextUnpublishedEpisode = (new EpisodeModel())->getSecondsToNextUnpublishedEpisode(
$this->podcast->id,
);

17
app/Controllers/Install.php
View File

@ -111,7 +111,7 @@ class Install extends Controller
// show database config view to fix value
session()->setFlashdata(
'error',
lang('Install.messages.databaseConnectError')
lang('Install.messages.databaseConnectError'),
);
return view('install/database_config');
@ -159,7 +159,7 @@ class Install extends Controller
return redirect()
->to(
(empty(host_url()) ? config('App')->baseURL : host_url()) .
config('App')->installGateway
config('App')->installGateway,
)
->withInput()
->with('errors', $this->validator->getErrors());
@ -181,8 +181,8 @@ class Install extends Controller
// redirect to full install url with new baseUrl input
return redirect(0)->to(
reduce_double_slashes(
$baseUrl . '/' . config('App')->installGateway
)
$baseUrl . '/' . config('App')->installGateway,
),
);
}
@ -209,14 +209,14 @@ class Install extends Controller
self::writeEnv([
'database.default.hostname' => $this->request->getPost(
'db_hostname'
'db_hostname',
),
'database.default.database' => $this->request->getPost('db_name'),
'database.default.username' => $this->request->getPost(
'db_username'
'db_username',
),
'database.default.password' => $this->request->getPost(
'db_password'
'db_password',
),
'database.default.DBPrefix' => $this->request->getPost('db_prefix'),
]);
@ -258,6 +258,7 @@ class Install extends Controller
!$migrations->setNamespace('Myth\Auth')->latest();
!$migrations->setNamespace('ActivityPub')->latest();
!$migrations->setNamespace('Analytics')->latest();
!$migrations->setNamespace(APP_NAMESPACE)->latest();
}
@ -296,7 +297,7 @@ class Install extends Controller
[
'email' => 'required|valid_email|is_unique[users.email]',
'password' => 'required|strong_password',
]
],
);
if (!$this->validate($rules)) {

2
app/Controllers/Podcast.php
View File

@ -37,6 +37,8 @@ class Podcast extends BaseController
public function activity()
{
self::triggerWebpageHit($this->podcast->id);
helper('persons');
$persons = [];
construct_person_array($this->podcast->persons, $persons);

15
app/Database/Seeds/FakePodcastsAnalyticsSeeder.php
View File

@ -10,6 +10,7 @@
*/
namespace App\Database\Seeds;
use App\Models\PodcastModel;
use App\Models\EpisodeModel;
@ -23,16 +24,16 @@ class FakePodcastsAnalyticsSeeder extends Seeder
$jsonUserAgents = json_decode(
file_get_contents(
'https://raw.githubusercontent.com/opawg/user-agents/master/src/user-agents.json'
'https://raw.githubusercontent.com/opawg/user-agents/master/src/user-agents.json',
),
true
true,
);
$jsonRSSUserAgents = json_decode(
file_get_contents(
'https://raw.githubusercontent.com/opawg/podcast-rss-useragents/master/src/rss-ua.json'
'https://raw.githubusercontent.com/opawg/podcast-rss-useragents/master/src/rss-ua.json',
),
true
true,
);
if ($podcast) {
@ -60,7 +61,7 @@ class FakePodcastsAnalyticsSeeder extends Seeder
->findAll();
foreach ($episodes as $episode) {
$age = floor(
($date - strtotime($episode->published_at)) / 86400
($date - strtotime($episode->published_at)) / 86400,
);
$proba1 = floor(exp(3 - $age / 40)) + 1;
@ -97,7 +98,7 @@ class FakePodcastsAnalyticsSeeder extends Seeder
$cityReader = new \GeoIp2\Database\Reader(
WRITEPATH .
'uploads/GeoLite2-City/GeoLite2-City.mmdb'
'uploads/GeoLite2-City/GeoLite2-City.mmdb',
);
$countryCode = 'N/A';
@ -196,7 +197,7 @@ class FakePodcastsAnalyticsSeeder extends Seeder
->insertBatch($analytics_podcasts_by_region);
}
} else {
echo "Create one podcast and some episodes first.\n";
echo "COULD NOT POPULATE DATABASE:\n\tCreate a podcast with episodes first.\n";
}
}
}

5
app/Database/Seeds/FakeWebsiteAnalyticsSeeder.php
View File

@ -10,6 +10,7 @@
*/
namespace App\Database\Seeds;
use App\Models\PodcastModel;
use App\Models\EpisodeModel;
@ -193,7 +194,7 @@ class FakeWebsiteAnalyticsSeeder extends Seeder
->findAll();
foreach ($episodes as $episode) {
$age = floor(
($date - strtotime($episode->published_at)) / 86400
($date - strtotime($episode->published_at)) / 86400,
);
$proba1 = floor(exp(3 - $age / 40)) + 1;
@ -254,7 +255,7 @@ class FakeWebsiteAnalyticsSeeder extends Seeder
->insertBatch($website_by_referer);
}
} else {
echo "Create one podcast and some episodes first.\n";
echo "COULD NOT POPULATE DATABASE:\n\tCreate a podcast with episodes first.\n";
}
}
}

43
app/Entities/Episode.php
View File

@ -336,37 +336,14 @@ class Episode extends Entity
{
helper('analytics');
return base_url(
route_to(
'analytics_hit',
base64_url_encode(
pack(
'I*',
$this->attributes['podcast_id'],
$this->attributes['id'],
// bytes_threshold: number of bytes that must be downloaded for an episode to be counted in download analytics
// - if file is shorter than 60sec, then it's enclosure_filesize
// - if file is longer than 60 seconds then it's enclosure_headersize + 60 seconds
$this->attributes['enclosure_duration'] <= 60
? $this->attributes['enclosure_filesize']
: $this->attributes['enclosure_headersize'] +
floor(
(($this->attributes['enclosure_filesize'] -
$this->attributes[
'enclosure_headersize'
]) /
$this->attributes[
'enclosure_duration'
]) *
60,
),
$this->attributes['enclosure_filesize'],
$this->attributes['enclosure_duration'],
strtotime($this->attributes['published_at']),
),
),
$this->attributes['enclosure_uri'],
),
return generate_episode_analytics_url(
$this->podcast_id,
$this->id,
$this->enclosure_uri,
$this->enclosure_duration,
$this->enclosure_filesize,
$this->enclosure_headersize,
$this->published_at,
);
}
@ -520,9 +497,9 @@ class Episode extends Entity
empty($this->getPodcast()->partner_image_url)
? ''
: "<div><a href=\"{$this->getPartnerLink(
$serviceSlug
$serviceSlug,
)}\" rel=\"sponsored noopener noreferrer\" target=\"_blank\"><img src=\"{$this->getPartnerImage(
$serviceSlug
$serviceSlug,
)}\" alt=\"Partner image\" /></a></div>") .
$this->attributes['description_html'] .
(empty($this->getPodcast()->episode_description_footer_html)

357
app/Helpers/analytics_helper.php
View File

@ -1,357 +0,0 @@
<?php
/**
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
/**
* Encode Base64 for URLs
*/
function base64_url_encode($input)
{
return strtr(base64_encode($input), '+/=', '._-');
}
/**
* Decode Base64 from URL
*/
function base64_url_decode($input)
{
return base64_decode(strtr($input, '._-', '+/='));
}
/**
* Set user country in session variable, for analytics purpose
*/
function set_user_session_deny_list_ip()
{
$session = \Config\Services::session();
$session->start();
if (!$session->has('denyListIp')) {
$session->set(
'denyListIp',
\Podlibre\Ipcat\IpDb::find($_SERVER['REMOTE_ADDR']) != null,
);
}
}
/**
* Set user country in session variable, for analytics purpose
*/
function set_user_session_location()
{
$session = \Config\Services::session();
$session->start();
$location = [
'countryCode' => 'N/A',
'regionCode' => 'N/A',
'latitude' => null,
'longitude' => null,
];
// Finds location:
if (!$session->has('location')) {
try {
$cityReader = new \GeoIp2\Database\Reader(
WRITEPATH . 'uploads/GeoLite2-City/GeoLite2-City.mmdb',
);
$city = $cityReader->city($_SERVER['REMOTE_ADDR']);
$location = [
'countryCode' => empty($city->country->isoCode)
? 'N/A'
: $city->country->isoCode,
'regionCode' => empty($city->subdivisions[0]->isoCode)
? 'N/A'
: $city->subdivisions[0]->isoCode,
'latitude' => round($city->location->latitude, 3),
'longitude' => round($city->location->longitude, 3),
];
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
}
$session->set('location', $location);
}
}
/**
* Set user player in session variable, for analytics purpose
*/
function set_user_session_player()
{
$session = \Config\Services::session();
$session->start();
if (!$session->has('player')) {
$playerFound = null;
$userAgent = $_SERVER['HTTP_USER_AGENT'];
try {
$playerFound = \Opawg\UserAgentsPhp\UserAgents::find($userAgent);
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
}
if ($playerFound) {
$session->set('player', $playerFound);
} else {
$session->set('player', [
'app' => '- unknown -',
'device' => '',
'os' => '',
'bot' => 0,
]);
// Add to unknown list
try {
$db = \Config\Database::connect();
$procedureNameAnalyticsUnknownUseragents = $db->prefixTable(
'analytics_unknown_useragents',
);
$db->query("CALL $procedureNameAnalyticsUnknownUseragents(?)", [
$userAgent,
]);
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
}
}
}
}
/**
* Set user browser in session variable, for analytics purpose
*/
function set_user_session_browser()
{
$session = \Config\Services::session();
$session->start();
if (!$session->has('browser')) {
$browserName = '- Other -';
try {
$whichbrowser = new \WhichBrowser\Parser(getallheaders());
$browserName = $whichbrowser->browser->name;
} catch (\Exception $e) {
$browserName = '- Could not get browser name -';
}
if ($browserName == null) {
$browserName = '- Could not get browser name -';
}
$session->set('browser', $browserName);
}
}
/**
* Set user referer in session variable, for analytics purpose
*/
function set_user_session_referer()
{
$session = \Config\Services::session();
$session->start();
$newreferer = isset($_SERVER['HTTP_REFERER'])
? $_SERVER['HTTP_REFERER']
: '- Direct -';
$newreferer =
parse_url($newreferer, PHP_URL_HOST) ==
parse_url(current_url(false), PHP_URL_HOST)
? '- Direct -'
: $newreferer;
if (!$session->has('referer') or $newreferer != '- Direct -') {
$session->set('referer', $newreferer);
}
}
/**
* Set user entry page in session variable, for analytics purpose
*/
function set_user_session_entry_page()
{
$session = \Config\Services::session();
$session->start();
$entryPage = $_SERVER['REQUEST_URI'];
if (!$session->has('entryPage')) {
$session->set('entryPage', $entryPage);
}
}
function webpage_hit($podcast_id)
{
$session = \Config\Services::session();
$session->start();
if (!$session->get('denyListIp')) {
$db = \Config\Database::connect();
$referer = $session->get('referer');
$domain = empty(parse_url($referer, PHP_URL_HOST))
? '- Direct -'
: parse_url($referer, PHP_URL_HOST);
parse_str(parse_url($referer, PHP_URL_QUERY), $queries);
$keywords = empty($queries['q']) ? null : $queries['q'];
$procedureName = $db->prefixTable('analytics_website');
$db->query("call $procedureName(?,?,?,?,?,?)", [
$podcast_id,
$session->get('browser'),
$session->get('entryPage'),
$referer,
$domain,
$keywords,
]);
}
}
/**
* Counting podcast episode downloads for analytics purposes
* ✅ No IP address is ever stored on the server.
* ✅ Only aggregate data is stored in the database.
* We follow IAB Podcast Measurement Technical Guidelines Version 2.0:
* https://iabtechlab.com/standards/podcast-measurement-guidelines/
* https://iabtechlab.com/wp-content/uploads/2017/12/Podcast_Measurement_v2-Dec-20-2017.pdf
* ✅ Rolling 24-hour window
* ✅ Castopod does not do pre-load
* ✅ IP deny list https://github.com/client9/ipcat
* ✅ User-agent Filtering https://github.com/opawg/user-agents
* ✅ RSS User-agent https://github.com/opawg/podcast-rss-useragents
* ✅ Ignores 2 bytes range "Range: 0-1" (performed by official Apple iOS Podcast app)
* ✅ In case of partial content, adds up all requests to check >1mn was downloaded
* ✅ Identifying Uniques is done with a combination of IP Address and User Agent
* @param int $podcastId The podcast ID
* @param int $episodeId The Episode ID
* @param int $bytesThreshold The minimum total number of bytes that must be downloaded so that an episode is counted (>1mn)
* @param int $fileSize The podcast complete file size
* @param string $serviceName The name of the service that had fetched the RSS feed
*
* @return void
*/
function podcast_hit(
$podcastId,
$episodeId,
$bytesThreshold,
$fileSize,
$duration,
$publicationDate,
$serviceName
) {
$session = \Config\Services::session();
$session->start();
// We try to count (but if things went wrong the show should go on and the user should be able to download the file):
try {
// If the user IP is denied it's probably a bot:
if ($session->get('denyListIp')) {
$session->get('player')['bot'] = true;
}
//We get the HTTP header field `Range`:
$httpRange = isset($_SERVER['HTTP_RANGE'])
? $_SERVER['HTTP_RANGE']
: null;
// We create a sha1 hash for this IP_Address+User_Agent+Episode_ID (used to count only once multiple episode downloads):
$episodeHashId =
'_IpUaEp_' .
sha1(
$_SERVER['REMOTE_ADDR'] .
'_' .
$_SERVER['HTTP_USER_AGENT'] .
'_' .
$episodeId,
);
// Was this episode downloaded in the past 24h:
$downloadedBytes = cache($episodeHashId);
// Rolling window is 24 hours (86400 seconds):
$rollingTTL = 86400;
if ($downloadedBytes) {
// In case it was already downloaded, TTL should be adjusted (rolling window is 24h since 1st download):
$rollingTTL =
cache()->getMetadata($episodeHashId)['expire'] - time();
} else {
// If it was never downloaded that means that zero byte were downloaded:
$downloadedBytes = 0;
}
// If the number of downloaded bytes was previously below the 1mn threshold we go on:
// (Otherwise it means that this was already counted, therefore we don't do anything)
if ($downloadedBytes < $bytesThreshold) {
// If HTTP_RANGE is null we are downloading the complete file:
if (!$httpRange) {
$downloadedBytes = $fileSize;
} else {
// [0-1] bytes range requests are used (by Apple) to check that file exists and that 206 partial content is working.
// We don't count these requests:
if ($httpRange != 'bytes=0-1') {
// We calculate how many bytes are being downloaded based on HTTP_RANGE values:
$ranges = explode(',', substr($httpRange, 6));
foreach ($ranges as $range) {
$parts = explode('-', $range);
$downloadedBytes += empty($parts[1])
? $fileSize
: $parts[1] - (empty($parts[0]) ? 0 : $parts[0]);
}
}
}
// We save the number of downloaded bytes for this user and this episode:
cache()->save($episodeHashId, $downloadedBytes, $rollingTTL);
// If more that 1mn was downloaded, that's a hit, we send that to the database:
if ($downloadedBytes >= $bytesThreshold) {
$db = \Config\Database::connect();
$procedureName = $db->prefixTable('analytics_podcasts');
$age = intdiv(time() - $publicationDate, 86400);
// We create a sha1 hash for this IP_Address+User_Agent+Podcast_ID (used to count unique listeners):
$listenerHashId =
'_IpUaPo_' .
sha1(
$_SERVER['REMOTE_ADDR'] .
'_' .
$_SERVER['HTTP_USER_AGENT'] .
'_' .
$podcastId,
);
$newListener = 1;
// Has this listener already downloaded an episode today:
$downloadsByUser = cache($listenerHashId);
// We add one download
if ($downloadsByUser) {
$newListener = 0;
$downloadsByUser++;
} else {
$downloadsByUser = 1;
}
// Listener count is calculated from 00h00 to 23h59:
$midnightTTL = strtotime('tomorrow') - time();
// We save the download count for this user until midnight:
cache()->save($listenerHashId, $downloadsByUser, $midnightTTL);
$db->query(
"CALL $procedureName(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);",
[
$podcastId,
$episodeId,
$session->get('location')['countryCode'],
$session->get('location')['regionCode'],
$session->get('location')['latitude'],
$session->get('location')['longitude'],
$serviceName,
$session->get('player')['app'],
$session->get('player')['device'],
$session->get('player')['os'],
$session->get('player')['bot'],
$fileSize,
$duration,
$age,
$newListener,
],
);
}
}
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
log_message('critical', $e);
}
}

2
app/Helpers/url_helper.php
View File

@ -88,7 +88,7 @@ if (!function_exists('extract_params_from_episode_uri')) {
preg_match(
'/@(?P<podcastName>[a-zA-Z0-9\_]{1,32})\/episodes\/(?P<episodeSlug>[a-zA-Z0-9\-]{1,191})/',
$episodeUri->getPath(),
$matches
$matches,
);
if (

38
app/Libraries/Analytics/Config/Analytics.php Normal file
View File

@ -0,0 +1,38 @@
<?php
namespace Analytics\Config;
use CodeIgniter\Config\BaseConfig;
class Analytics extends BaseConfig
{
/**
* Gateway to analytic routes.
* By default, all analytics routes will be under `/analytics` path
*
* @var string
*/
public $gateway = 'analytics';
/**
* --------------------------------------------------------------------
* Route filters options
* --------------------------------------------------------------------
*/
public $routeFilters = [
'analytics-full-data' => '',
'analytics-data' => '',
'analytics-filtered-data' => '',
];
/**
* get the full enclosure url
*
* @param string $filename
* @return string
*/
public function getEnclosureUrl(string $enclosureUri)
{
return base_url($enclosureUri);
}
}

74
app/Libraries/Analytics/Config/Routes.php Normal file
View File

@ -0,0 +1,74 @@
<?php
/**
* @copyright 2021 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
/**
* Analytics routes file
*/
$routes->addPlaceholder(
'class',
'\bPodcastByCountry|\bPodcastByEpisode|\bPodcastByHour|\bPodcastByPlayer|\bPodcastByRegion|\bPodcastByService|\bPodcast|\bWebsiteByBrowser|\bWebsiteByEntryPage|\bWebsiteByReferer',
);
$routes->addPlaceholder(
'filter',
'\bWeekly|\bYearly|\bByDay|\bByWeekday|\bByMonth|\bByAppWeekly|\bByAppYearly|\bByOsWeekly|\bByDeviceWeekly|\bBots|\bByServiceWeekly|\bBandwidthByDay|\bUniqueListenersByDay|\bUniqueListenersByMonth|\bTotalListeningTimeByDay|\bTotalListeningTimeByMonth|\bByDomainWeekly|\bByDomainYearly',
);
$routes->group('', ['namespace' => 'Analytics\Controllers'], function (
$routes
) {
$routes->group(config('Analytics')->gateway . '/(:num)/(:class)', function (
$routes
) {
$routes->get('/', 'AnalyticsController::getData/$1/$2', [
'as' => 'analytics-full-data',
'filter' => config('Analytics')->routeFilters[
'analytics-full-data'
],
]);
$routes->get('(:filter)', 'AnalyticsController::getData/$1/$2/$3', [
'as' => 'analytics-data',
'filter' => config('Analytics')->routeFilters['analytics-data'],
]);
$routes->get(
'(:filter)/(:num)',
'AnalyticsController::getData/$1/$2/$3/$4',
[
'as' => 'analytics-filtered-data',
'filter' => config('Analytics')->routeFilters[
'analytics-filtered-data'
],
],
);
});
// Route for podcast audio file analytics (/audio/pack(podcast_id,episode_id,bytes_threshold,filesize,duration,date)/podcast_folder/filename.mp3)
$routes->head(
'audio/(:base64)/(:any)',
'EpisodeAnalyticsController::hit/$1/$2',
[
'as' => 'episode-analytics-hit',
],
);
$routes->get(
'audio/(:base64)/(:any)',
'EpisodeAnalyticsController::hit/$1/$2',
[
'as' => 'episode-analytics-hit',
],
);
});
// Show the Unknown UserAgents
$routes->get('.well-known/unknown-useragents', 'UnknownUserAgentsController');
$routes->get(
'.well-known/unknown-useragents/(:num)',
'UnknownUserAgentsController/$1',
);

54
app/Libraries/Analytics/Controllers/AnalyticsController.php Normal file
View File

@ -0,0 +1,54 @@
<?php
/**
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
namespace Analytics\Controllers;
use CodeIgniter\Controller;
class AnalyticsController extends Controller
{
/**
* @var string
*/
protected $className;
/**
* @var string
*/
protected $methodName;
public function _remap($method, ...$params)
{
if (!isset($params[1])) {
throw \CodeIgniter\Exceptions\PageNotFoundException::forPageNotFound();
}
$this->className = model('Analytics' . $params[1] . 'Model');
$this->methodName = 'getData' . (empty($params[2]) ? '' : $params[2]);
return $this->$method(
$params[0],
isset($params[3]) ? $params[3] : null,
);
}
public function getData($podcastId, $episodeId)
{
$analytics_model = new $this->className();
$methodName = $this->methodName;
if ($episodeId) {
return $this->response->setJSON(
$analytics_model->$methodName($podcastId, $episodeId),
);
} else {
return $this->response->setJSON(
$analytics_model->$methodName($podcastId),
);
}
}
}

22
app/Controllers/Analytics.php → app/Libraries/Analytics/Controllers/EpisodeAnalyticsController.php
View File

@ -1,18 +1,16 @@
<?php
/**
* Class Analytics
* Creates Analytics controller
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
namespace App\Controllers;
namespace Analytics\Controllers;
use CodeIgniter\Controller;
class Analytics extends Controller
class EpisodeAnalyticsController extends Controller
{
/**
* An array of helpers to be loaded automatically upon
@ -23,6 +21,10 @@ class Analytics extends Controller
*/
protected $helpers = ['analytics'];
/**
* @var \Analytics\Config\Analytics
*/
protected $config;
/**
* Constructor.
*/
@ -43,12 +45,13 @@ class Analytics extends Controller
set_user_session_deny_list_ip();
set_user_session_location();
set_user_session_player();
$this->config = config('Analytics');
}
// Add one hit to this episode:
public function hit($base64EpisodeData, ...$filename)
public function hit($base64EpisodeData, ...$enclosureUri)
{
helper('media', 'analytics');
$session = \Config\Services::session();
$session->start();
$serviceName = '';
@ -62,7 +65,7 @@ class Analytics extends Controller
$episodeData = unpack(
'IpodcastId/IepisodeId/IbytesThreshold/IfileSize/Iduration/IpublicationDate',
base64_url_decode($base64EpisodeData)
base64_url_decode($base64EpisodeData),
);
podcast_hit(
@ -72,8 +75,9 @@ class Analytics extends Controller
$episodeData['fileSize'],
$episodeData['duration'],
$episodeData['publicationDate'],
$serviceName
$serviceName,
);
return redirect()->to(media_base_url($filename));
return redirect()->to($this->config->getEnclosureUrl($enclosureUri));
}
}

6
app/Controllers/UnknownUserAgents.php → app/Libraries/Analytics/Controllers/UnknownUserAgentsController.php
View File

@ -6,15 +6,15 @@
* @link https://castopod.org/
*/
namespace App\Controllers;
namespace Analytics\Controllers;
use CodeIgniter\Controller;
class UnknownUserAgents extends Controller
class UnknownUserAgentsController extends Controller
{
public function index($lastKnownId = 0)
{
$model = new \App\Models\UnknownUserAgentsModel();
$model = model('UnknownUserAgentsModel');
return $this->response->setJSON($model->getUserAgents($lastKnownId));
}

2
app/Database/Migrations/2020-06-08-120000_add_analytics_podcasts.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-120000_add_analytics_podcasts.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Database/Migrations/2020-06-08-130000_add_analytics_podcasts_by_episode.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-130000_add_analytics_podcasts_by_episode.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Database/Migrations/2020-06-08-130000_add_analytics_podcasts_by_hour.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-130000_add_analytics_podcasts_by_hour.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Database/Migrations/2020-06-08-140000_add_analytics_podcasts_by_player.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-140000_add_analytics_podcasts_by_player.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Database/Migrations/2020-06-08-150000_add_analytics_podcasts_by_country.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-150000_add_analytics_podcasts_by_country.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Database/Migrations/2020-06-08-160000_add_analytics_podcasts_by_region.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-160000_add_analytics_podcasts_by_region.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Database/Migrations/2020-06-08-160000_add_podcasts_platforms.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-160000_add_podcasts_platforms.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Database/Migrations/2020-06-08-170000_add_analytics_website_by_browser.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-170000_add_analytics_website_by_browser.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Database/Migrations/2020-06-08-180000_add_analytics_website_by_referer.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-180000_add_analytics_website_by_referer.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Database/Migrations/2020-06-08-190000_add_analytics_website_by_entry_page.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-190000_add_analytics_website_by_entry_page.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

6
app/Database/Migrations/2020-06-08-210000_add_analytics_unknown_useragents.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-200000_add_analytics_unknown_useragents.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;
@ -38,10 +38,10 @@ class AddAnalyticsUnknownUseragents extends Migration
$this->forge->addPrimaryKey('id');
// `created_at` and `updated_at` are created with SQL because Model class won’t be used for insertion (Procedure will be used instead)
$this->forge->addField(
'`created_at` timestamp NOT NULL DEFAULT current_timestamp()'
'`created_at` timestamp NOT NULL DEFAULT current_timestamp()',
);
$this->forge->addField(
'`updated_at` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp()'
'`updated_at` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp()',
);
$this->forge->createTable('analytics_unknown_useragents');
}

2
app/Database/Migrations/2020-06-11-210000_add_analytics_podcasts_procedure.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-210000_add_analytics_podcasts_procedure.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Database/Migrations/2020-06-11-210000_add_analytics_unknown_useragents_procedure.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-210000_add_analytics_unknown_useragents_procedure.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Database/Migrations/2020-06-11-210000_add_analytics_website_procedure.php → app/Libraries/Analytics/Database/Migrations/2017-12-01-210000_add_analytics_website_procedure.php
View File

@ -9,7 +9,7 @@
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
namespace Analytics\Database\Migrations;
use CodeIgniter\Database\Migration;

2
app/Entities/AnalyticsPodcasts.php → app/Libraries/Analytics/Entities/AnalyticsPodcasts.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Entities;
namespace Analytics\Entities;
use CodeIgniter\Entity;

2
app/Entities/AnalyticsPodcastsByCountry.php → app/Libraries/Analytics/Entities/AnalyticsPodcastsByCountry.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Entities;
namespace Analytics\Entities;
use CodeIgniter\Entity;

2
app/Entities/AnalyticsPodcastsByEpisode.php → app/Libraries/Analytics/Entities/AnalyticsPodcastsByEpisode.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Entities;
namespace Analytics\Entities;
use CodeIgniter\Entity;

2
app/Entities/AnalyticsPodcastsByHour.php → app/Libraries/Analytics/Entities/AnalyticsPodcastsByHour.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Entities;
namespace Analytics\Entities;
use CodeIgniter\Entity;

2
app/Entities/AnalyticsPodcastsByPlayer.php → app/Libraries/Analytics/Entities/AnalyticsPodcastsByPlayer.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Entities;
namespace Analytics\Entities;
use CodeIgniter\Entity;

2
app/Entities/AnalyticsPodcastsByRegion.php → app/Libraries/Analytics/Entities/AnalyticsPodcastsByRegion.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Entities;
namespace Analytics\Entities;
use CodeIgniter\Entity;

4
app/Entities/AnalyticsPodcastsByService.php → app/Libraries/Analytics/Entities/AnalyticsPodcastsByService.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Entities;
namespace Analytics\Entities;
use CodeIgniter\Entity;
@ -32,7 +32,7 @@ class AnalyticsPodcastsByService extends Entity
public function getLabels()
{
return \Opawg\UserAgentsPhp\UserAgentsRSS::getName(
$this->attributes['labels']
$this->attributes['labels'],
) ?? $this->attributes['labels'];
}
}

2
app/Entities/AnalyticsUnknownUseragents.php → app/Libraries/Analytics/Entities/AnalyticsUnknownUseragents.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Entities;
namespace Analytics\Entities;
use CodeIgniter\Entity;

2
app/Entities/AnalyticsWebsiteByBrowser.php → app/Libraries/Analytics/Entities/AnalyticsWebsiteByBrowser.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Entities;
namespace Analytics\Entities;
use CodeIgniter\Entity;

2
app/Entities/AnalyticsWebsiteByEntryPage.php → app/Libraries/Analytics/Entities/AnalyticsWebsiteByEntryPage.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Entities;
namespace Analytics\Entities;
use CodeIgniter\Entity;

2
app/Entities/AnalyticsWebsiteByReferer.php → app/Libraries/Analytics/Entities/AnalyticsWebsiteByReferer.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Entities;
namespace Analytics\Entities;
use CodeIgniter\Entity;

451
app/Libraries/Analytics/Helpers/analytics_helper.php Normal file
View File

@ -0,0 +1,451 @@
<?php
/**
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
use CodeIgniter\Router\Exceptions\RouterException;
if (!function_exists('base64_url_encode')) {
/**
* Encode Base64 for URLs
*/
function base64_url_encode($input)
{
return strtr(base64_encode($input), '+/=', '._-');
}
}
if (!function_exists('base64_url_decode')) {
/**
* Decode Base64 from URL
*/
function base64_url_decode($input)
{
return base64_decode(strtr($input, '._-', '+/='));
}
}
if (!function_exists('generate_episode_analytics_url')) {
/**
* Builds the episode analytics url that redirects to the enclosure url
* after analytics hit.
*
* @param int $podcastId
* @param int $episodeId
* @param string $enclosureUri
* @param int $enclosureDuration
* @param int $enclosureFilesize
* @param int $enclosureHeadersize
* @param \CodeIgniter\I18n\Time $publicationDate
*
* @return string
* @throws RouterException
*/
function generate_episode_analytics_url(
$podcastId,
$episodeId,
$enclosureUri,
$enclosureDuration,
$enclosureFilesize,
$enclosureHeadersize,
$publicationDate
) {
return url_to(
'episode-analytics-hit',
base64_url_encode(
pack(
'I*',
$podcastId,
$episodeId,
// bytes_threshold: number of bytes that must be downloaded for an episode to be counted in download analytics
// - if file is shorter than 60sec, then it's enclosure_filesize
// - if file is longer than 60 seconds then it's enclosure_headersize + 60 seconds
$enclosureDuration <= 60
? $enclosureFilesize
: $enclosureHeadersize +
floor(
(($enclosureFilesize - $enclosureHeadersize) /
$enclosureDuration) *
60,
),
$enclosureFilesize,
$enclosureDuration,
strtotime($publicationDate),
),
),
$enclosureUri,
);
}
}
if (!function_exists('set_user_session_deny_list_ip')) {
/**
* Set user country in session variable, for analytic purposes
*/
function set_user_session_deny_list_ip()
{
$session = \Config\Services::session();
$session->start();
if (!$session->has('denyListIp')) {
$session->set(
'denyListIp',
\Podlibre\Ipcat\IpDb::find($_SERVER['REMOTE_ADDR']) != null,
);
}
}
}
if (!function_exists('set_user_session_location')) {
/**
* Set user country in session variable, for analytic purposes
*/
function set_user_session_location()
{
$session = \Config\Services::session();
$session->start();
$location = [
'countryCode' => 'N/A',
'regionCode' => 'N/A',
'latitude' => null,
'longitude' => null,
];
// Finds location:
if (!$session->has('location')) {
try {
$cityReader = new \GeoIp2\Database\Reader(
WRITEPATH . 'uploads/GeoLite2-City/GeoLite2-City.mmdb',
);
$city = $cityReader->city($_SERVER['REMOTE_ADDR']);
$location = [
'countryCode' => empty($city->country->isoCode)
? 'N/A'
: $city->country->isoCode,
'regionCode' => empty($city->subdivisions[0]->isoCode)
? 'N/A'
: $city->subdivisions[0]->isoCode,
'latitude' => round($city->location->latitude, 3),
'longitude' => round($city->location->longitude, 3),
];
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
}
$session->set('location', $location);
}
}
}
if (!function_exists('set_user_session_player')) {
/**
* Set user player in session variable, for analytic purposes
*/
function set_user_session_player()
{
$session = \Config\Services::session();
$session->start();
if (!$session->has('player')) {
$playerFound = null;
$userAgent = $_SERVER['HTTP_USER_AGENT'];
try {
$playerFound = \Opawg\UserAgentsPhp\UserAgents::find(
$userAgent,
);
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
}
if ($playerFound) {
$session->set('player', $playerFound);
} else {
$session->set('player', [
'app' => '- unknown -',
'device' => '',
'os' => '',
'bot' => 0,
]);
// Add to unknown list
try {
$db = \Config\Database::connect();
$procedureNameAnalyticsUnknownUseragents = $db->prefixTable(
'analytics_unknown_useragents',
);
$db->query(
"CALL $procedureNameAnalyticsUnknownUseragents(?)",
[$userAgent],
);
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
}
}
}
}
}
if (!function_exists('set_user_session_browser')) {
/**
* Set user browser in session variable, for analytic purposes
*
* @return void
*/
function set_user_session_browser()
{
$session = \Config\Services::session();
$session->start();
if (!$session->has('browser')) {
$browserName = '- Other -';
try {
$whichbrowser = new \WhichBrowser\Parser(getallheaders());
$browserName = $whichbrowser->browser->name;
} catch (\Exception $e) {
$browserName = '- Could not get browser name -';
}
if ($browserName == null) {
$browserName = '- Could not get browser name -';
}
$session->set('browser', $browserName);
}
}
}
if (!function_exists('set_user_session_referer')) {
/**
* Set user referer in session variable, for analytic purposes
*
* @return void
*/
function set_user_session_referer()
{
$session = \Config\Services::session();
$session->start();
$newreferer = isset($_SERVER['HTTP_REFERER'])
? $_SERVER['HTTP_REFERER']
: '- Direct -';
$newreferer =
parse_url($newreferer, PHP_URL_HOST) ==
parse_url(current_url(false), PHP_URL_HOST)
? '- Direct -'
: $newreferer;
if (!$session->has('referer') or $newreferer != '- Direct -') {
$session->set('referer', $newreferer);
}
}
}
if (!function_exists('set_user_session_entry_page')) {
/**
* Set user entry page in session variable, for analytic purposes
*
* @return void
*/
function set_user_session_entry_page()
{
$session = \Config\Services::session();
$session->start();
$entryPage = $_SERVER['REQUEST_URI'];
if (!$session->has('entryPage')) {
$session->set('entryPage', $entryPage);
}
}
}
if (!function_exists('webpage_hit')) {
/**
*
* @param integer $podcastId
* @return void
*/
function webpage_hit($podcastId)
{
$session = \Config\Services::session();
$session->start();
if (!$session->get('denyListIp')) {
$db = \Config\Database::connect();
$referer = $session->get('referer');
$domain = empty(parse_url($referer, PHP_URL_HOST))
? '- Direct -'
: parse_url($referer, PHP_URL_HOST);
parse_str(parse_url($referer, PHP_URL_QUERY), $queries);
$keywords = empty($queries['q']) ? null : $queries['q'];
$procedureName = $db->prefixTable('analytics_website');
$db->query("call $procedureName(?,?,?,?,?,?)", [
$podcastId,
$session->get('browser'),
$session->get('entryPage'),
$referer,
$domain,
$keywords,
]);
}
}
}
if (!function_exists('podcast_hit')) {
/**
* Counting podcast episode downloads for analytic purposes
* ✅ No IP address is ever stored on the server.
* ✅ Only aggregate data is stored in the database.
* We follow IAB Podcast Measurement Technical Guidelines Version 2.0:
* https://iabtechlab.com/standards/podcast-measurement-guidelines/
* https://iabtechlab.com/wp-content/uploads/2017/12/Podcast_Measurement_v2-Dec-20-2017.pdf
* ✅ Rolling 24-hour window
* ✅ Castopod does not do pre-load
* ✅ IP deny list https://github.com/client9/ipcat
* ✅ User-agent Filtering https://github.com/opawg/user-agents
* ✅ RSS User-agent https://github.com/opawg/podcast-rss-useragents
* ✅ Ignores 2 bytes range "Range: 0-1" (performed by official Apple iOS Podcast app)
* ✅ In case of partial content, adds up all requests to check >1mn was downloaded
* ✅ Identifying Uniques is done with a combination of IP Address and User Agent
* @param integer $podcastId The podcast ID
* @param integer $episodeId The Episode ID
* @param integer $bytesThreshold The minimum total number of bytes that must be downloaded so that an episode is counted (>1mn)
* @param integer $fileSize The podcast complete file size
* @param string $serviceName The name of the service that had fetched the RSS feed
*
* @return void
*/
function podcast_hit(
$podcastId,
$episodeId,
$bytesThreshold,
$fileSize,
$duration,
$publicationDate,
$serviceName
) {
$session = \Config\Services::session();
$session->start();
// We try to count (but if things went wrong the show should go on and the user should be able to download the file):
try {
// If the user IP is denied it's probably a bot:
if ($session->get('denyListIp')) {
$session->get('player')['bot'] = true;
}
//We get the HTTP header field `Range`:
$httpRange = isset($_SERVER['HTTP_RANGE'])
? $_SERVER['HTTP_RANGE']
: null;
// We create a sha1 hash for this IP_Address+User_Agent+Episode_ID (used to count only once multiple episode downloads):
$episodeHashId =
'_IpUaEp_' .
sha1(
$_SERVER['REMOTE_ADDR'] .
'_' .
$_SERVER['HTTP_USER_AGENT'] .
'_' .
$episodeId,
);
// Was this episode downloaded in the past 24h:
$downloadedBytes = cache($episodeHashId);
// Rolling window is 24 hours (86400 seconds):
$rollingTTL = 86400;
if ($downloadedBytes) {
// In case it was already downloaded, TTL should be adjusted (rolling window is 24h since 1st download):
$rollingTTL =
cache()->getMetadata($episodeHashId)['expire'] - time();
} else {
// If it was never downloaded that means that zero byte were downloaded:
$downloadedBytes = 0;
}
// If the number of downloaded bytes was previously below the 1mn threshold we go on:
// (Otherwise it means that this was already counted, therefore we don't do anything)
if ($downloadedBytes < $bytesThreshold) {
// If HTTP_RANGE is null we are downloading the complete file:
if (!$httpRange) {
$downloadedBytes = $fileSize;
} else {
// [0-1] bytes range requests are used (by Apple) to check that file exists and that 206 partial content is working.
// We don't count these requests:
if ($httpRange != 'bytes=0-1') {
// We calculate how many bytes are being downloaded based on HTTP_RANGE values:
$ranges = explode(',', substr($httpRange, 6));
foreach ($ranges as $range) {
$parts = explode('-', $range);
$downloadedBytes += empty($parts[1])
? $fileSize
: $parts[1] -
(empty($parts[0]) ? 0 : $parts[0]);
}
}
}
// We save the number of downloaded bytes for this user and this episode:
cache()->save($episodeHashId, $downloadedBytes, $rollingTTL);
// If more that 1mn was downloaded, that's a hit, we send that to the database:
if ($downloadedBytes >= $bytesThreshold) {
$db = \Config\Database::connect();
$procedureName = $db->prefixTable('analytics_podcasts');
$age = intdiv(time() - $publicationDate, 86400);
// We create a sha1 hash for this IP_Address+User_Agent+Podcast_ID (used to count unique listeners):
$listenerHashId =
'_IpUaPo_' .
sha1(
$_SERVER['REMOTE_ADDR'] .
'_' .
$_SERVER['HTTP_USER_AGENT'] .
'_' .
$podcastId,
);
$newListener = 1;
// Has this listener already downloaded an episode today:
$downloadsByUser = cache($listenerHashId);
// We add one download
if ($downloadsByUser) {
$newListener = 0;
$downloadsByUser++;
} else {
$downloadsByUser = 1;
}
// Listener count is calculated from 00h00 to 23h59:
$midnightTTL = strtotime('tomorrow') - time();
// We save the download count for this user until midnight:
cache()->save(
$listenerHashId,
$downloadsByUser,
$midnightTTL,
);
$db->query(
"CALL $procedureName(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);",
[
$podcastId,
$episodeId,
$session->get('location')['countryCode'],
$session->get('location')['regionCode'],
$session->get('location')['latitude'],
$session->get('location')['longitude'],
$serviceName,
$session->get('player')['app'],
$session->get('player')['device'],
$session->get('player')['os'],
$session->get('player')['bot'],
$fileSize,
$duration,
$age,
$newListener,
],
);
}
}
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
log_message('critical', $e);
}
}
}

4
app/Models/AnalyticsPodcastByCountryModel.php → app/Libraries/Analytics/Models/AnalyticsPodcastByCountryModel.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Models;
namespace Analytics\Models;
use CodeIgniter\Model;
@ -18,7 +18,7 @@ class AnalyticsPodcastByCountryModel extends Model
protected $allowedFields = [];
protected $returnType = \App\Entities\AnalyticsPodcastsByCountry::class;
protected $returnType = \Analytics\Entities\AnalyticsPodcastsByCountry::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;

90
app/Libraries/Analytics/Models/AnalyticsPodcastByEpisodeModel.php Normal file
View File

@ -0,0 +1,90 @@
<?php
/**
* Class AnalyticsPodcastByEpisodeModel
* Model for analytics_podcasts_by_episodes table in database
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
namespace Analytics\Models;
use CodeIgniter\Model;
class AnalyticsPodcastByEpisodeModel extends Model
{
protected $table = 'analytics_podcasts_by_episode';
protected $allowedFields = [];
protected $returnType = \Analytics\Entities\AnalyticsPodcastsByEpisode::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;
/**
* @param int $podcastId
* @param int $episodeId
*
* @return array
*/
public function getDataByDay(int $podcastId, int $episodeId): array
{
if (
!($found = cache(
"{$podcastId}_{$episodeId}_analytics_podcast_by_episode_by_day",
))
) {
$found = $this->select('date as labels')
->selectSum('hits', 'values')
->where([
'episode_id' => $episodeId,
'podcast_id' => $podcastId,
'age <' => 60,
])
->groupBy('labels')
->orderBy('labels', 'ASC')
->findAll();
cache()->save(
"{$podcastId}_{$episodeId}_analytics_podcast_by_episode_by_day",
$found,
600,
);
}
return $found;
}
/**
* @param int $podcastId
* @param int $episodeId
*
* @return array
*/
public function getDataByMonth(int $podcastId, int $episodeId = null): array
{
if (
!($found = cache(
"{$podcastId}_{$episodeId}_analytics_podcast_by_episode_by_month",
))
) {
$found = $this->select('DATE_FORMAT(date,"%Y-%m-01") as labels')
->selectSum('hits', 'values')
->where([
'episode_id' => $episodeId,
'podcast_id' => $podcastId,
])
->groupBy('labels')
->orderBy('labels', 'ASC')
->findAll();
cache()->save(
"{$podcastId}_{$episodeId}_analytics_podcast_by_episode_by_month",
$found,
600,
);
}
return $found;
}
}

4
app/Models/AnalyticsPodcastByHourModel.php → app/Libraries/Analytics/Models/AnalyticsPodcastByHourModel.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Models;
namespace Analytics\Models;
use CodeIgniter\Model;
@ -18,7 +18,7 @@ class AnalyticsPodcastByHourModel extends Model
protected $allowedFields = [];
protected $returnType = \App\Entities\AnalyticsPodcastsByHour::class;
protected $returnType = \Analytics\Entities\AnalyticsPodcastsByHour::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;

4
app/Models/AnalyticsPodcastByPlayerModel.php → app/Libraries/Analytics/Models/AnalyticsPodcastByPlayerModel.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Models;
namespace Analytics\Models;
use CodeIgniter\Model;
@ -18,7 +18,7 @@ class AnalyticsPodcastByPlayerModel extends Model
protected $allowedFields = [];
protected $returnType = \App\Entities\AnalyticsPodcastsByPlayer::class;
protected $returnType = \Analytics\Entities\AnalyticsPodcastsByPlayer::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;

4
app/Models/AnalyticsPodcastByRegionModel.php → app/Libraries/Analytics/Models/AnalyticsPodcastByRegionModel.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Models;
namespace Analytics\Models;
use CodeIgniter\Model;
@ -18,7 +18,7 @@ class AnalyticsPodcastByRegionModel extends Model
protected $allowedFields = [];
protected $returnType = \App\Entities\AnalyticsPodcastsByRegion::class;
protected $returnType = \Analytics\Entities\AnalyticsPodcastsByRegion::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;

4
app/Models/AnalyticsPodcastByServiceModel.php → app/Libraries/Analytics/Models/AnalyticsPodcastByServiceModel.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Models;
namespace Analytics\Models;
use CodeIgniter\Model;
@ -18,7 +18,7 @@ class AnalyticsPodcastByServiceModel extends Model
protected $allowedFields = [];
protected $returnType = \App\Entities\AnalyticsPodcastsByService::class;
protected $returnType = \Analytics\Entities\AnalyticsPodcastsByService::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;

4
app/Models/AnalyticsPodcastModel.php → app/Libraries/Analytics/Models/AnalyticsPodcastModel.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Models;
namespace Analytics\Models;
use CodeIgniter\Model;
@ -18,7 +18,7 @@ class AnalyticsPodcastModel extends Model
protected $allowedFields = [];
protected $returnType = \App\Entities\AnalyticsPodcasts::class;
protected $returnType = \Analytics\Entities\AnalyticsPodcasts::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;

4
app/Models/AnalyticsUnknownUseragentsModel.php → app/Libraries/Analytics/Models/AnalyticsUnknownUseragentsModel.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Models;
namespace Analytics\Models;
use CodeIgniter\Model;
@ -19,7 +19,7 @@ class AnalyticsUnknownUseragentsModel extends Model
protected $allowedFields = [];
protected $returnType = \App\Entities\AnalyticsUnknownUseragents::class;
protected $returnType = \Analytics\Entities\AnalyticsUnknownUseragents::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;

4
app/Models/AnalyticsWebsiteByBrowserModel.php → app/Libraries/Analytics/Models/AnalyticsWebsiteByBrowserModel.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Models;
namespace Analytics\Models;
use CodeIgniter\Model;
@ -18,7 +18,7 @@ class AnalyticsWebsiteByBrowserModel extends Model
protected $allowedFields = [];
protected $returnType = \App\Entities\AnalyticsWebsiteByBrowser::class;
protected $returnType = \Analytics\Entities\AnalyticsWebsiteByBrowser::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;

4
app/Models/AnalyticsWebsiteByEntryPageModel.php → app/Libraries/Analytics/Models/AnalyticsWebsiteByEntryPageModel.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Models;
namespace Analytics\Models;
use CodeIgniter\Model;
@ -18,7 +18,7 @@ class AnalyticsWebsiteByEntryPageModel extends Model
protected $allowedFields = [];
protected $returnType = \App\Entities\AnalyticsWebsiteByEntryPage::class;
protected $returnType = \Analytics\Entities\AnalyticsWebsiteByEntryPage::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;

4
app/Models/AnalyticsWebsiteByRefererModel.php → app/Libraries/Analytics/Models/AnalyticsWebsiteByRefererModel.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Models;
namespace Analytics\Models;
use CodeIgniter\Model;
@ -18,7 +18,7 @@ class AnalyticsWebsiteByRefererModel extends Model
protected $allowedFields = [];
protected $returnType = \App\Entities\AnalyticsWebsiteByReferer::class;
protected $returnType = \Analytics\Entities\AnalyticsWebsiteByReferer::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;

2
app/Models/UnknownUserAgentsModel.php → app/Libraries/Analytics/Models/UnknownUserAgentsModel.php
View File

@ -8,7 +8,7 @@
* @link https://castopod.org/
*/
namespace App\Models;
namespace Analytics\Models;
use CodeIgniter\Model;

145
app/Models/AnalyticsPodcastByEpisodeModel.php
View File

@ -1,145 +0,0 @@
<?php
/**
* Class AnalyticsPodcastByEpisodeModel
* Model for analytics_podcasts_by_episodes table in database
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
namespace App\Models;
use CodeIgniter\Model;
class AnalyticsPodcastByEpisodeModel extends Model
{
protected $table = 'analytics_podcasts_by_episode';
protected $allowedFields = [];
protected $returnType = \App\Entities\AnalyticsPodcastsByEpisode::class;
protected $useSoftDeletes = false;
protected $useTimestamps = false;
/**
* @param int $podcastId, $episodeId
*
* @return array
*/
public function getDataByDay(int $podcastId, int $episodeId = null): array
{
if (!$episodeId) {
if (
!($found = cache(
"{$podcastId}_analytics_podcast_by_episode_by_day",
))
) {
$lastEpisodes = (new EpisodeModel())
->select('id, season_number, number, title')
->orderBy('id', 'DESC')
->where(['podcast_id' => $podcastId])
->findAll(5);
$found = $this->select('age AS X');
$letter = 97;
foreach ($lastEpisodes as $episode) {
$found = $found
->selectSum(
'(CASE WHEN episode_id=' .
$episode->id .
' THEN hits END)',
'' . chr($letter) . 'Y',
)
->select(
'"' .
(empty($episode->season_number)
? ''
: $episode->season_number) .
(empty($episode->number)
? ''
: '-' . $episode->number . '/ ') .
$episode->title .
'" AS ' .
chr($letter) .
'Value',
);
$letter++;
}
$found = $found
->where([
'podcast_id' => $podcastId,
'age <' => 60,
])
->groupBy('X')
->orderBy('X', 'ASC')
->findAll();
cache()->save(
"{$podcastId}_analytics_podcast_by_episode_by_day",
$found,
600,
);
}
return $found;
} else {
if (
!($found = cache(
"{$podcastId}_{$episodeId}_analytics_podcast_by_episode_by_day",
))
) {
$found = $this->select('date as labels')
->selectSum('hits', 'values')
->where([
'episode_id' => $episodeId,
'podcast_id' => $podcastId,
'age <' => 60,
])
->groupBy('labels')
->orderBy('labels', 'ASC')
->findAll();
cache()->save(
"{$podcastId}_{$episodeId}_analytics_podcast_by_episode_by_day",
$found,
600,
);
}
return $found;
}
}
/**
* @param int $podcastId, $episodeId
*
* @return array
*/
public function getDataByMonth(int $podcastId, int $episodeId = null): array
{
if (
!($found = cache(
"{$podcastId}_{$episodeId}_analytics_podcast_by_episode_by_month",
))
) {
$found = $this->select('DATE_FORMAT(date,"%Y-%m-01") as labels')
->selectSum('hits', 'values')
->where([
'episode_id' => $episodeId,
'podcast_id' => $podcastId,
])
->groupBy('labels')
->orderBy('labels', 'ASC')
->findAll();
cache()->save(
"{$podcastId}_{$episodeId}_analytics_podcast_by_episode_by_month",
$found,
600,
);
}
return $found;
}
}

16
app/Views/admin/podcast/analytics/index.php
View File

@ -15,7 +15,7 @@
'analytics-data',
$podcast->id,
'Podcast',
'ByDay'
'ByDay',
) ?>"></div>
</div>
@ -25,7 +25,7 @@
'analytics-data',
$podcast->id,
'Podcast',
'ByMonth'
'ByMonth',
) ?>"></div>
</div>
@ -35,17 +35,7 @@
'analytics-data',
$podcast->id,
'Podcast',
'BandwidthByDay'
) ?>"></div>
</div>
<div class="mb-12 text-center">
<h2><?= lang('Charts.episodes_by_day') ?></h2>
<div class="chart-xy" id="by-age-graph" data-chart-type="xy-series-chart" data-chart-url="<?= route_to(
'analytics-data',
$podcast->id,
'PodcastByEpisode',
'ByDay'
'BandwidthByDay',
) ?>"></div>
</div>

26
docs/setup-development.md
View File

@ -95,9 +95,9 @@ Go to project's root folder and run:
docker-compose up -d
# See all running processes (you should see 3 processes running)
docker ps
docker-compose ps
# Alternatively, you can check all processes (you should see composer with an Exited status)
# Alternatively, you can check all docker processes (you should see composer and npm with an Exited status)
docker ps -a
```
@ -146,17 +146,20 @@ docker-compose run --rm app php spark db:seed LanguageSeeder
docker-compose run --rm app php spark db:seed PlatformSeeder
# Populates all Authentication data (roles definition…)
docker-compose run --rm app php spark db:seed AuthSeeder
# Populates test data (login: admin / password: AGUehL3P)
docker-compose run --rm app php spark db:seed TestSeeder
```
3. (optionnal) Populate the database with test data:
```bash
# Populates test data (login: admin / password: AGUehL3P)
docker-compose run --rm app php spark db:seed TestSeeder
# Populates with fake podcast analytics
docker-compose run --rm app php spark db:seed FakePodcastsAnalyticsSeeder
# Populates with fake website analytics
docker-compose run --rm app php spark db:seed FakeWebsiteAnalyticsSeeder
```
This will add an active superadmin user with the following credentials:
TestSeeder will add an active superadmin user with the following credentials:
- username: **admin**
- password: **AGUehL3P**
@ -205,8 +208,8 @@ To see your changes, go to:
- [localhost:8080](http://localhost:8080/) for the castopod app
- [localhost:8888](http://localhost:8888/) for the phpmyadmin interface:
- **Username**: podlibre
- **Password**: castopod
- username: **podlibre**
- password: **castopod**
---
@ -216,19 +219,22 @@ To see your changes, go to:
```bash
# monitor the app container
docker logs --tail 50 --follow --timestamps castopod_app
docker-compose logs --tail 50 --follow --timestamps app
# monitor the mariadb container
docker logs --tail 50 --follow --timestamps castopod_mariadb
docker-compose logs --tail 50 --follow --timestamps mariadb
# monitor the phpmyadmin container
docker logs --tail 50 --follow --timestamps castopod_phpmyadmin
docker-compose logs --tail 50 --follow --timestamps phpmyadmin
# restart docker containers
docker-compose restart
# Destroy all containers, opposite of `up` command
docker-compose down
# Rebuild app container
docker-compose build app
```
Check [docker](https://docs.docker.com/engine/reference/commandline/docker/) and