2018-07-22 12:50:51 +02:00
|
|
|
#include "network.h"
|
|
|
|
|
2021-08-22 03:26:09 +02:00
|
|
|
#include "log.h"
|
2021-09-04 13:40:37 +02:00
|
|
|
#include "memcache.h"
|
2021-09-03 22:39:31 +02:00
|
|
|
#include "util.h"
|
2019-04-30 09:05:46 +02:00
|
|
|
|
2018-07-24 18:37:23 +02:00
|
|
|
#include <openssl/crypto.h>
|
|
|
|
|
2018-07-21 01:39:51 +02:00
|
|
|
#include <errno.h>
|
2018-07-22 12:50:51 +02:00
|
|
|
#include <string.h>
|
|
|
|
#include <stdio.h>
|
2018-07-24 03:40:53 +02:00
|
|
|
#include <unistd.h>
|
2018-07-18 17:26:26 +02:00
|
|
|
|
2021-08-30 12:24:32 +02:00
|
|
|
/*
|
|
|
|
* ----------------- External variables ----------------------
|
|
|
|
*/
|
2018-07-30 15:20:04 +02:00
|
|
|
CURLSH *CURL_SHARE;
|
2018-07-24 04:16:47 +02:00
|
|
|
|
2021-08-30 12:24:32 +02:00
|
|
|
/*
|
|
|
|
* ----------------- Static variable -----------------------
|
|
|
|
*/
|
2018-07-23 21:55:20 +02:00
|
|
|
/** \brief curl multi interface handle */
|
|
|
|
static CURLM *curl_multi;
|
2018-07-25 04:01:12 +02:00
|
|
|
/** \brief mutex for transfer functions */
|
2018-07-24 16:22:18 +02:00
|
|
|
static pthread_mutex_t transfer_lock;
|
2018-07-24 18:37:23 +02:00
|
|
|
/** \brief the lock array for cryptographic functions */
|
|
|
|
static pthread_mutex_t *crypto_lockarray;
|
2018-07-30 15:20:04 +02:00
|
|
|
/** \brief mutex for curl share interface itself */
|
2018-07-24 16:22:18 +02:00
|
|
|
static pthread_mutex_t curl_lock;
|
2018-07-23 19:46:52 +02:00
|
|
|
|
2021-08-30 12:24:32 +02:00
|
|
|
/*
|
|
|
|
* -------------------- Functions --------------------------
|
|
|
|
*/
|
2019-08-24 19:13:47 +02:00
|
|
|
#pragma GCC diagnostic push
|
|
|
|
#pragma GCC diagnostic ignored "-Wunused-function"
|
2019-08-27 11:52:46 +02:00
|
|
|
/**
|
2019-09-01 09:52:18 +02:00
|
|
|
* \brief OpenSSL 1.02 cryptography callback function
|
|
|
|
* \details Required for OpenSSL 1.02, but not OpenSSL 1.1
|
2019-08-27 11:52:46 +02:00
|
|
|
*/
|
2018-07-25 02:06:54 +02:00
|
|
|
static void crypto_lock_callback(int mode, int type, char *file, int line)
|
2018-07-25 01:51:05 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
(void) file;
|
|
|
|
(void) line;
|
|
|
|
if (mode & CRYPTO_LOCK) {
|
|
|
|
PTHREAD_MUTEX_LOCK(&(crypto_lockarray[type]));
|
|
|
|
} else {
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&(crypto_lockarray[type]));
|
|
|
|
}
|
2018-07-25 02:06:54 +02:00
|
|
|
}
|
2018-07-24 16:22:18 +02:00
|
|
|
|
2019-08-27 11:52:46 +02:00
|
|
|
/**
|
2019-09-01 09:52:18 +02:00
|
|
|
* \brief OpenSSL 1.02 thread ID function
|
|
|
|
* \details Required for OpenSSL 1.02, but not OpenSSL 1.1
|
2019-08-27 11:52:46 +02:00
|
|
|
*/
|
2019-04-22 14:32:15 +02:00
|
|
|
static unsigned long thread_id(void)
|
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
unsigned long ret;
|
2019-04-22 14:32:15 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
ret = (unsigned long) pthread_self();
|
|
|
|
return ret;
|
2019-04-22 14:32:15 +02:00
|
|
|
}
|
2021-08-30 12:24:32 +02:00
|
|
|
|
2019-08-24 19:13:47 +02:00
|
|
|
#pragma GCC diagnostic pop
|
2019-04-22 14:32:15 +02:00
|
|
|
|
2018-07-25 02:06:54 +02:00
|
|
|
static void crypto_lock_init(void)
|
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
int i;
|
|
|
|
|
|
|
|
crypto_lockarray =
|
|
|
|
(pthread_mutex_t *) OPENSSL_malloc(CRYPTO_num_locks() *
|
|
|
|
sizeof(pthread_mutex_t));
|
|
|
|
for (i = 0; i < CRYPTO_num_locks(); i++) {
|
|
|
|
if (pthread_mutex_init(&(crypto_lockarray[i]), NULL)) {
|
|
|
|
lprintf(fatal, "crypto_lockarray[%d] initialisation \
|
2021-08-30 12:24:32 +02:00
|
|
|
failed!\n", i);
|
2021-08-31 12:18:39 +02:00
|
|
|
};
|
|
|
|
}
|
2021-08-30 12:24:32 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
CRYPTO_set_id_callback((unsigned long (*)()) thread_id);
|
|
|
|
CRYPTO_set_locking_callback((void (*)()) crypto_lock_callback);
|
2018-07-23 21:55:20 +02:00
|
|
|
}
|
2018-07-23 19:46:52 +02:00
|
|
|
|
2018-07-24 03:40:53 +02:00
|
|
|
/**
|
2019-09-01 09:52:18 +02:00
|
|
|
* \brief Curl share handle callback function
|
|
|
|
* \details Adapted from:
|
2019-04-24 04:43:43 +02:00
|
|
|
* https://curl.haxx.se/libcurl/c/threaded-shared-conn.html
|
2018-07-24 03:40:53 +02:00
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static void
|
2021-09-03 15:56:11 +02:00
|
|
|
curl_callback_lock(CURL *handle, curl_lock_data data,
|
2021-08-31 12:15:00 +02:00
|
|
|
curl_lock_access access, void *userptr)
|
2018-07-25 02:06:54 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
(void) access; /* unused */
|
|
|
|
(void) userptr; /* unused */
|
|
|
|
(void) handle; /* unused */
|
|
|
|
(void) data; /* unused */
|
|
|
|
PTHREAD_MUTEX_LOCK(&curl_lock);
|
2018-07-25 02:06:54 +02:00
|
|
|
}
|
|
|
|
|
2021-08-30 12:24:32 +02:00
|
|
|
static void
|
2021-09-03 15:56:11 +02:00
|
|
|
curl_callback_unlock(CURL *handle, curl_lock_data data, void *userptr)
|
2018-07-25 02:06:54 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
(void) userptr; /* unused */
|
|
|
|
(void) handle; /* unused */
|
|
|
|
(void) data; /* unused */
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&curl_lock);
|
2018-07-25 02:06:54 +02:00
|
|
|
}
|
|
|
|
|
2019-04-24 04:43:43 +02:00
|
|
|
/**
|
2019-09-01 09:52:18 +02:00
|
|
|
* \brief Process a curl message
|
|
|
|
* \details Adapted from:
|
2019-04-24 04:43:43 +02:00
|
|
|
* https://curl.haxx.se/libcurl/c/10-at-a-time.html
|
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static void
|
2021-09-03 15:56:11 +02:00
|
|
|
curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, int n_mesgs)
|
2019-04-22 14:32:15 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
(void) n_running_curl;
|
|
|
|
(void) n_mesgs;
|
|
|
|
static volatile int slept = 0;
|
|
|
|
if (curl_msg->msg == CURLMSG_DONE) {
|
2021-09-03 13:40:35 +02:00
|
|
|
TransferStruct *ts;
|
2021-08-31 12:18:39 +02:00
|
|
|
CURL *curl = curl_msg->easy_handle;
|
2021-09-03 13:47:48 +02:00
|
|
|
CURLcode ret =
|
|
|
|
curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_PRIVATE,
|
|
|
|
&ts);
|
|
|
|
if (ret) {
|
|
|
|
lprintf(error, "%s", curl_easy_strerror(ret));
|
|
|
|
}
|
2021-09-03 13:40:35 +02:00
|
|
|
ts->transferring = 0;
|
2021-08-31 12:18:39 +02:00
|
|
|
char *url = NULL;
|
2021-09-03 13:47:48 +02:00
|
|
|
ret = curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url);
|
|
|
|
if (ret) {
|
|
|
|
lprintf(error, "%s", curl_easy_strerror(ret));
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
|
|
|
/*
|
2021-08-31 12:18:39 +02:00
|
|
|
* Wait for 5 seconds if we get HTTP 429
|
2021-08-31 12:15:00 +02:00
|
|
|
*/
|
2021-08-31 12:18:39 +02:00
|
|
|
long http_resp = 0;
|
2021-09-03 13:47:48 +02:00
|
|
|
ret = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
|
|
|
if (ret) {
|
|
|
|
lprintf(error, "%s", curl_easy_strerror(ret));
|
|
|
|
}
|
2021-08-31 12:18:39 +02:00
|
|
|
if (HTTP_temp_failure(http_resp)) {
|
|
|
|
if (!slept) {
|
|
|
|
lprintf(warning,
|
|
|
|
"HTTP %ld, sleeping for %d sec\n",
|
|
|
|
http_resp, CONFIG.http_wait_sec);
|
|
|
|
sleep(CONFIG.http_wait_sec);
|
|
|
|
slept = 1;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
slept = 0;
|
2021-08-31 12:15:00 +02:00
|
|
|
}
|
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (!curl_msg->data.result) {
|
|
|
|
/*
|
|
|
|
* Transfer successful, set the file size
|
|
|
|
*/
|
2021-09-03 13:40:35 +02:00
|
|
|
if (ts->type == FILESTAT) {
|
|
|
|
Link_set_file_stat(ts->link, curl);
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
lprintf(error, "%d - %s <%s>\n",
|
|
|
|
curl_msg->data.result,
|
|
|
|
curl_easy_strerror(curl_msg->data.result), url);
|
2021-08-31 12:15:00 +02:00
|
|
|
}
|
2021-08-31 12:18:39 +02:00
|
|
|
curl_multi_remove_handle(curl_multi, curl);
|
2021-08-31 12:15:00 +02:00
|
|
|
/*
|
2021-08-31 12:18:39 +02:00
|
|
|
* clean up the handle, if we are querying the file size
|
2021-08-31 12:15:00 +02:00
|
|
|
*/
|
2021-09-03 13:40:35 +02:00
|
|
|
if (ts->type == FILESTAT) {
|
2021-08-31 12:18:39 +02:00
|
|
|
curl_easy_cleanup(curl);
|
2021-09-03 13:40:35 +02:00
|
|
|
FREE(ts);
|
2021-08-31 12:15:00 +02:00
|
|
|
}
|
2021-08-31 12:18:39 +02:00
|
|
|
} else {
|
|
|
|
lprintf(warning, "curl_msg->msg: %d\n", curl_msg->msg);
|
|
|
|
}
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
/**
|
|
|
|
* \details effectively based on
|
|
|
|
* https://curl.haxx.se/libcurl/c/multi-double.html
|
|
|
|
*/
|
|
|
|
int curl_multi_perform_once(void)
|
|
|
|
{
|
|
|
|
lprintf(network_lock_debug,
|
|
|
|
"thread %x: locking transfer_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_LOCK(&transfer_lock);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get curl multi interface to perform pending tasks
|
|
|
|
*/
|
|
|
|
int n_running_curl;
|
|
|
|
CURLMcode mc = curl_multi_perform(curl_multi, &n_running_curl);
|
2021-09-03 17:36:50 +02:00
|
|
|
if (mc) {
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(error, "%s\n", curl_multi_strerror(mc));
|
|
|
|
}
|
|
|
|
|
2021-09-03 17:36:50 +02:00
|
|
|
mc = curl_multi_poll(curl_multi, NULL, 0, 100, NULL);
|
|
|
|
if (mc) {
|
|
|
|
lprintf(error, "%s\n", curl_multi_strerror(mc));
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Process the message queue
|
|
|
|
*/
|
|
|
|
int n_mesgs;
|
|
|
|
CURLMsg *curl_msg;
|
|
|
|
while ((curl_msg = curl_multi_info_read(curl_multi, &n_mesgs))) {
|
|
|
|
curl_process_msgs(curl_msg, n_running_curl, n_mesgs);
|
|
|
|
}
|
|
|
|
|
|
|
|
lprintf(network_lock_debug,
|
|
|
|
"thread %x: unlocking transfer_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&transfer_lock);
|
|
|
|
|
|
|
|
return n_running_curl;
|
2018-07-25 05:51:38 +02:00
|
|
|
}
|
|
|
|
|
2019-10-22 02:49:53 +02:00
|
|
|
void NetworkSystem_init(void)
|
2018-07-30 15:20:04 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
/*
|
|
|
|
* ------- Global related ----------
|
|
|
|
*/
|
|
|
|
if (curl_global_init(CURL_GLOBAL_ALL)) {
|
|
|
|
lprintf(fatal, "curl_global_init() failed!\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* -------- Share related ----------
|
|
|
|
*/
|
|
|
|
CURL_SHARE = curl_share_init();
|
|
|
|
if (!(CURL_SHARE)) {
|
|
|
|
lprintf(fatal, "curl_share_init() failed!\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
curl_share_setopt(CURL_SHARE, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
|
|
|
|
curl_share_setopt(CURL_SHARE, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
|
|
|
|
curl_share_setopt(CURL_SHARE, CURLSHOPT_SHARE,
|
|
|
|
CURL_LOCK_DATA_SSL_SESSION);
|
|
|
|
|
|
|
|
if (pthread_mutex_init(&curl_lock, NULL)) {
|
|
|
|
lprintf(fatal, "curl_lock initialisation failed!\n");
|
|
|
|
}
|
|
|
|
curl_share_setopt(CURL_SHARE, CURLSHOPT_LOCKFUNC, curl_callback_lock);
|
|
|
|
curl_share_setopt(CURL_SHARE, CURLSHOPT_UNLOCKFUNC,
|
|
|
|
curl_callback_unlock);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ------------- Multi related -----------
|
|
|
|
*/
|
|
|
|
curl_multi = curl_multi_init();
|
|
|
|
if (!curl_multi) {
|
|
|
|
lprintf(fatal, "curl_multi_init() failed!\n");
|
|
|
|
}
|
|
|
|
curl_multi_setopt(curl_multi, CURLMOPT_MAX_TOTAL_CONNECTIONS,
|
|
|
|
CONFIG.max_conns);
|
|
|
|
curl_multi_setopt(curl_multi, CURLMOPT_MAX_HOST_CONNECTIONS,
|
|
|
|
CONFIG.max_conns);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ------------ Initialise locks ---------
|
|
|
|
*/
|
|
|
|
if (pthread_mutex_init(&transfer_lock, NULL)) {
|
|
|
|
lprintf(fatal, "transfer_lock initialisation failed!\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* cryptographic lock functions were shamelessly copied from
|
|
|
|
* https://curl.haxx.se/libcurl/c/threaded-ssl.html
|
|
|
|
*/
|
|
|
|
crypto_lock_init();
|
2018-07-25 01:51:05 +02:00
|
|
|
}
|
|
|
|
|
2021-09-03 15:56:11 +02:00
|
|
|
void transfer_blocking(CURL *curl)
|
2018-07-25 01:51:05 +02:00
|
|
|
{
|
2021-09-03 13:40:35 +02:00
|
|
|
TransferStruct *ts;
|
2021-09-03 13:47:48 +02:00
|
|
|
CURLcode ret = curl_easy_getinfo(curl, CURLINFO_PRIVATE, &ts);
|
|
|
|
if (ret) {
|
|
|
|
lprintf(error, "%s", curl_easy_strerror(ret));
|
|
|
|
}
|
2021-08-31 12:18:39 +02:00
|
|
|
|
|
|
|
lprintf(network_lock_debug,
|
|
|
|
"thread %x: locking transfer_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_LOCK(&transfer_lock);
|
|
|
|
|
|
|
|
CURLMcode res = curl_multi_add_handle(curl_multi, curl);
|
|
|
|
|
|
|
|
lprintf(network_lock_debug,
|
|
|
|
"thread %x: unlocking transfer_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&transfer_lock);
|
|
|
|
|
|
|
|
if (res > 0) {
|
|
|
|
lprintf(error, "%d, %s\n", res, curl_multi_strerror(res));
|
|
|
|
}
|
|
|
|
|
2021-09-03 13:40:35 +02:00
|
|
|
while (ts->transferring) {
|
2021-08-31 12:18:39 +02:00
|
|
|
curl_multi_perform_once();
|
|
|
|
}
|
2018-07-25 02:06:54 +02:00
|
|
|
}
|
2018-07-25 01:51:05 +02:00
|
|
|
|
2021-09-03 15:56:11 +02:00
|
|
|
void transfer_nonblocking(CURL *curl)
|
2018-07-25 02:06:54 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(network_lock_debug,
|
|
|
|
"thread %x: locking transfer_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_LOCK(&transfer_lock);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
CURLMcode res = curl_multi_add_handle(curl_multi, curl);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(network_lock_debug,
|
|
|
|
"thread %x: unlocking transfer_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&transfer_lock);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (res > 0) {
|
|
|
|
lprintf(error, "%s\n", curl_multi_strerror(res));
|
|
|
|
}
|
2018-07-25 01:51:05 +02:00
|
|
|
}
|
|
|
|
|
2019-09-04 19:30:57 +02:00
|
|
|
int HTTP_temp_failure(HTTPResponseCode http_resp)
|
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
switch (http_resp) {
|
|
|
|
case HTTP_TOO_MANY_REQUESTS:
|
|
|
|
case HTTP_CLOUDFLARE_UNKNOWN_ERROR:
|
|
|
|
case HTTP_CLOUDFLARE_TIMEOUT:
|
|
|
|
return 1;
|
|
|
|
default:
|
|
|
|
return 0;
|
|
|
|
}
|
2019-09-04 19:30:57 +02:00
|
|
|
}
|