/*
* @file istream68_curl.c
* @brief implements istream68 VFS for cURL
* @author http://sourceforge.net/users/benjihan
*
* Copyright (c) 1998-2015 Benjamin Gerard
*
* This program is free software: you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program.
*
* If not, see .
*
*/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif
#include "file68_api.h"
#include "file68_vfs_curl.h"
#include "file68_msg.h"
/* Define this if you want CURL support. */
#ifdef USE_CURL
#ifndef DEBUG_CURL_O
# define DEBUG_CURL_O 0
#endif
static int curl_cat = msg68_DEFAULT;
static const char curl_schemes[] = CURL_SCHEMES;
#include "file68_vfs_def.h"
#include "file68_uri.h"
#include "file68_str.h"
#include
#include
#include
#if defined (HAVE_UNISTD_H) && (defined (HAVE_USLEEP) || defined (HAVE_SLEEP))
/* Need this to have usleep prototype. */
# if defined (HAVE_USLEEP) && !defined(__USE_BSD) && !defined(__USE_XOPEN)
# define __USE_BSD
# endif
# include
#endif
#ifndef ISF_VERBOSE
# ifdef _DEBUG
# define ISF_VERBOSE 1
# else
# define ISF_VERBOSE 0
# endif
#endif
/* Timeout in ms */
#define IS68_TIMEOUT 10000
#define IS68_CACHE_FIX 8
#define IS68_CACHE_SZ (1<cache+i;
c->top = -1;
c->bytes = 0;
}
}
static const char * isf_name(vfs68_t * vfs)
{
vfs68_curl_t * isf = (vfs68_curl_t *)vfs;
return (!isf->name[0])
? 0
: isf->name
;
}
/* $$$ TODO */
static size_t isf_read_cb(void *ptr, size_t size, size_t nmemb, void *stream)
{
vfs68_curl_t * isf = (vfs68_curl_t *)stream;
int bytes, org_bytes/* , pos */;
bytes = org_bytes = (int)(size * nmemb);
/* pos = isf->curl_pos; */
isf->has_read = 1;
if (!bytes) {
return 0; /* Avoid 0 divide at ending return */
}
return (org_bytes - bytes) / size;
}
static size_t isf_write_cb(void *ptr, size_t size, size_t nmemb, void *stream)
{
vfs68_curl_t * isf = (vfs68_curl_t *)stream;
int bytes, org_bytes, pos;
int off, top, blk;
bytes = org_bytes = (int)(size * nmemb);
pos = isf->curl_pos;
isf->has_write = 1;
if (!bytes) {
return 0; /* Avoid 0 divide at ending return */
}
off = pos & (IS68_CACHE_SZ-1);
top = pos - off;
blk = (pos>>IS68_CACHE_FIX) & (IS68_CACHE_BLKS-1);
while (bytes > 0) {
int n;
if (!off) {
/* Starting a new cache block */
isf->cache[blk].top = top;
} else if (isf->cache[blk].top != top) {
/* Oops, got a problem here */
break;
}
n = IS68_CACHE_SZ - off;
if (n > bytes) {
n = bytes;
}
memcpy(isf->cache[blk].buffer+off, ptr, n);
ptr = (char *)ptr + n;
isf->cache[blk].bytes = n+off;
bytes -= n;
off = 0;
blk = (blk+1) & (IS68_CACHE_BLKS-1);
top += IS68_CACHE_SZ;
}
org_bytes -= bytes;
isf->curl_pos = pos + org_bytes;
return org_bytes / size;
}
#if 0
static int isf_passwd_cb(void *clientp,
const char *prompt, char *buffer, int buflen)
{
/* Produce a CURLE_BAD_PASSWORD_ENTERED error. */
return -1;
}
#endif
static int isf_prog_cb(void *clientp,
double dltotal,double dlnow,
double ultotal,double ulnow)
{
return 0;
}
static int isf_debug_cb(CURL *handle, curl_infotype type,
char *data, size_t size,
void *userp)
{
#if 0
vfs68_curl_t * isf = (vfs68_curl_t *)userp;
const char * typestr;
switch (type) {
case CURLINFO_TEXT:
typestr = "TEXT";
break;
case CURLINFO_HEADER_IN:
typestr = "HEADER_IN";
break;
case CURLINFO_HEADER_OUT:
typestr = "HEADER_OUT";
break;
case CURLINFO_DATA_IN:
typestr = "DATA_IN";
data = 0;
break;
case CURLINFO_DATA_OUT:
typestr = "DATA_OUT";
data = 0;
break;
case CURLINFO_END:
typestr = "DATA_END";
data = 0;
break;
default:
typestr = "UNKNOWN";
data = 0;
break;
}
TRACE68(curl_cat,"%s:%p:%p >%s\n", isf->name, isf->curm, handle, typestr);
if (data) {
int i;
char tmp[256];
int max = (size > sizeof(tmp)-1) ? sizeof(tmp)-1 : size;
for (i=0; i 127) break;
tmp[i] = c;
}
tmp[i] = 0;
if (i>0) {
TRACE68(curl_cat,"[%s]\n", tmp);
}
}
#endif
return 0;
}
static int match_header(int *v,
char * ptr, const int bytes,
const char *header, const int hdsize)
{
if (bytes > hdsize+1 &&
!memcmp(ptr,header,hdsize-1)) {
int i, c;
/* Find first digit */
for (i=hdsize-1, c=0; i'9')); ++i)
;
/* Convert base 10 number */
if (c>='0' || c<='9') {
int len = c - '0';
for (++i; i='0' && c<='9')); ++i) {
len = len * 10 + c - '0';
}
*v = len;
/* TRACE68(curl_cat,"Match header %s-> %d\n", header, len); */
return 1;
}
}
return 0;
}
static size_t isf_header_cb(void *ptr, size_t size,
size_t nmemb, void *stream)
{
vfs68_curl_t * isf = (vfs68_curl_t *)stream;
static const char ftp_size[] = "213";
/* static const char ftp_complete[] = "226"; */
static const char content_length[] = "Content-Length:";
/* static const char content_type[] = "Content-Type:"; */
int len;
int bytes = nmemb*size;
isf->has_hd_write = 1;
/* TRACE68(curl_cat,"curl68::header_cb(%s:%p, %d)\n", */
/* isf->name, isf->curm, size*nmemb); */
if (match_header(&len, ptr, bytes, content_length, sizeof(content_length))
|| match_header(&len, ptr, bytes, ftp_size, sizeof(ftp_size))) {
isf->length = len;
/* TRACE68(curl_cat,"%s::Set length to %d\n", isf->name, len); */
}
/* $$$ TODO : Add transfert complete checks */
return nmemb;
}
static int safe_curl_cleanup(vfs68_curl_t * isf)
{
int err = 1;
if (isf) {
err = 0;
if (isf->curm) {
if (isf->curl) {
err |= ( CURLM_OK != curl_multi_remove_handle(isf->curm, isf->curl) );
curl_easy_cleanup(isf->curl);
isf->curl = 0;
}
err |= ( CURLM_OK != curl_multi_cleanup(isf->curm) );
isf->curm = 0;
} else if (isf->curl) {
curl_easy_cleanup(isf->curl);
isf->curl = 0;
}
}
return -err;
}
static int isf_open(vfs68_t * vfs)
{
vfs68_curl_t * isf = (vfs68_curl_t *)vfs;
const int verbose = ISF_VERBOSE;
CURLcode code = 0;
CURLMcode mcode = 0;
int err;
if (!isf->name[0] || isf->curm || isf->curl) {
return -1;
}
/* SC68os_pdebug("curl68::open(%s,%c%c)\n",isf->name, */
/* (isf->mode & VFS68_OPEN_READ)?'R':'r', */
/* (isf->mode & VFS68_OPEN_WRITE)?'W':'w'); */
/* Reset info. */
isf->has_read = isf->has_write = isf->has_hd_read = isf->has_hd_write = 0;
isf->curl_pos = isf->pos = 0;
isf->length = -1;
isf->error[0] = 0;
isf->code = 0;
isf->mcode = 0;
/*
FD_ZERO(&isf->sets.r);
FD_ZERO(&isf->sets.w);
FD_ZERO(&isf->sets.e);
*/
/* Currently accapts only read only open mode. */
if (isf->mode != VFS68_OPEN_READ) {
strcpy(isf->error, "Invalid open mode");
return -1;
}
/* Create curl handle */
isf->curl = curl_easy_init();
isf->curm = curl_multi_init();
/* TRACE68(curl_cat,"%s:%p:%p:open()\n",isf->name, isf->curm, isf->curl); */
if (!isf->curl || !isf->curm) {
goto error;
}
/* URL, most important thing to setup. */
err = 0;
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_URL,isf->name)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_ERRORBUFFER,isf->error)));
/* Set curl internal buffer (just an hint) */
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_BUFFERSIZE,1024)));
/* Debug options. */
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_VERBOSE,verbose)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_DEBUGFUNCTION,isf_debug_cb)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_DEBUGDATA,isf)));
/* Our read/write/progress callback. */
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_WRITEFUNCTION,isf_write_cb)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_WRITEDATA,isf)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_READFUNCTION,isf_read_cb)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_READDATA,isf)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_PROGRESSFUNCTION,isf_prog_cb)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_PROGRESSDATA,isf)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_HEADERFUNCTION,isf_header_cb)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_HEADERDATA,isf)));
/* Password callback (currently raises an error) */
/* $$$ OBSOLETE */
/*
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_PASSWDFUNCTION,isf_passwd_cb)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_PASSWDDATA,isf)));
*/
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_NETRC,CURL_NETRC_OPTIONAL)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_FOLLOWLOCATION,1)));
/* MAC-OSX (7.10.2) version does not have it ?? */
#ifdef CURLOPT_UNRESTRICTED_AUTH
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_UNRESTRICTED_AUTH,0)));
#endif
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_TRANSFERTEXT,0)));
// $$$ FIXME
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_INFILESIZE,0)));
err = err || (CURLE_OK != (code=curl_easy_setopt(isf->curl,CURLOPT_UPLOAD,0)));
if (err) {
goto error;
}
isf_reset_cache_blocks(isf);
mcode = curl_multi_add_handle(isf->curm, isf->curl);
if (mcode != CURLM_OK && mcode != CURLM_CALL_MULTI_PERFORM) {
goto error;
} else {
const int re_timeout = IS68_TIMEOUT;
int timeout = re_timeout;
do {
int n;
mcode = curl_multi_perform(isf->curm, &n);
/* TRACE68(curl_cat,"READ_HEADER: MCODE=[%s]\n", */
/* mcode == CURLM_OK ? "OK" : */
/* (mcode == CURLM_CALL_MULTI_PERFORM) ? "MULTI" : "OTHER"); */
if (mcode == CURLM_OK) {
/* TRACE68(curl_cat,"OK, timeleft:%d\n",timeout); */
timeout -= mysleep(200);
if (timeout < 0) {
TRACE68(curl_cat,"vfs_curl: read-header -- *%s*\n","TIME-OUT");
break;
}
} else if (mcode != CURLM_CALL_MULTI_PERFORM) {
goto error;
} else {
timeout = re_timeout;
}
} while (!(isf->has_read | isf->has_write));
if ( ! (isf->has_hd_write | isf->has_read | isf->has_write) ) {
goto error;
}
}
isf->code = code;
isf->mcode = mcode;
return 0;
error:
isf->code = code;
isf->mcode = mcode;
safe_curl_cleanup(isf);
return -1;
}
static int isf_close(vfs68_t * vfs)
{
vfs68_curl_t * isf = (vfs68_curl_t *)vfs;
return safe_curl_cleanup(isf);
}
static int isf_need_more(vfs68_curl_t * isf)
{
int n=0, pos;
pos = isf->curl_pos;
isf->mcode = curl_multi_perform(isf->curm, &n);
n = isf->curl_pos - pos;
if (n) {
/* TRACE68(curl_cat,"%s:%p:GETS MORE\n" */
/* "-> %d (code=%d)\n", */
/* isf->name, isf->curm, */
/* n, isf->mcode); */
} else {
switch (isf->mcode) {
case CURLM_OK:
break;
case CURLM_CALL_MULTI_PERFORM:
default:
/* TRACE68(curl_cat,"%s:%p:GETS MORE DETECTS EOF\n" */
/* "-> pos:%d length:%d (code=%d)\n", */
/* isf->name, isf->curm, */
/* isf->curl_pos, isf->length, isf->mcode); */
n = -1;
break;
}
}
return n;
}
static int isf_read(vfs68_t * vfs, void * data, int n)
{
vfs68_curl_t * isf = (vfs68_curl_t *)vfs;
int pos, off, top, blk, bytes;
const int re_timeout = IS68_TIMEOUT;
int timeout = re_timeout;
/* SC68os_pdebug("%s:%p::read: %d bytes -> %p\n", */
/* isf->name, isf->curm, n, data); */
if (!isf->curl) {
return -1;
} else if (!n) {
return 0;
}
pos = isf->pos;
bytes = n;
while (bytes > 0) {
char * src = 0;
int n = -1;
if (isf->length != -1 && pos >= isf->length) {
/* TRACE68(curl_cat,"%s:%p, EOF:\n" */
/* "->pos:%p/%p\n", */
/* isf->name, isf->curm, */
/* pos, isf->length); */
break;
} else if (timeout < 0) {
/* TRACE68(curl_cat,"%s:%p, TIME-OUT :\n" */
/* "->pos:%d/%d (%d ms)\n", */
/* isf->name, isf->curm, */
/* pos, isf->length, re_timeout); */
break;
}
off = pos & (IS68_CACHE_SZ-1);
top = pos - off;
blk = (pos>>IS68_CACHE_FIX) & (IS68_CACHE_BLKS-1);
if (top == isf->cache[blk].top) {
/* TRACE68(curl_cat,"%s:%p, read block:\n" */
/* "->pos:%x top:%x blk:%x off:%x bytes:%d cache:[%x:%d]\n", */
/* isf->name, isf->curm, */
/* pos, top, blk, off, bytes, */
/* isf->cache[blk].top, isf->cache[blk].bytes); */
src = isf->cache[blk].buffer + off;
n = isf->cache[blk].bytes - off;
if (n > bytes) {
n = bytes;
}
} else {
/* TRACE68(curl_cat,"NOT MY TOP\n"); */
}
if (n <= 0) {
if (pos < isf->curl_pos) {
TRACE68(curl_cat,"%s:%p: Can not seek backward from %d to %d\n",
isf->name, isf->curm, isf->curl_pos, pos);
break;
}
n=isf_need_more(isf);
if (!n) {
/* $$$ FIXME : should use fd_set here ! */
timeout -= mysleep(100);
} else if (n == -1) {
break;
} else {
timeout = re_timeout;
}
} else {
memcpy(data, src, n);
data = (char *)data + n;
pos += n;
bytes -= n;
}
}
isf->pos = pos;
if (isf->mcode != CURLM_OK && isf->mcode != CURLM_CALL_MULTI_PERFORM) {
return -1;
}
/* TRACE68(curl_cat,"%s:%p:read -> %d\n", isf->name, isf->curm, n-bytes); */
return n - bytes;
}
/* $$$ TODO */
static int isf_write(vfs68_t * vfs, const void * data, int n)
{
vfs68_curl_t * isf = (vfs68_curl_t *)vfs;
return (!isf->curl)
? -1
: 0
;
}
/* $$$ TODO */
static int isf_flush(vfs68_t * vfs)
{
vfs68_curl_t * isf = (vfs68_curl_t *)vfs;
return (!isf->curl)
? -1
: 0
;
}
/* We could have store the length value at opening, but this way it handles
* dynamic changes of curl size.
*/
static int isf_length(vfs68_t * vfs)
{
vfs68_curl_t * isf = (vfs68_curl_t *)vfs;
return (!isf->curl)
? -1
: isf->length
;
}
static int isf_tell(vfs68_t * vfs)
{
vfs68_curl_t * isf = (vfs68_curl_t *)vfs;
return (!isf->curl)
? -1
: isf->pos
;
}
static int isf_seek(vfs68_t * vfs, int offset)
{
vfs68_curl_t * isf = (vfs68_curl_t *)vfs;
int pos;
if (!isf->curl) {
return -1;
}
pos = isf->pos + offset;
if (pos < 0 || (isf->length != -1 && pos > isf->length)) {
return -1;
}
isf->pos = pos;
return 0;
}
static void isf_destroy(vfs68_t * vfs)
{
free(vfs);
}
static const vfs68_t vfs68_curl = {
isf_name,
isf_open, isf_close,
isf_read, isf_write, isf_flush,
isf_length, isf_tell, isf_seek, isf_seek,
isf_destroy
};
int vfs68_curl_init(void)
{
if (curl_cat == msg68_DEFAULT) {
curl_cat = msg68_cat("curl","cURL VFS",DEBUG_CURL_O);
curl_cat = curl_cat != -1 ? curl_cat : msg68_DEFAULT;
}
if (!init) {
init = ( curl_global_init(CURL_GLOBAL_ALL) == CURLE_OK ) ? 1 : -1;
if (init == 1)
uri68_register(&curl_scheme);
}
return -(init != 1);
}
void vfs68_curl_shutdown(void)
{
if (init == 1) {
uri68_unregister(&curl_scheme);
curl_global_cleanup();
init = 0;
}
if (curl_cat != msg68_DEFAULT) {
msg68_cat_free(curl_cat);
curl_cat = msg68_DEFAULT;
}
}
static vfs68_t * curl_create(const char *uri, int mode,
int argc, va_list list)
{
vfs68_curl_t *isf;
int len;
if (vfs68_curl_init()) {
return 0;
}
if (!uri || !uri[0]) {
return 0;
}
/* Don't need +1 because 1 byte already allocated in the
* vfs68_curl_t::uri.
*/
len = strlen(uri);
isf = calloc(sizeof(vfs68_curl_t) + len, 1);
if (!isf) {
return 0;
}
/* Copy vfs functions. */
memcpy(&isf->vfs, &vfs68_curl, sizeof(vfs68_curl));
/* Clean curl handle. */
isf->mode = mode & (VFS68_OPEN_READ|VFS68_OPEN_WRITE);
strcpy(isf->name, uri);
return &isf->vfs;
}
#else /* #ifdef USE_CURL */
/* vfs curl must not be include in this package. Anyway the creation
* still exist but it always returns error.
*/
#include "file68_vfs_curl.h"
#include "file68_vfs_def.h"
int vfs68_curl_init(void) { return 0; }
void vfs68_curl_shutdown(void) {}
#endif