1 /*****************************************************************************
2 *
3 * This example source code introduces a c library buffered I/O interface to
4 * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
5 * rewind(). Supported functions have identical prototypes to their normal c
6 * lib namesakes and are preceaded by url_ .
7 *
8 * Using this code you can replace your program's fopen() with url_fopen()
9 * and fread() with url_fread() and it become possible to read remote streams
10 * instead of (only) local files. Local files (ie those that can be directly
11 * fopened) will drop back to using the underlying clib implementations
12 *
13 * See the main() function at the bottom that shows an app that retrives from a
14 * specified url using fgets() and fread() and saves as two output files.
15 *
16 * Copyright (c) 2003 Simtec Electronics
17 *
18 * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
19 * reference to original curl example code
20 *
21 * Redistribution and use in source and binary forms, with or without
22 * modification, are permitted provided that the following conditions
23 * are met:
24 * 1. Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * 2. Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in the
28 * documentation and/or other materials provided with the distribution.
29 * 3. The name of the author may not be used to endorse or promote products
30 * derived from this software without specific prior written permission.
31 *
32 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
33 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
34 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
35 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
36 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
41 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 *
43 * This example requires libcurl 7.9.7 or later.
44 *
45 * Modified for aide by Hannes von Haugwitz <hannes@vonhaugwitz.com>
46 * based on modifications of previous version by Pablo Virolainen
47 * (pablo@ipi.fi):
48 * 2013-05-14: - moved declarations to 'include/fopen.h'
49 * - removed (unneeded) main method
50 */
51
52 #include "fopen.h"
53
54 /* we use a global one for convenience */
55 CURLM *multi_handle;
56
57 /* curl calls this routine to get more data */
write_callback(char * buffer,size_t size,size_t nitems,void * userp)58 static size_t write_callback(char *buffer,
59 size_t size,
60 size_t nitems,
61 void *userp)
62 {
63 char *newbuff;
64 size_t rembuff;
65
66 URL_FILE *url = (URL_FILE *)userp;
67 size *= nitems;
68
69 rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */
70
71 if(size > rembuff) {
72 /* not enough space in buffer */
73 newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
74 if(newbuff==NULL) {
75 fprintf(stderr,"callback buffer grow failed\n");
76 size=rembuff;
77 }
78 else {
79 /* realloc suceeded increase buffer size*/
80 url->buffer_len+=size - rembuff;
81 url->buffer=newbuff;
82 }
83 }
84
85 memcpy(&url->buffer[url->buffer_pos], buffer, size);
86 url->buffer_pos += size;
87
88 return size;
89 }
90
91 /* use to attempt to fill the read buffer up to requested number of bytes */
fill_buffer(URL_FILE * file,size_t want)92 static int fill_buffer(URL_FILE *file, size_t want)
93 {
94 fd_set fdread;
95 fd_set fdwrite;
96 fd_set fdexcep;
97 struct timeval timeout;
98 int rc;
99
100 /* only attempt to fill buffer if transactions still running and buffer
101 * doesnt exceed required size already
102 */
103 if((!file->still_running) || (file->buffer_pos > want))
104 return 0;
105
106 /* attempt to fill buffer */
107 do {
108 int maxfd = -1;
109 long curl_timeo = -1;
110
111 FD_ZERO(&fdread);
112 FD_ZERO(&fdwrite);
113 FD_ZERO(&fdexcep);
114
115 /* set a suitable timeout to fail on */
116 timeout.tv_sec = 60; /* 1 minute */
117 timeout.tv_usec = 0;
118
119 curl_multi_timeout(multi_handle, &curl_timeo);
120 if(curl_timeo >= 0) {
121 timeout.tv_sec = curl_timeo / 1000;
122 if(timeout.tv_sec > 1)
123 timeout.tv_sec = 1;
124 else
125 timeout.tv_usec = (curl_timeo % 1000) * 1000;
126 }
127
128 /* get file descriptors from the transfers */
129 curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
130
131 /* In a real-world program you OF COURSE check the return code of the
132 function calls. On success, the value of maxfd is guaranteed to be
133 greater or equal than -1. We call select(maxfd + 1, ...), specially
134 in case of (maxfd == -1), we call select(0, ...), which is basically
135 equal to sleep. */
136
137 rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
138
139 switch(rc) {
140 case -1:
141 /* select error */
142 break;
143
144 case 0:
145 default:
146 /* timeout or readable/writable sockets */
147 curl_multi_perform(multi_handle, &file->still_running);
148 break;
149 }
150 } while(file->still_running && (file->buffer_pos < want));
151 return 1;
152 }
153
154 /* use to remove want bytes from the front of a files buffer */
use_buffer(URL_FILE * file,int want)155 static int use_buffer(URL_FILE *file,int want)
156 {
157 /* sort out buffer */
158 if((file->buffer_pos - want) <=0) {
159 /* ditch buffer - write will recreate */
160 if(file->buffer)
161 free(file->buffer);
162
163 file->buffer=NULL;
164 file->buffer_pos=0;
165 file->buffer_len=0;
166 }
167 else {
168 /* move rest down make it available for later */
169 memmove(file->buffer,
170 &file->buffer[want],
171 (file->buffer_pos - want));
172
173 file->buffer_pos -= want;
174 }
175 return 0;
176 }
177
url_fopen(const char * url,const char * operation)178 URL_FILE *url_fopen(const char *url,const char *operation)
179 {
180 /* this code could check for URLs or types in the 'url' and
181 basicly use the real fopen() for standard files */
182
183 URL_FILE *file;
184 (void)operation;
185
186 file = malloc(sizeof(URL_FILE));
187 if(!file)
188 return NULL;
189
190 memset(file, 0, sizeof(URL_FILE));
191
192 if((file->handle.file=fopen(url,operation)))
193 file->type = CFTYPE_FILE; /* marked as URL */
194
195 else {
196 file->type = CFTYPE_CURL; /* marked as URL */
197 file->handle.curl = curl_easy_init();
198
199 curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
200 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
201 curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
202 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
203
204 if(!multi_handle)
205 multi_handle = curl_multi_init();
206
207 curl_multi_add_handle(multi_handle, file->handle.curl);
208
209 /* lets start the fetch */
210 curl_multi_perform(multi_handle, &file->still_running);
211
212 if((file->buffer_pos == 0) && (!file->still_running)) {
213 /* if still_running is 0 now, we should return NULL */
214
215 /* make sure the easy handle is not in the multi handle anymore */
216 curl_multi_remove_handle(multi_handle, file->handle.curl);
217
218 /* cleanup */
219 curl_easy_cleanup(file->handle.curl);
220
221 free(file);
222
223 file = NULL;
224 }
225 }
226 return file;
227 }
228
url_fclose(URL_FILE * file)229 int url_fclose(URL_FILE *file)
230 {
231 int ret=0;/* default is good return */
232
233 switch(file->type) {
234 case CFTYPE_FILE:
235 ret=fclose(file->handle.file); /* passthrough */
236 break;
237
238 case CFTYPE_CURL:
239 /* make sure the easy handle is not in the multi handle anymore */
240 curl_multi_remove_handle(multi_handle, file->handle.curl);
241
242 /* cleanup */
243 curl_easy_cleanup(file->handle.curl);
244 break;
245
246 default: /* unknown or supported type - oh dear */
247 ret=EOF;
248 errno=EBADF;
249 break;
250 }
251
252 if(file->buffer)
253 free(file->buffer);/* free any allocated buffer space */
254
255 free(file);
256
257 return ret;
258 }
259
url_feof(URL_FILE * file)260 int url_feof(URL_FILE *file)
261 {
262 int ret=0;
263
264 switch(file->type) {
265 case CFTYPE_FILE:
266 ret=feof(file->handle.file);
267 break;
268
269 case CFTYPE_CURL:
270 if((file->buffer_pos == 0) && (!file->still_running))
271 ret = 1;
272 break;
273
274 default: /* unknown or supported type - oh dear */
275 ret=-1;
276 errno=EBADF;
277 break;
278 }
279 return ret;
280 }
281
url_fread(void * ptr,size_t size,size_t nmemb,URL_FILE * file)282 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
283 {
284 size_t want;
285
286 switch(file->type) {
287 case CFTYPE_FILE:
288 want=fread(ptr,size,nmemb,file->handle.file);
289 break;
290
291 case CFTYPE_CURL:
292 want = nmemb * size;
293
294 fill_buffer(file,want);
295
296 /* check if theres data in the buffer - if not fill_buffer()
297 * either errored or EOF */
298 if(!file->buffer_pos)
299 return 0;
300
301 /* ensure only available data is considered */
302 if(file->buffer_pos < want)
303 want = file->buffer_pos;
304
305 /* xfer data to caller */
306 memcpy(ptr, file->buffer, want);
307
308 use_buffer(file,want);
309
310 want = want / size; /* number of items */
311 break;
312
313 default: /* unknown or supported type - oh dear */
314 want=0;
315 errno=EBADF;
316 break;
317
318 }
319 return want;
320 }
321
url_fgets(char * ptr,size_t size,URL_FILE * file)322 char *url_fgets(char *ptr, size_t size, URL_FILE *file)
323 {
324 size_t want = size - 1;/* always need to leave room for zero termination */
325 size_t loop;
326
327 switch(file->type) {
328 case CFTYPE_FILE:
329 ptr = fgets(ptr,size,file->handle.file);
330 break;
331
332 case CFTYPE_CURL:
333 fill_buffer(file,want);
334
335 /* check if theres data in the buffer - if not fill either errored or
336 * EOF */
337 if(!file->buffer_pos)
338 return NULL;
339
340 /* ensure only available data is considered */
341 if(file->buffer_pos < want)
342 want = file->buffer_pos;
343
344 /*buffer contains data */
345 /* look for newline or eof */
346 for(loop=0;loop < want;loop++) {
347 if(file->buffer[loop] == '\n') {
348 want=loop+1;/* include newline */
349 break;
350 }
351 }
352
353 /* xfer data to caller */
354 memcpy(ptr, file->buffer, want);
355 ptr[want]=0;/* allways null terminate */
356
357 use_buffer(file,want);
358
359 break;
360
361 default: /* unknown or supported type - oh dear */
362 ptr=NULL;
363 errno=EBADF;
364 break;
365 }
366
367 return ptr;/*success */
368 }
369
url_rewind(URL_FILE * file)370 void url_rewind(URL_FILE *file)
371 {
372 switch(file->type) {
373 case CFTYPE_FILE:
374 rewind(file->handle.file); /* passthrough */
375 break;
376
377 case CFTYPE_CURL:
378 /* halt transaction */
379 curl_multi_remove_handle(multi_handle, file->handle.curl);
380
381 /* restart */
382 curl_multi_add_handle(multi_handle, file->handle.curl);
383
384 /* ditch buffer - write will recreate - resets stream pos*/
385 if(file->buffer)
386 free(file->buffer);
387
388 file->buffer=NULL;
389 file->buffer_pos=0;
390 file->buffer_len=0;
391
392 break;
393
394 default: /* unknown or supported type - oh dear */
395 break;
396 }
397 }
398