1
2 /*
3 * Copyright (C) Yichun Zhang (agentzh)
4 */
5
6
7 #ifndef DDEBUG
8 #define DDEBUG 0
9 #endif
10 #include "ddebug.h"
11
12
13 #include "ngx_http_rds_csv_filter_module.h"
14 #include "ngx_http_rds_csv_util.h"
15 #include "ngx_http_rds_csv_processor.h"
16 #include "ngx_http_rds_csv_output.h"
17
18 #include <ngx_config.h>
19
20
21 #define ngx_http_rds_csv_content_type "text/csv"
22 #define ngx_http_rds_csv_row_term "\r\n"
23
24
25 static volatile ngx_cycle_t *ngx_http_rds_csv_prev_cycle = NULL;
26
27
28 ngx_http_output_header_filter_pt ngx_http_rds_csv_next_header_filter;
29 ngx_http_output_body_filter_pt ngx_http_rds_csv_next_body_filter;
30
31
32 static void *ngx_http_rds_csv_create_loc_conf(ngx_conf_t *cf);
33 static char *ngx_http_rds_csv_merge_loc_conf(ngx_conf_t *cf, void *parent,
34 void *child);
35 static ngx_int_t ngx_http_rds_csv_filter_init(ngx_conf_t *cf);
36 static char *ngx_http_rds_csv_row_terminator(ngx_conf_t *cf,
37 ngx_command_t *cmd, void *conf);
38 static char *ngx_http_rds_csv_field_separator(ngx_conf_t *cf,
39 ngx_command_t *cmd, void *conf);
40 static char *ngx_http_rds_csv(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
41 static void *ngx_http_rds_csv_create_main_conf(ngx_conf_t *cf);
42
43
44 static ngx_command_t ngx_http_rds_csv_commands[] = {
45
46 { ngx_string("rds_csv"),
47 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF
48 |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF
49 |NGX_CONF_FLAG,
50 ngx_http_rds_csv,
51 NGX_HTTP_LOC_CONF_OFFSET,
52 offsetof(ngx_http_rds_csv_loc_conf_t, enabled),
53 NULL },
54
55 { ngx_string("rds_csv_row_terminator"),
56 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF
57 |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF
58 |NGX_CONF_TAKE1,
59 ngx_http_rds_csv_row_terminator,
60 NGX_HTTP_LOC_CONF_OFFSET,
61 offsetof(ngx_http_rds_csv_loc_conf_t, row_term),
62 NULL },
63
64 { ngx_string("rds_csv_field_separator"),
65 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF
66 |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF
67 |NGX_CONF_TAKE1,
68 ngx_http_rds_csv_field_separator,
69 NGX_HTTP_LOC_CONF_OFFSET,
70 0,
71 NULL },
72
73 { ngx_string("rds_csv_field_name_header"),
74 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF
75 |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF
76 |NGX_CONF_FLAG,
77 ngx_conf_set_flag_slot,
78 NGX_HTTP_LOC_CONF_OFFSET,
79 offsetof(ngx_http_rds_csv_loc_conf_t, field_name_header),
80 NULL },
81
82 { ngx_string("rds_csv_content_type"),
83 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF
84 |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF
85 |NGX_CONF_TAKE1,
86 ngx_conf_set_str_slot,
87 NGX_HTTP_LOC_CONF_OFFSET,
88 offsetof(ngx_http_rds_csv_loc_conf_t, content_type),
89 NULL },
90
91 { ngx_string("rds_csv_buffer_size"),
92 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF
93 |NGX_CONF_TAKE1,
94 ngx_conf_set_size_slot,
95 NGX_HTTP_LOC_CONF_OFFSET,
96 offsetof(ngx_http_rds_csv_loc_conf_t, buf_size),
97 NULL },
98
99 ngx_null_command
100 };
101
102
103 static ngx_http_module_t ngx_http_rds_csv_filter_module_ctx = {
104 NULL, /* preconfiguration */
105 ngx_http_rds_csv_filter_init, /* postconfiguration */
106
107 ngx_http_rds_csv_create_main_conf, /* create main configuration */
108 NULL, /* init main configuration */
109
110 NULL, /* create server configuration */
111 NULL, /* merge server configuration */
112
113 ngx_http_rds_csv_create_loc_conf, /* create location configuration */
114 ngx_http_rds_csv_merge_loc_conf /* merge location configuration */
115 };
116
117
118 ngx_module_t ngx_http_rds_csv_filter_module = {
119 NGX_MODULE_V1,
120 &ngx_http_rds_csv_filter_module_ctx, /* module context */
121 ngx_http_rds_csv_commands, /* module directives */
122 NGX_HTTP_MODULE, /* module type */
123 NULL, /* init master */
124 NULL, /* init module */
125 NULL, /* init process */
126 NULL, /* init thread */
127 NULL, /* exit thread */
128 NULL, /* exit process */
129 NULL, /* exit master */
130 NGX_MODULE_V1_PADDING
131 };
132
133
134 static ngx_int_t
ngx_http_rds_csv_header_filter(ngx_http_request_t * r)135 ngx_http_rds_csv_header_filter(ngx_http_request_t *r)
136 {
137 ngx_http_rds_csv_ctx_t *ctx;
138 ngx_http_rds_csv_loc_conf_t *conf;
139 size_t len;
140 u_char *p;
141
142 /* XXX maybe we can generate stub JSON strings like
143 * {"errcode":403,"error":"Permission denied"}
144 * for HTTP error pages? */
145 if ((r->headers_out.status < NGX_HTTP_OK)
146 || (r->headers_out.status >= NGX_HTTP_SPECIAL_RESPONSE)
147 || (r->headers_out.status == NGX_HTTP_NO_CONTENT)
148 || (r->headers_out.status == NGX_HTTP_RESET_CONTENT))
149 {
150 ngx_http_set_ctx(r, NULL, ngx_http_rds_csv_filter_module);
151
152 dd("status is not OK: %d, skipping", (int) r->headers_out.status);
153
154 return ngx_http_rds_csv_next_header_filter(r);
155 }
156
157 /* r->headers_out.status = 0; */
158
159 conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_csv_filter_module);
160
161 if (!conf->enabled) {
162 return ngx_http_rds_csv_next_header_filter(r);
163 }
164
165 if (ngx_http_rds_csv_test_content_type(r) != NGX_OK) {
166 return ngx_http_rds_csv_next_header_filter(r);
167 }
168
169 if (conf->content_type.len == sizeof(ngx_http_rds_csv_content_type) - 1
170 && ngx_strncmp(conf->content_type.data, ngx_http_rds_csv_content_type,
171 sizeof(ngx_http_rds_csv_content_type) - 1) == 0)
172 {
173 /* MIME type is text/csv, we process Content-Type
174 * according to RFC 4180 */
175
176 len = sizeof(ngx_http_rds_csv_content_type) - 1
177 + sizeof("; header=") - 1;
178
179 if (conf->field_name_header) {
180 len += sizeof("presence") - 1;
181
182 } else {
183 len += sizeof("absence") - 1;
184 }
185
186 p = ngx_palloc(r->pool, len);
187 if (p == NULL) {
188 return NGX_ERROR;
189 }
190
191 r->headers_out.content_type.len = len;
192 r->headers_out.content_type_len = len;
193
194 r->headers_out.content_type.data = p;
195
196 p = ngx_copy(p, conf->content_type.data, conf->content_type.len);
197
198 if (conf->field_name_header) {
199 p = ngx_copy_literal(p, "; header=presence");
200
201 } else {
202 p = ngx_copy_literal(p, "; header=absence");
203 }
204
205 if (p - r->headers_out.content_type.data != (ssize_t) len) {
206 ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
207 "rds_csv: content type buffer error: %uz != %uz",
208 (size_t) (p - r->headers_out.content_type.data),
209 len);
210
211 return NGX_ERROR;
212 }
213
214 } else {
215 /* custom MIME-type, we just pass it through */
216
217 r->headers_out.content_type = conf->content_type;
218 r->headers_out.content_type_len = conf->content_type.len;
219 }
220
221 ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_rds_csv_ctx_t));
222
223 if (ctx == NULL) {
224 return NGX_ERROR;
225 }
226
227 ctx->tag = (ngx_buf_tag_t) &ngx_http_rds_csv_filter_module;
228
229 ctx->state = state_expect_header;
230
231 ctx->header_sent = 0;
232
233 ctx->last_out = &ctx->out;
234
235 /* set by ngx_pcalloc
236 * ctx->out = NULL
237 * ctx->busy_bufs = NULL
238 * ctx->free_bufs = NULL
239 * ctx->cached = (ngx_buf_t) 0
240 * ctx->postponed = (ngx_buf_t) 0
241 * ctx->avail_out = 0
242 * ctx->col_names = NULL
243 * ctx->col_count = 0
244 * ctx->cur_col = 0
245 * ctx->field_offset = 0
246 * ctx->field_total = 0
247 * ctx->field_data_rest = 0
248 */
249
250 ngx_http_set_ctx(r, ctx, ngx_http_rds_csv_filter_module);
251
252 ngx_http_clear_content_length(r);
253
254 r->filter_need_in_memory = 1;
255
256 /* we do postpone the header sending to the body filter */
257 return NGX_OK;
258 }
259
260
261 static ngx_int_t
ngx_http_rds_csv_body_filter(ngx_http_request_t * r,ngx_chain_t * in)262 ngx_http_rds_csv_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
263 {
264 ngx_http_rds_csv_ctx_t *ctx;
265 ngx_int_t rc;
266
267 if (in == NULL || r->header_only) {
268 return ngx_http_rds_csv_next_body_filter(r, in);
269 }
270
271 ctx = ngx_http_get_module_ctx(r, ngx_http_rds_csv_filter_module);
272
273 if (ctx == NULL) {
274 return ngx_http_rds_csv_next_body_filter(r, in);
275 }
276
277 switch (ctx->state) {
278
279 case state_expect_header:
280 rc = ngx_http_rds_csv_process_header(r, in, ctx);
281 break;
282
283 case state_expect_col:
284 rc = ngx_http_rds_csv_process_col(r, in, ctx);
285 break;
286
287 case state_expect_row:
288 rc = ngx_http_rds_csv_process_row(r, in, ctx);
289 break;
290
291 case state_expect_field:
292 rc = ngx_http_rds_csv_process_field(r, in, ctx);
293 break;
294
295 case state_expect_more_field_data:
296 rc = ngx_http_rds_csv_process_more_field_data(r, in, ctx);
297 break;
298
299 case state_done:
300
301 /* mark the remaining bufs as consumed */
302
303 dd("discarding bufs");
304
305 ngx_http_rds_csv_discard_bufs(r->pool, in);
306
307 return NGX_OK;
308 break;
309 default:
310 ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
311 "rds_csv: invalid internal state: %d",
312 ctx->state);
313
314 rc = NGX_ERROR;
315
316 break;
317 }
318
319 dd("body filter rc: %d", (int) rc);
320
321 if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) {
322 ctx->state = state_done;
323
324 if (!ctx->header_sent) {
325 ctx->header_sent = 1;
326
327 if (rc == NGX_ERROR) {
328 rc = NGX_HTTP_INTERNAL_SERVER_ERROR;
329 }
330
331 r->headers_out.status = rc;
332
333 dd("sending ERROR headers");
334
335 ngx_http_rds_csv_next_header_filter(r);
336 ngx_http_send_special(r, NGX_HTTP_LAST);
337
338 return NGX_ERROR;
339 }
340
341 return NGX_ERROR;
342 }
343
344 dd("output bufs");
345
346 return ngx_http_rds_csv_output_bufs(r, ctx);
347 }
348
349
350 static ngx_int_t
ngx_http_rds_csv_filter_init(ngx_conf_t * cf)351 ngx_http_rds_csv_filter_init(ngx_conf_t *cf)
352 {
353 int multi_http_blocks;
354 ngx_http_rds_csv_main_conf_t *rmcf;
355
356 rmcf = ngx_http_conf_get_module_main_conf(cf,
357 ngx_http_rds_csv_filter_module);
358
359 if (ngx_http_rds_csv_prev_cycle != ngx_cycle) {
360 ngx_http_rds_csv_prev_cycle = ngx_cycle;
361 multi_http_blocks = 0;
362
363 } else {
364 multi_http_blocks = 1;
365 }
366
367 if (multi_http_blocks || rmcf->requires_filter) {
368 ngx_http_rds_csv_next_header_filter = ngx_http_top_header_filter;
369 ngx_http_top_header_filter = ngx_http_rds_csv_header_filter;
370
371 ngx_http_rds_csv_next_body_filter = ngx_http_top_body_filter;
372 ngx_http_top_body_filter = ngx_http_rds_csv_body_filter;
373 }
374
375 return NGX_OK;
376 }
377
378
379 static void *
ngx_http_rds_csv_create_loc_conf(ngx_conf_t * cf)380 ngx_http_rds_csv_create_loc_conf(ngx_conf_t *cf)
381 {
382 ngx_http_rds_csv_loc_conf_t *conf;
383
384 conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_rds_csv_loc_conf_t));
385 if (conf == NULL) {
386 return NULL;
387 }
388
389 /*
390 * set by ngx_pcalloc():
391 *
392 * conf->content_type = { 0, NULL };
393 * conf->row_term = { 0, NULL };
394 */
395
396 conf->enabled = NGX_CONF_UNSET;
397 conf->field_sep = NGX_CONF_UNSET_UINT;
398 conf->buf_size = NGX_CONF_UNSET_SIZE;
399 conf->field_name_header = NGX_CONF_UNSET;
400
401 return conf;
402 }
403
404
405 static char *
ngx_http_rds_csv_merge_loc_conf(ngx_conf_t * cf,void * parent,void * child)406 ngx_http_rds_csv_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
407 {
408 ngx_http_rds_csv_loc_conf_t *prev = parent;
409 ngx_http_rds_csv_loc_conf_t *conf = child;
410
411 ngx_conf_merge_value(conf->enabled, prev->enabled, 0);
412
413 ngx_conf_merge_value(conf->field_name_header, prev->field_name_header, 1);
414
415 ngx_conf_merge_uint_value(conf->field_sep, prev->field_sep,
416 (ngx_uint_t) ',');
417
418 ngx_conf_merge_str_value(conf->row_term, prev->row_term,
419 ngx_http_rds_csv_row_term);
420
421 ngx_conf_merge_str_value(conf->content_type, prev->content_type,
422 ngx_http_rds_csv_content_type);
423
424 ngx_conf_merge_size_value(conf->buf_size, prev->buf_size,
425 (size_t) ngx_pagesize);
426
427 return NGX_CONF_OK;
428 }
429
430
431 static char *
ngx_http_rds_csv_row_terminator(ngx_conf_t * cf,ngx_command_t * cmd,void * conf)432 ngx_http_rds_csv_row_terminator(ngx_conf_t *cf, ngx_command_t *cmd,
433 void *conf)
434 {
435 ngx_http_rds_csv_loc_conf_t *rlcf = conf;
436 ngx_str_t *value;
437 ngx_str_t *term;
438
439 if (rlcf->row_term.len != 0) {
440 return "is duplicate";
441 }
442
443 value = cf->args->elts;
444
445 term = &value[1];
446
447 if (term->len == 0) {
448 return "takes empty string value";
449 }
450
451 if ((term->len == 1 && term->data[0] == '\n')
452 || (term->len == 2 && term->data[0] == '\r' && term->data[1] == '\n'))
453 {
454 return ngx_conf_set_str_slot(cf, cmd, conf);
455 }
456
457 return "takes a value other than \"\\n\" and \"\\r\\n\"";
458 }
459
460
461 static char *
ngx_http_rds_csv_field_separator(ngx_conf_t * cf,ngx_command_t * cmd,void * conf)462 ngx_http_rds_csv_field_separator(ngx_conf_t *cf, ngx_command_t *cmd,
463 void *conf)
464 {
465 ngx_http_rds_csv_loc_conf_t *rlcf = conf;
466 ngx_str_t *value;
467 ngx_str_t *sep;
468
469 if (rlcf->field_sep != NGX_CONF_UNSET_UINT) {
470 return "is duplicate";
471 }
472
473 value = cf->args->elts;
474
475 sep = &value[1];
476
477 if (sep->len != 1) {
478 return "takes a string value not of length 1";
479 }
480
481 if (sep->data[0] == ',' || sep->data[0] == ';' || sep->data[0] == '\t') {
482 rlcf->field_sep = (ngx_uint_t) (sep->data[0]);
483 return NGX_CONF_OK;
484 }
485
486 return "takes a value other than \",\", \";\", and \"\\t\"";
487 }
488
489
490 static char *
ngx_http_rds_csv(ngx_conf_t * cf,ngx_command_t * cmd,void * conf)491 ngx_http_rds_csv(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
492 {
493 ngx_http_rds_csv_main_conf_t *rmcf;
494
495 rmcf = ngx_http_conf_get_module_main_conf(cf,
496 ngx_http_rds_csv_filter_module);
497
498 rmcf->requires_filter = 1;
499
500 return ngx_conf_set_flag_slot(cf, cmd, conf);
501 }
502
503
504 static void *
ngx_http_rds_csv_create_main_conf(ngx_conf_t * cf)505 ngx_http_rds_csv_create_main_conf(ngx_conf_t *cf)
506 {
507 ngx_http_rds_csv_main_conf_t *rmcf;
508
509 rmcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_rds_csv_main_conf_t));
510 if (rmcf == NULL) {
511 return NULL;
512 }
513
514 /* set by ngx_pcalloc:
515 * rmcf->requires_filter = 0;
516 */
517
518 return rmcf;
519 }
520