1 struct percpu_counter {
2 	signed long long count;
3 };
4 struct blkg_rwstat {
5 	struct percpu_counter cpu_cnt[4];
6 };
7 struct cfq_group {
8 	struct blkg_rwstat service_time;
9 };
10 struct cfq_queue {
11 	struct cfq_group *group;
12 };
13 struct request {
14 	struct cfq_queue *active_queue;
15 	unsigned long long cmd_flags;
16 	void *priv;
17 };
blkg_rwstat_add(struct blkg_rwstat * rwstat,int rw,unsigned long long val)18 static void blkg_rwstat_add(struct blkg_rwstat *rwstat, int rw, unsigned long long val)
19 {
20 	struct percpu_counter *cnt;
21 	if (rw & 1)
22 		cnt = &rwstat->cpu_cnt[1];
23 	else
24 		cnt = &rwstat->cpu_cnt[0];
25 	cnt->count += val;
26 	if (rw & 2)
27 		cnt = &rwstat->cpu_cnt[2];
28 	else
29 		cnt = &rwstat->cpu_cnt[3];
30 	cnt->count += val;
31 }
32 extern unsigned long long rq_start_time_ns(void);
33 extern unsigned long long rq_io_start_time_ns(void);
34 extern int rq_is_sync(void);
35 extern void cfq_arm_slice_timer(void);
cfq_completed_request(struct request * rq)36 void cfq_completed_request(struct request *rq)
37 {
38 	struct cfq_queue *queue = rq->priv;
39 	int sync = rq_is_sync();
40 	struct cfq_group *group = queue->group;
41 	long long start_time = rq_start_time_ns();
42 	long long io_start_time = rq_io_start_time_ns();
43 	int rw = rq->cmd_flags;
44 
45 	if (io_start_time < 1)
46 		blkg_rwstat_add(&group->service_time, rw, 1 - io_start_time);
47 	blkg_rwstat_add(0, rw, io_start_time - start_time);
48 
49 	if (rq->active_queue == queue && sync)
50 		cfq_arm_slice_timer();
51 }
52 
53