42 static inline size_t buffer_size(struct persistent_ram_zone *prz)  in buffer_size()  argument
44 	return atomic_read(&prz->buffer->size);  in buffer_size()
47 static inline size_t buffer_start(struct persistent_ram_zone *prz)  in buffer_start()  argument
49 	return atomic_read(&prz->buffer->start);  in buffer_start()
53 static size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a)  in buffer_start_add()  argument
59 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))  in buffer_start_add()
60 		raw_spin_lock_irqsave(&prz->buffer_lock, flags);  in buffer_start_add()
62 	old = atomic_read(&prz->buffer->start);  in buffer_start_add()
64 	while (unlikely(new >= prz->buffer_size))  in buffer_start_add()
65 		new -= prz->buffer_size;  in buffer_start_add()
66 	atomic_set(&prz->buffer->start, new);  in buffer_start_add()
68 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))  in buffer_start_add()
69 		raw_spin_unlock_irqrestore(&prz->buffer_lock, flags);  in buffer_start_add()
75 static void buffer_size_add(struct persistent_ram_zone *prz, size_t a)  in buffer_size_add()  argument
81 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))  in buffer_size_add()
82 		raw_spin_lock_irqsave(&prz->buffer_lock, flags);  in buffer_size_add()
84 	old = atomic_read(&prz->buffer->size);  in buffer_size_add()
85 	if (old == prz->buffer_size)  in buffer_size_add()
89 	if (new > prz->buffer_size)  in buffer_size_add()
90 		new = prz->buffer_size;  in buffer_size_add()
91 	atomic_set(&prz->buffer->size, new);  in buffer_size_add()
94 	if (!(prz->flags & PRZ_FLAG_NO_LOCK))  in buffer_size_add()
95 		raw_spin_unlock_irqrestore(&prz->buffer_lock, flags);  in buffer_size_add()
98 static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz,  in persistent_ram_encode_rs8()  argument
104 	memset(prz->ecc_info.par, 0,  in persistent_ram_encode_rs8()
105 	       prz->ecc_info.ecc_size * sizeof(prz->ecc_info.par[0]));  in persistent_ram_encode_rs8()
106 	encode_rs8(prz->rs_decoder, data, len, prz->ecc_info.par, 0);  in persistent_ram_encode_rs8()
107 	for (i = 0; i < prz->ecc_info.ecc_size; i++)  in persistent_ram_encode_rs8()
108 		ecc[i] = prz->ecc_info.par[i];  in persistent_ram_encode_rs8()
111 static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz,  in persistent_ram_decode_rs8()  argument
116 	for (i = 0; i < prz->ecc_info.ecc_size; i++)  in persistent_ram_decode_rs8()
117 		prz->ecc_info.par[i] = ecc[i];  in persistent_ram_decode_rs8()
118 	return decode_rs8(prz->rs_decoder, data, prz->ecc_info.par, len,  in persistent_ram_decode_rs8()
122 static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz,  in persistent_ram_update_ecc()  argument
125 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_update_ecc()
126 	uint8_t *buffer_end = buffer->data + prz->buffer_size;  in persistent_ram_update_ecc()
129 	int ecc_block_size = prz->ecc_info.block_size;  in persistent_ram_update_ecc()
130 	int ecc_size = prz->ecc_info.ecc_size;  in persistent_ram_update_ecc()
137 	par = prz->par_buffer + (start / ecc_block_size) * ecc_size;  in persistent_ram_update_ecc()
142 		persistent_ram_encode_rs8(prz, block, size, par);  in persistent_ram_update_ecc()
148 static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz)  in persistent_ram_update_header_ecc()  argument
150 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_update_header_ecc()
152 	if (!prz->ecc_info.ecc_size)  in persistent_ram_update_header_ecc()
155 	persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer),  in persistent_ram_update_header_ecc()
156 				  prz->par_header);  in persistent_ram_update_header_ecc()
159 static void persistent_ram_ecc_old(struct persistent_ram_zone *prz)  in persistent_ram_ecc_old()  argument
161 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_ecc_old()
165 	if (!prz->ecc_info.ecc_size)  in persistent_ram_ecc_old()
169 	par = prz->par_buffer;  in persistent_ram_ecc_old()
170 	while (block < buffer->data + buffer_size(prz)) {  in persistent_ram_ecc_old()
172 		int size = prz->ecc_info.block_size;  in persistent_ram_ecc_old()
173 		if (block + size > buffer->data + prz->buffer_size)  in persistent_ram_ecc_old()
174 			size = buffer->data + prz->buffer_size - block;  in persistent_ram_ecc_old()
175 		numerr = persistent_ram_decode_rs8(prz, block, size, par);  in persistent_ram_ecc_old()
178 			prz->corrected_bytes += numerr;  in persistent_ram_ecc_old()
181 			prz->bad_blocks++;  in persistent_ram_ecc_old()
183 		block += prz->ecc_info.block_size;  in persistent_ram_ecc_old()
184 		par += prz->ecc_info.ecc_size;  in persistent_ram_ecc_old()
188 static int persistent_ram_init_ecc(struct persistent_ram_zone *prz,  in persistent_ram_init_ecc()  argument
192 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_init_ecc()
199 	prz->ecc_info.block_size = ecc_info->block_size ?: 128;  in persistent_ram_init_ecc()
200 	prz->ecc_info.ecc_size = ecc_info->ecc_size ?: 16;  in persistent_ram_init_ecc()
201 	prz->ecc_info.symsize = ecc_info->symsize ?: 8;  in persistent_ram_init_ecc()
202 	prz->ecc_info.poly = ecc_info->poly ?: 0x11d;  in persistent_ram_init_ecc()
204 	ecc_blocks = DIV_ROUND_UP(prz->buffer_size - prz->ecc_info.ecc_size,  in persistent_ram_init_ecc()
205 				  prz->ecc_info.block_size +  in persistent_ram_init_ecc()
206 				  prz->ecc_info.ecc_size);  in persistent_ram_init_ecc()
207 	ecc_total = (ecc_blocks + 1) * prz->ecc_info.ecc_size;  in persistent_ram_init_ecc()
208 	if (ecc_total >= prz->buffer_size) {  in persistent_ram_init_ecc()
210 		       __func__, prz->ecc_info.ecc_size,  in persistent_ram_init_ecc()
211 		       ecc_total, prz->buffer_size);  in persistent_ram_init_ecc()
215 	prz->buffer_size -= ecc_total;  in persistent_ram_init_ecc()
216 	prz->par_buffer = buffer->data + prz->buffer_size;  in persistent_ram_init_ecc()
217 	prz->par_header = prz->par_buffer +  in persistent_ram_init_ecc()
218 			  ecc_blocks * prz->ecc_info.ecc_size;  in persistent_ram_init_ecc()
224 	prz->rs_decoder = init_rs(prz->ecc_info.symsize, prz->ecc_info.poly,  in persistent_ram_init_ecc()
225 				  0, 1, prz->ecc_info.ecc_size);  in persistent_ram_init_ecc()
226 	if (prz->rs_decoder == NULL) {  in persistent_ram_init_ecc()
232 	prz->ecc_info.par = kmalloc_array(prz->ecc_info.ecc_size,  in persistent_ram_init_ecc()
233 					  sizeof(*prz->ecc_info.par),  in persistent_ram_init_ecc()
235 	if (!prz->ecc_info.par) {  in persistent_ram_init_ecc()
240 	prz->corrected_bytes = 0;  in persistent_ram_init_ecc()
241 	prz->bad_blocks = 0;  in persistent_ram_init_ecc()
243 	numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer),  in persistent_ram_init_ecc()
244 					   prz->par_header);  in persistent_ram_init_ecc()
247 		prz->corrected_bytes += numerr;  in persistent_ram_init_ecc()
250 		prz->bad_blocks++;  in persistent_ram_init_ecc()
256 ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz,  in persistent_ram_ecc_string()  argument
261 	if (!prz->ecc_info.ecc_size)  in persistent_ram_ecc_string()
264 	if (prz->corrected_bytes || prz->bad_blocks)  in persistent_ram_ecc_string()
267 			prz->corrected_bytes, prz->bad_blocks);  in persistent_ram_ecc_string()
274 static void notrace persistent_ram_update(struct persistent_ram_zone *prz,  in persistent_ram_update()  argument
277 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_update()
279 	persistent_ram_update_ecc(prz, start, count);  in persistent_ram_update()
282 static int notrace persistent_ram_update_user(struct persistent_ram_zone *prz,  in persistent_ram_update_user()  argument
285 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_update_user()
288 	persistent_ram_update_ecc(prz, start, count);  in persistent_ram_update_user()
292 void persistent_ram_save_old(struct persistent_ram_zone *prz)  in persistent_ram_save_old()  argument
294 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_save_old()
295 	size_t size = buffer_size(prz);  in persistent_ram_save_old()
296 	size_t start = buffer_start(prz);  in persistent_ram_save_old()
301 	if (!prz->old_log) {  in persistent_ram_save_old()
302 		persistent_ram_ecc_old(prz);  in persistent_ram_save_old()
303 		prz->old_log = kmalloc(size, GFP_KERNEL);  in persistent_ram_save_old()
305 	if (!prz->old_log) {  in persistent_ram_save_old()
310 	prz->old_log_size = size;  in persistent_ram_save_old()
311 	memcpy_fromio(prz->old_log, &buffer->data[start], size - start);  in persistent_ram_save_old()
312 	memcpy_fromio(prz->old_log + size - start, &buffer->data[0], start);  in persistent_ram_save_old()
315 int notrace persistent_ram_write(struct persistent_ram_zone *prz,  in persistent_ram_write()  argument
322 	if (unlikely(c > prz->buffer_size)) {  in persistent_ram_write()
323 		s += c - prz->buffer_size;  in persistent_ram_write()
324 		c = prz->buffer_size;  in persistent_ram_write()
327 	buffer_size_add(prz, c);  in persistent_ram_write()
329 	start = buffer_start_add(prz, c);  in persistent_ram_write()
331 	rem = prz->buffer_size - start;  in persistent_ram_write()
333 		persistent_ram_update(prz, s, start, rem);  in persistent_ram_write()
338 	persistent_ram_update(prz, s, start, c);  in persistent_ram_write()
340 	persistent_ram_update_header_ecc(prz);  in persistent_ram_write()
345 int notrace persistent_ram_write_user(struct persistent_ram_zone *prz,  in persistent_ram_write_user()  argument
351 	if (unlikely(c > prz->buffer_size)) {  in persistent_ram_write_user()
352 		s += c - prz->buffer_size;  in persistent_ram_write_user()
353 		c = prz->buffer_size;  in persistent_ram_write_user()
356 	buffer_size_add(prz, c);  in persistent_ram_write_user()
358 	start = buffer_start_add(prz, c);  in persistent_ram_write_user()
360 	rem = prz->buffer_size - start;  in persistent_ram_write_user()
362 		ret = persistent_ram_update_user(prz, s, start, rem);  in persistent_ram_write_user()
368 		ret = persistent_ram_update_user(prz, s, start, c);  in persistent_ram_write_user()
370 	persistent_ram_update_header_ecc(prz);  in persistent_ram_write_user()
375 size_t persistent_ram_old_size(struct persistent_ram_zone *prz)  in persistent_ram_old_size()  argument
377 	return prz->old_log_size;  in persistent_ram_old_size()
380 void *persistent_ram_old(struct persistent_ram_zone *prz)  in persistent_ram_old()  argument
382 	return prz->old_log;  in persistent_ram_old()
385 void persistent_ram_free_old(struct persistent_ram_zone *prz)  in persistent_ram_free_old()  argument
387 	kfree(prz->old_log);  in persistent_ram_free_old()
388 	prz->old_log = NULL;  in persistent_ram_free_old()
389 	prz->old_log_size = 0;  in persistent_ram_free_old()
392 void persistent_ram_zap(struct persistent_ram_zone *prz)  in persistent_ram_zap()  argument
394 	atomic_set(&prz->buffer->start, 0);  in persistent_ram_zap()
395 	atomic_set(&prz->buffer->size, 0);  in persistent_ram_zap()
396 	persistent_ram_update_header_ecc(prz);  in persistent_ram_zap()
479 		struct persistent_ram_zone *prz, int memtype)  in persistent_ram_buffer_map()  argument
481 	prz->paddr = start;  in persistent_ram_buffer_map()
482 	prz->size = size;  in persistent_ram_buffer_map()
485 		prz->vaddr = persistent_ram_vmap(start, size, memtype);  in persistent_ram_buffer_map()
487 		prz->vaddr = persistent_ram_iomap(start, size, memtype,  in persistent_ram_buffer_map()
488 						  prz->label);  in persistent_ram_buffer_map()
490 	if (!prz->vaddr) {  in persistent_ram_buffer_map()
496 	prz->buffer = prz->vaddr;  in persistent_ram_buffer_map()
497 	prz->buffer_size = size - sizeof(struct persistent_ram_buffer);  in persistent_ram_buffer_map()
502 static int persistent_ram_post_init(struct persistent_ram_zone *prz, u32 sig,  in persistent_ram_post_init()  argument
506 	bool zap = !!(prz->flags & PRZ_FLAG_ZAP_OLD);  in persistent_ram_post_init()
508 	ret = persistent_ram_init_ecc(prz, ecc_info);  in persistent_ram_post_init()
510 		pr_warn("ECC failed %s\n", prz->label);  in persistent_ram_post_init()
516 	if (prz->buffer->sig == sig) {  in persistent_ram_post_init()
517 		if (buffer_size(prz) == 0) {  in persistent_ram_post_init()
522 		if (buffer_size(prz) > prz->buffer_size ||  in persistent_ram_post_init()
523 		    buffer_start(prz) > buffer_size(prz)) {  in persistent_ram_post_init()
525 				buffer_size(prz), buffer_start(prz));  in persistent_ram_post_init()
529 				 buffer_size(prz), buffer_start(prz));  in persistent_ram_post_init()
530 			persistent_ram_save_old(prz);  in persistent_ram_post_init()
534 			 prz->buffer->sig);  in persistent_ram_post_init()
535 		prz->buffer->sig = sig;  in persistent_ram_post_init()
541 		persistent_ram_zap(prz);  in persistent_ram_post_init()
546 void persistent_ram_free(struct persistent_ram_zone *prz)  in persistent_ram_free()  argument
548 	if (!prz)  in persistent_ram_free()
551 	if (prz->vaddr) {  in persistent_ram_free()
552 		if (pfn_valid(prz->paddr >> PAGE_SHIFT)) {  in persistent_ram_free()
554 			vunmap(prz->vaddr - offset_in_page(prz->paddr));  in persistent_ram_free()
556 			iounmap(prz->vaddr);  in persistent_ram_free()
557 			release_mem_region(prz->paddr, prz->size);  in persistent_ram_free()
559 		prz->vaddr = NULL;  in persistent_ram_free()
561 	if (prz->rs_decoder) {  in persistent_ram_free()
562 		free_rs(prz->rs_decoder);  in persistent_ram_free()
563 		prz->rs_decoder = NULL;  in persistent_ram_free()
565 	kfree(prz->ecc_info.par);  in persistent_ram_free()
566 	prz->ecc_info.par = NULL;  in persistent_ram_free()
568 	persistent_ram_free_old(prz);  in persistent_ram_free()
569 	kfree(prz->label);  in persistent_ram_free()
570 	kfree(prz);  in persistent_ram_free()
577 	struct persistent_ram_zone *prz;  in persistent_ram_new()  local
580 	prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL);  in persistent_ram_new()
581 	if (!prz) {  in persistent_ram_new()
587 	raw_spin_lock_init(&prz->buffer_lock);  in persistent_ram_new()
588 	prz->flags = flags;  in persistent_ram_new()
589 	prz->label = kstrdup(label, GFP_KERNEL);  in persistent_ram_new()
591 	ret = persistent_ram_buffer_map(start, size, prz, memtype);  in persistent_ram_new()
595 	ret = persistent_ram_post_init(prz, sig, ecc_info);  in persistent_ram_new()
600 		prz->label, prz->size, (unsigned long long)prz->paddr,  in persistent_ram_new()
601 		sizeof(*prz->buffer), prz->buffer_size,  in persistent_ram_new()
602 		prz->size - sizeof(*prz->buffer) - prz->buffer_size,  in persistent_ram_new()
603 		prz->ecc_info.ecc_size, prz->ecc_info.block_size);  in persistent_ram_new()
605 	return prz;  in persistent_ram_new()
607 	persistent_ram_free(prz);  in persistent_ram_new()