1 /*
2  * Copyright (C) 2021 Jakub Kruszona-Zawadzki, Core Technology Sp. z o.o.
3  *
4  * This file is part of MooseFS.
5  *
6  * MooseFS is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation, version 2 (only).
9  *
10  * MooseFS is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with MooseFS; if not, write to the Free Software
17  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301, USA
18  * or visit http://www.gnu.org/licenses/gpl-2.0.html
19  */
20 
21 #ifdef HAVE_CONFIG_H
22 # include "config.h"
23 #endif
24 
25 #ifdef WIN32
26 # include "portable.h"
27 #else
28 # define MMAP_ALLOC 1
29 #endif
30 
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <unistd.h>
35 #ifndef WIN32
36 # ifdef HAVE_POLL_H
37 #  include <poll.h>
38 # else
39 #  include <sys/poll.h>
40 # endif
41 # include <syslog.h>
42 #endif
43 #include <sys/time.h>
44 #include <time.h>
45 #include <limits.h>
46 #include <errno.h>
47 #include <pthread.h>
48 #ifdef MMAP_ALLOC
49 # include <sys/types.h>
50 # include <sys/mman.h>
51 #endif
52 #ifndef WIN32
53 # include <pwd.h>
54 # include <grp.h>
55 #endif
56 
57 #include "MFSCommunication.h"
58 #ifndef WIN32
59 # include "stats.h"
60 #endif
61 #include "sockets.h"
62 #include "strerr.h"
63 #include "md5.h"
64 #include "datapack.h"
65 #include "clocks.h"
66 #include "portable.h"
67 #include "heapsorter.h"
68 #include "extrapackets.h"
69 #include "massert.h"
70 #ifdef MFSMOUNT
71 #include "mfs_fuse.h"
72 #include "mfsmount.h"
73 #endif
74 #include "chunksdatacache.h"
75 //#include "readdata.h"
76 // #include "dircache.h"
77 
78 #define CONNECT_TIMEOUT 2000
79 
80 typedef struct _threc {
81 	pthread_mutex_t mutex;
82 	pthread_cond_t cond;
83 	uint8_t *obuff;
84 	uint32_t obuffsize;
85 	uint32_t odataleng;
86 	uint8_t *ibuff;
87 	uint32_t ibuffsize;
88 	uint32_t idataleng;
89 
90 	uint8_t sent;		// packet was sent
91 	uint8_t status;		// receive status
92 	uint8_t rcvd;		// packet was received
93 //	uint8_t waiting;	// thread is waiting for answer
94 	uint8_t receiving;
95 
96 	uint32_t rcvd_cmd;
97 
98 	uint32_t packetid;
99 	struct _threc *next;
100 } threc;
101 
102 #define THRECHASHSIZE 256
103 
104 static threc *threchash[THRECHASHSIZE];
105 static threc *threcfree = NULL;
106 static uint16_t threcnextid = 0;
107 
108 
109 /*
110 typedef struct _threc {
111 	pthread_t thid;
112 	pthread_mutex_t mutex;
113 	pthread_cond_t cond;
114 	uint8_t *buff;
115 	uint32_t buffsize;
116 	uint8_t sent;
117 	uint8_t status;
118 	uint8_t release;	// cond variable
119 	uint8_t waiting;
120 	uint32_t size;
121 	uint32_t cmd;
122 	uint32_t packetid;
123 	struct _threc *next;
124 } threc;
125 */
126 
127 #define AMTIME_HASH_SIZE 4096
128 #define AMTIME_MAX_AGE 10
129 
130 typedef struct _amtime_file {
131 	uint32_t inode;
132 	uint16_t atimeage;
133 	uint16_t mtimeage;
134 	uint64_t atime;
135 	uint64_t mtime;
136 	struct _amtime_file *next;
137 } amtime_file;
138 
139 static amtime_file *amtime_hash[AMTIME_HASH_SIZE];
140 
141 #define ACQFILES_HASH_SIZE 4096
142 #define ACQFILES_LRU_LIMIT 5000
143 #define ACQFILES_MAX_AGE 10
144 
145 typedef struct _acquired_file {
146 	uint32_t inode;
147 	uint16_t cnt;
148 	uint8_t age;
149 	uint8_t dentry;
150 	struct _acquired_file *next;
151 	struct _acquired_file *lrunext,**lruprev;
152 } acquired_file;
153 
154 static acquired_file *af_hash[ACQFILES_HASH_SIZE];
155 static acquired_file *af_lruhead,**af_lrutail;
156 static uint32_t af_lru_cnt;
157 static int64_t timediffusec = 0;
158 
159 #define DEFAULT_OUTPUT_BUFFSIZE 0x1000
160 #define DEFAULT_INPUT_BUFFSIZE 0x10000
161 
162 #define RECEIVE_TIMEOUT 10
163 
164 
165 static int fd;
166 static int disconnect;
167 static int donotsendsustainedinodes;
168 static double lastwrite;
169 static int sessionlost;
170 static uint64_t lastsyncsend = 0;
171 
172 static uint64_t usectimeout;
173 static uint32_t maxretries;
174 
175 static pthread_t rpthid,npthid;
176 static pthread_mutex_t fdlock,reclock,aflock,amtimelock;
177 static pthread_key_t reckey;
178 
179 static uint32_t sessionid;
180 static uint64_t metaid;
181 static uint32_t masterversion;
182 static uint8_t attrsize;
183 
184 static char masterstrip[17];
185 static uint32_t masterip=0;
186 static uint16_t masterport=0;
187 static char srcstrip[17];
188 static uint32_t srcip=0;
189 
190 static uint8_t fterm;
191 
fs_getmasterlocation(uint8_t loc[14])192 void fs_getmasterlocation(uint8_t loc[14]) {
193 	pthread_mutex_lock(&fdlock);
194 	put32bit(&loc,masterip);
195 	put16bit(&loc,masterport);
196 	put32bit(&loc,sessionid);
197 	put32bit(&loc,masterversion);
198 	pthread_mutex_unlock(&fdlock);
199 }
200 
master_version(void)201 uint32_t master_version(void) {
202 	uint32_t mver;
203 	pthread_mutex_lock(&fdlock);
204 	mver = masterversion;
205 	pthread_mutex_unlock(&fdlock);
206 	return mver;
207 }
208 
master_attrsize(void)209 uint8_t master_attrsize(void) {
210 	uint8_t asize;
211 	pthread_mutex_lock(&fdlock);
212 	asize = attrsize;
213 	pthread_mutex_unlock(&fdlock);
214 	return asize;
215 }
216 
fs_getsrcip()217 uint32_t fs_getsrcip() {
218 	uint32_t sip;
219 	pthread_mutex_lock(&fdlock);
220 	sip = srcip;
221 	pthread_mutex_unlock(&fdlock);
222 	return sip;
223 }
224 
copy_attr(const uint8_t * rptr,uint8_t attr[ATTR_RECORD_SIZE],uint8_t asize)225 static inline void copy_attr(const uint8_t *rptr,uint8_t attr[ATTR_RECORD_SIZE],uint8_t asize) {
226 	if (asize>=ATTR_RECORD_SIZE) {
227 		memcpy(attr,rptr,ATTR_RECORD_SIZE);
228 	} else {
229 		memcpy(attr,rptr,asize);
230 		memset(attr+asize,0,(ATTR_RECORD_SIZE-asize));
231 	}
232 }
233 
234 enum {
235 	MASTER_CONNECTS = 0,
236 	MASTER_BYTESSENT,
237 	MASTER_BYTESRCVD,
238 	MASTER_PACKETSSENT,
239 	MASTER_PACKETSRCVD,
240 	MASTER_PING,
241 	MASTER_TIMEDIFF,
242 	STATNODES
243 };
244 
245 static void *statsptr[STATNODES];
246 
247 struct connect_args_t {
248 	char *bindhostname;
249 	char *masterhostname;
250 	char *masterportname;
251 	uint8_t meta;
252 	uint8_t clearpassword;
253 	char *info;
254 	char *subfolder;
255 	uint8_t *passworddigest;
256 };
257 
258 static struct connect_args_t connect_args;
259 
260 #ifndef WIN32
master_statsptr_init(void)261 void master_statsptr_init(void) {
262 	void *s;
263 	s = stats_get_subnode(NULL,"master",0,0);
264 	statsptr[MASTER_PACKETSRCVD] = stats_get_subnode(s,"packets_received",0,1);
265 	statsptr[MASTER_PACKETSSENT] = stats_get_subnode(s,"packets_sent",0,1);
266 	statsptr[MASTER_BYTESRCVD] = stats_get_subnode(s,"bytes_received",0,1);
267 	statsptr[MASTER_BYTESSENT] = stats_get_subnode(s,"bytes_sent",0,1);
268 	statsptr[MASTER_CONNECTS] = stats_get_subnode(s,"reconnects",0,1);
269 	statsptr[MASTER_PING] = stats_get_subnode(s,"usec_ping",1,1);
270 	statsptr[MASTER_TIMEDIFF] = stats_get_subnode(s,"usec_timediff",1,1);
271 }
272 
master_stats_inc(uint8_t id)273 void master_stats_inc(uint8_t id) {
274 	if (id<STATNODES) {
275 		stats_counter_inc(statsptr[id]);
276 	}
277 }
278 
master_stats_add(uint8_t id,uint64_t s)279 void master_stats_add(uint8_t id,uint64_t s) {
280 	if (id<STATNODES) {
281 		stats_counter_add(statsptr[id],s);
282 	}
283 }
284 
master_stats_set(uint8_t id,uint64_t s)285 void master_stats_set(uint8_t id,uint64_t s) {
286 	if (id<STATNODES) {
287 		stats_counter_set(statsptr[id],s);
288 	}
289 }
290 #else
master_statsptr_init(void)291 void master_statsptr_init(void) {
292 }
293 
master_stats_inc(uint8_t id)294 void master_stats_inc(uint8_t id) {
295 	(void)id;
296 }
297 
master_stats_add(uint8_t id,uint64_t s)298 void master_stats_add(uint8_t id,uint64_t s) {
299 	(void)id;
300 	(void)s;
301 }
302 
master_stats_set(uint8_t id,uint64_t s)303 void master_stats_set(uint8_t id,uint64_t s) {
304 	(void)id;
305 	(void)s;
306 }
307 #endif
308 
309 const char* errtab[]={MFS_ERROR_STRINGS};
310 
mfs_strerror(uint8_t status)311 static inline const char* mfs_strerror(uint8_t status) {
312 	if (status>MFS_ERROR_MAX) {
313 		status=MFS_ERROR_MAX;
314 	}
315 	return errtab[status];
316 }
317 
318 // read
fs_atime(uint32_t inode)319 void fs_atime(uint32_t inode) {
320 	amtime_file *amfptr;
321 	uint32_t amhash;
322 	pthread_mutex_lock(&amtimelock);
323 	amhash = inode % AMTIME_HASH_SIZE;
324 	for (amfptr = amtime_hash[amhash] ; amfptr ; amfptr = amfptr->next) {
325 		if (amfptr->inode == inode) {
326 			amfptr->atime = monotonic_useconds()+timediffusec;
327 			amfptr->atimeage = 0;
328 			pthread_mutex_unlock(&amtimelock);
329 			return;
330 		}
331 	}
332 	amfptr = malloc(sizeof(amtime_file));
333 	amfptr->inode = inode;
334 	amfptr->atimeage = 0;
335 	amfptr->mtimeage = 0;
336 	amfptr->atime = monotonic_useconds()+timediffusec;
337 	amfptr->mtime = 0;
338 	amfptr->next = amtime_hash[amhash];
339 	amtime_hash[amhash] = amfptr;
340 	pthread_mutex_unlock(&amtimelock);
341 }
342 
343 // write
fs_mtime(uint32_t inode)344 void fs_mtime(uint32_t inode) {
345 	amtime_file *amfptr;
346 	uint32_t amhash;
347 	pthread_mutex_lock(&amtimelock);
348 	amhash = inode % AMTIME_HASH_SIZE;
349 	for (amfptr = amtime_hash[amhash] ; amfptr ; amfptr = amfptr->next) {
350 		if (amfptr->inode == inode) {
351 			amfptr->mtime = monotonic_useconds()+timediffusec;
352 			amfptr->mtimeage = 0;
353 			pthread_mutex_unlock(&amtimelock);
354 			return;
355 		}
356 	}
357 	amfptr = malloc(sizeof(amtime_file));
358 	amfptr->inode = inode;
359 	amfptr->atimeage = 0;
360 	amfptr->mtimeage = 0;
361 	amfptr->mtime = monotonic_useconds()+timediffusec;
362 	amfptr->atime = 0;
363 	amfptr->next = amtime_hash[amhash];
364 	amtime_hash[amhash] = amfptr;
365 	pthread_mutex_unlock(&amtimelock);
366 }
367 
368 // set atime (setattr)
fs_no_atime(uint32_t inode)369 void fs_no_atime(uint32_t inode) {
370 	amtime_file *amfptr;
371 	uint32_t amhash;
372 	pthread_mutex_lock(&amtimelock);
373 	amhash = inode % AMTIME_HASH_SIZE;
374 	for (amfptr = amtime_hash[amhash] ; amfptr ; amfptr = amfptr->next) {
375 		if (amfptr->inode == inode) {
376 			amfptr->atimeage = 0;
377 			amfptr->atime = 0;
378 			pthread_mutex_unlock(&amtimelock);
379 			return;
380 		}
381 	}
382 	pthread_mutex_unlock(&amtimelock);
383 }
384 
385 // set mtime (setattr)
fs_no_mtime(uint32_t inode)386 void fs_no_mtime(uint32_t inode) {
387 	amtime_file *amfptr;
388 	uint32_t amhash;
389 	pthread_mutex_lock(&amtimelock);
390 	amhash = inode % AMTIME_HASH_SIZE;
391 	for (amfptr = amtime_hash[amhash] ; amfptr ; amfptr = amfptr->next) {
392 		if (amfptr->inode == inode) {
393 			amfptr->mtimeage = 0;
394 			amfptr->mtime = 0;
395 			pthread_mutex_unlock(&amtimelock);
396 			return;
397 		}
398 	}
399 	pthread_mutex_unlock(&amtimelock);
400 }
401 
fs_fix_amtime(uint32_t inode,time_t * atime,time_t * mtime)402 void fs_fix_amtime(uint32_t inode,time_t *atime,time_t *mtime) {
403 	amtime_file *amfptr;
404 	uint32_t amhash;
405 	time_t ioatime,iomtime;
406 	pthread_mutex_lock(&amtimelock);
407 	amhash = inode % AMTIME_HASH_SIZE;
408 	for (amfptr = amtime_hash[amhash] ; amfptr ; amfptr = amfptr->next) {
409 		if (amfptr->inode == inode) {
410 			ioatime = amfptr->atime / 1000000;
411 			iomtime = amfptr->mtime / 1000000;
412 			if (ioatime > *atime) {
413 				*atime = ioatime;
414 			}
415 			if (iomtime > *mtime) {
416 				*mtime = iomtime;
417 			}
418 			pthread_mutex_unlock(&amtimelock);
419 			return;
420 		}
421 	}
422 	pthread_mutex_unlock(&amtimelock);
423 }
424 
fs_amtime_reference_clock(uint64_t localmonotonic,uint64_t remotewall)425 void fs_amtime_reference_clock(uint64_t localmonotonic,uint64_t remotewall) {
426 	pthread_mutex_lock(&amtimelock);
427 	timediffusec = remotewall - localmonotonic;
428 	pthread_mutex_unlock(&amtimelock);
429 }
430 
fs_af_remove_from_lru(acquired_file * afptr)431 static void fs_af_remove_from_lru(acquired_file *afptr) {
432 	if (afptr->lrunext) {
433 		afptr->lrunext->lruprev = afptr->lruprev;
434 	} else {
435 		af_lrutail = afptr->lruprev;
436 	}
437 	*(afptr->lruprev) = afptr->lrunext;
438 	af_lru_cnt--;
439 	afptr->lrunext = NULL;
440 	afptr->lruprev = NULL;
441 }
442 
fs_af_add_to_lru(acquired_file * afptr)443 static void fs_af_add_to_lru(acquired_file *afptr) {
444 	acquired_file *iafptr,**afpptr;
445 	uint32_t hash;
446 	if (af_lru_cnt>ACQFILES_LRU_LIMIT) {
447 		hash = af_lruhead->inode % ACQFILES_HASH_SIZE;
448 		afpptr = af_hash + hash;
449 		while ((iafptr = *afpptr)) {
450 			if (iafptr==af_lruhead) {
451 				*afpptr = iafptr->next;
452 				chunksdatacache_clear_inode(iafptr->inode,0);
453 				fs_af_remove_from_lru(iafptr);
454 				free(iafptr);
455 			} else {
456 				afpptr = &(iafptr->next);
457 			}
458 		}
459 	}
460 	massert(af_lru_cnt<=ACQFILES_LRU_LIMIT,"open files lru data mismatch !!!");
461 	afptr->lruprev = af_lrutail;
462 	*af_lrutail = afptr;
463 	afptr->lrunext = NULL;
464 	af_lrutail = &(afptr->lrunext);
465 	af_lru_cnt++;
466 }
467 
fs_add_entry(uint32_t inode)468 void fs_add_entry(uint32_t inode) {
469 	uint32_t afhash;
470 	acquired_file *afptr;
471 	pthread_mutex_lock(&aflock);
472 	afhash = inode % ACQFILES_HASH_SIZE;
473 	for (afptr = af_hash[afhash] ; afptr ; afptr = afptr->next) {
474 		if (afptr->inode == inode) {
475 			afptr->dentry = 1;
476 			if (afptr->lruprev!=NULL) {
477 				fs_af_remove_from_lru(afptr);
478 			}
479 			afptr->age = 0;
480 			pthread_mutex_unlock(&aflock);
481 			return;
482 		}
483 	}
484 	afptr = (acquired_file*)malloc(sizeof(acquired_file));
485 	afptr->inode = inode;
486 	afptr->cnt = 0;
487 	afptr->dentry = 1;
488 	afptr->age = 0;
489 	afptr->lrunext = NULL;
490 	afptr->lruprev = NULL;
491 	afptr->next = af_hash[afhash];
492 	af_hash[afhash] = afptr;
493 	pthread_mutex_unlock(&aflock);
494 }
495 
fs_forget_entry(uint32_t inode)496 void fs_forget_entry(uint32_t inode) {
497 	uint32_t afhash;
498 	acquired_file *afptr;
499 	pthread_mutex_lock(&aflock);
500 	afhash = inode % ACQFILES_HASH_SIZE;
501 	for (afptr = af_hash[afhash] ; afptr ; afptr = afptr->next) {
502 		if (afptr->inode == inode) {
503 			afptr->dentry = 0;
504 			if (afptr->cnt==0 && afptr->lruprev==NULL) {
505 				fs_af_add_to_lru(afptr);
506 			}
507 			afptr->age = 0;
508 			pthread_mutex_unlock(&aflock);
509 			return;
510 		}
511 	}
512 	pthread_mutex_unlock(&aflock);
513 }
514 
fs_isopen(uint32_t inode)515 int fs_isopen(uint32_t inode) {
516 	uint32_t afhash;
517 	acquired_file *afptr;
518 	pthread_mutex_lock(&aflock);
519 	afhash = inode % ACQFILES_HASH_SIZE;
520 	for (afptr = af_hash[afhash] ; afptr ; afptr = afptr->next) {
521 		if (afptr->inode == inode) {
522 			if (afptr->dentry || afptr->cnt) {
523 				pthread_mutex_unlock(&aflock);
524 				return 1;
525 			} else {
526 				pthread_mutex_unlock(&aflock);
527 				return 0;
528 			}
529 		}
530 	}
531 	pthread_mutex_unlock(&aflock);
532 	return 0;
533 }
534 
fs_inc_acnt(uint32_t inode)535 void fs_inc_acnt(uint32_t inode) {
536 	uint32_t afhash;
537 	acquired_file *afptr;
538 	pthread_mutex_lock(&aflock);
539 	afhash = inode % ACQFILES_HASH_SIZE;
540 	for (afptr = af_hash[afhash] ; afptr ; afptr = afptr->next) {
541 		if (afptr->inode == inode) {
542 			afptr->cnt++;
543 			if (afptr->lruprev!=NULL) {
544 				fs_af_remove_from_lru(afptr);
545 			}
546 			afptr->age = 0;
547 			pthread_mutex_unlock(&aflock);
548 			return;
549 		}
550 	}
551 	afptr = (acquired_file*)malloc(sizeof(acquired_file));
552 	afptr->inode = inode;
553 	afptr->cnt = 1;
554 	afptr->dentry = 0;
555 	afptr->age = 0;
556 	afptr->lrunext = NULL;
557 	afptr->lruprev = NULL;
558 	afptr->next = af_hash[afhash];
559 	af_hash[afhash] = afptr;
560 	pthread_mutex_unlock(&aflock);
561 }
562 
fs_dec_acnt(uint32_t inode)563 void fs_dec_acnt(uint32_t inode) {
564 	uint32_t afhash;
565 	acquired_file *afptr;
566 	pthread_mutex_lock(&aflock);
567 	afhash = inode % ACQFILES_HASH_SIZE;
568 	for (afptr = af_hash[afhash] ; afptr ; afptr = afptr->next) {
569 		if (afptr->inode == inode) {
570 			if (afptr->cnt>0) {
571 				afptr->cnt--;
572 			}
573 			if (afptr->cnt==0 && afptr->dentry==0 && afptr->lruprev==NULL) {
574 				fs_af_add_to_lru(afptr);
575 			}
576 			afptr->age = 0;
577 			pthread_mutex_unlock(&aflock);
578 			return;
579 		}
580 	}
581 	pthread_mutex_unlock(&aflock);
582 }
583 
fs_free_threc(void * vrec)584 void fs_free_threc(void *vrec) {
585 	threc *drec = (threc*)vrec;
586 	threc *rec,**recp;
587 	uint32_t rechash;
588 
589 	pthread_mutex_lock(&reclock);
590 	rechash = drec->packetid % THRECHASHSIZE;
591 	recp = threchash + rechash;
592 	while ((rec = *recp)) {
593 		if (rec==drec) {
594 			*recp = rec->next;
595 			rec->next = threcfree;
596 			threcfree = rec;
597 			pthread_mutex_lock(&(rec->mutex));
598 			if (rec->obuff!=NULL) {
599 #ifdef MMAP_ALLOC
600 				munmap((void*)(rec->obuff),rec->obuffsize);
601 #else
602 				free(rec->obuff);
603 #endif
604 				rec->obuff = NULL;
605 				rec->obuffsize = 0;
606 			}
607 			if (rec->ibuff!=NULL) {
608 #ifdef MMAP_ALLOC
609 				munmap((void*)(rec->ibuff),rec->ibuffsize);
610 #else
611 				free(rec->ibuff);
612 #endif
613 				rec->ibuff = NULL;
614 				rec->ibuffsize = 0;
615 			}
616 			pthread_mutex_unlock(&(rec->mutex));
617 			pthread_mutex_unlock(&reclock);
618 			return;
619 		} else {
620 			recp = &(rec->next);
621 		}
622 	}
623 	pthread_mutex_unlock(&reclock);
624 	syslog(LOG_WARNING,"threc not found in data structures !!!");
625 }
626 
fs_get_my_threc()627 threc* fs_get_my_threc() {
628 	threc *rec;
629 	uint32_t rechash;
630 
631 	rec = pthread_getspecific(reckey);
632 	if (rec!=NULL) {
633 		return rec;
634 	}
635 	pthread_mutex_lock(&reclock);
636 	if (threcfree!=NULL) {
637 		rec = threcfree;
638 		threcfree = rec->next;
639 	} else {
640 		rec = malloc(sizeof(threc));
641 		rec->packetid = ++threcnextid;
642 		pthread_mutex_init(&(rec->mutex),NULL);
643 		pthread_cond_init(&(rec->cond),NULL);
644 	}
645 	rechash = rec->packetid % THRECHASHSIZE;
646 	rec->next = threchash[rechash];
647 	threchash[rechash] = rec;
648 	rec->obuff = NULL;
649 	rec->ibuff = NULL;
650 	rec->obuffsize = 0;
651 	rec->ibuffsize = 0;
652 	rec->odataleng = 0;
653 	rec->idataleng = 0;
654 	rec->sent = 0;
655 	rec->status = 0;
656 	rec->rcvd = 0;
657 	rec->receiving = 0;
658 	rec->rcvd_cmd = 0;
659 	pthread_mutex_unlock(&reclock);
660 	pthread_setspecific(reckey,rec);
661 	return rec;
662 }
663 
fs_get_threc_by_id(uint32_t packetid)664 threc* fs_get_threc_by_id(uint32_t packetid) {
665 	threc *rec;
666 	uint32_t rechash;
667 	rechash = packetid % THRECHASHSIZE;
668 	pthread_mutex_lock(&reclock);
669 	for (rec = threchash[rechash] ; rec!=NULL ; rec=rec->next) {
670 		if (rec->packetid==packetid) {
671 			pthread_mutex_unlock(&reclock);
672 			return rec;
673 		}
674 	}
675 	pthread_mutex_unlock(&reclock);
676 	syslog(LOG_WARNING,"packet: %"PRIu32" - record not found !!!",packetid);
677 	return NULL;
678 }
679 
fs_output_buffer_init(threc * rec,uint32_t size)680 void fs_output_buffer_init(threc *rec,uint32_t size) {
681 	if (size>DEFAULT_OUTPUT_BUFFSIZE) {
682 #ifdef MMAP_ALLOC
683 		if (rec->obuff) {
684 			munmap((void*)(rec->obuff),rec->obuffsize);
685 		}
686 		rec->obuff = (void*)mmap(NULL,size,PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE,-1,0);
687 #else
688 		if (rec->obuff) {
689 			free(rec->obuff);
690 		}
691 		rec->obuff = malloc(size);
692 #endif
693 		passert(rec->obuff);
694 		rec->obuffsize = size;
695 	} else if (rec->obuffsize!=DEFAULT_OUTPUT_BUFFSIZE) {
696 #ifdef MMAP_ALLOC
697 		if (rec->obuff) {
698 			munmap((void*)(rec->obuff),rec->obuffsize);
699 		}
700 		rec->obuff = (void*)mmap(NULL,DEFAULT_OUTPUT_BUFFSIZE,PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE,-1,0);
701 #else
702 		if (rec->obuff) {
703 			free(rec->obuff);
704 		}
705 		rec->obuff = malloc(DEFAULT_OUTPUT_BUFFSIZE);
706 #endif
707 		passert(rec->obuff);
708 		rec->obuffsize = DEFAULT_OUTPUT_BUFFSIZE;
709 	}
710 //	if (rec->obuff==NULL) {
711 //		rec->obuffsize = 0;
712 //	}
713 }
714 
fs_input_buffer_init(threc * rec,uint32_t size)715 void fs_input_buffer_init(threc *rec,uint32_t size) {
716 	if (size>DEFAULT_INPUT_BUFFSIZE) {
717 #ifdef MMAP_ALLOC
718 		if (rec->ibuff) {
719 			munmap((void*)(rec->ibuff),rec->ibuffsize);
720 		}
721 		rec->ibuff = (void*)mmap(NULL,size,PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE,-1,0);
722 #else
723 		if (rec->ibuff) {
724 			free(rec->ibuff);
725 		}
726 		rec->ibuff = malloc(size);
727 #endif
728 		passert(rec->ibuff);
729 		rec->ibuffsize = size;
730 	} else if (rec->ibuffsize!=DEFAULT_INPUT_BUFFSIZE) {
731 #ifdef MMAP_ALLOC
732 		if (rec->ibuff) {
733 			munmap((void*)(rec->ibuff),rec->ibuffsize);
734 		}
735 		rec->ibuff = (void*)mmap(NULL,DEFAULT_INPUT_BUFFSIZE,PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE,-1,0);
736 #else
737 		if (rec->ibuff) {
738 			free(rec->ibuff);
739 		}
740 		rec->ibuff = malloc(DEFAULT_INPUT_BUFFSIZE);
741 #endif
742 		passert(rec->ibuff);
743 		rec->ibuffsize = DEFAULT_INPUT_BUFFSIZE;
744 	}
745 //	if (rec->ibuff==NULL) {
746 //		rec->ibuffsize = 0;
747 //	}
748 }
749 
fs_createpacket(threc * rec,uint32_t cmd,uint32_t size)750 uint8_t* fs_createpacket(threc *rec,uint32_t cmd,uint32_t size) {
751 	uint8_t *ptr;
752 	uint32_t hdrsize = size+4;
753 	pthread_mutex_lock(&(rec->mutex));	// make helgrind happy
754 	fs_output_buffer_init(rec,size+12);
755 	if (rec->obuff==NULL) {
756 		return NULL;
757 	}
758 	ptr = rec->obuff;
759 	put32bit(&ptr,cmd);
760 	put32bit(&ptr,hdrsize);
761 	put32bit(&ptr,rec->packetid);
762 	rec->odataleng = size+12;
763 	pthread_mutex_unlock(&(rec->mutex));	// make helgrind happy
764 	return ptr;
765 }
766 
fs_disconnect(void)767 static inline void fs_disconnect(void) {
768 #ifdef HAVE___SYNC_FETCH_AND_OP
769 	(void)__sync_fetch_and_or(&disconnect,1);
770 #else
771 	pthread_mutex_lock(&fdlock);
772 	disconnect = 1;
773 	pthread_mutex_unlock(&fdlock);
774 #endif
775 }
776 
777 /*
778 uint8_t fs_send_only(threc *rec) {
779 	uint32_t cnt;
780 	uint64_t start,period,usecto;
781 //	uint32_t size = rec->size;
782 
783 	start = 0; // make static code analysers happy
784 	if (usectimeout>0) {
785 		start = monotonic_useconds();
786 	}
787 	for (cnt=1 ; cnt<=maxretries ; cnt++) {
788 		pthread_mutex_lock(&fdlock);
789 		if (sessionlost==1) {
790 			pthread_mutex_unlock(&fdlock);
791 			return 0;
792 		}
793 		if (fd==-1) {
794 			pthread_mutex_unlock(&fdlock);
795 			usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
796 			if (usectimeout>0) {
797 				period = monotonic_useconds() - start;
798 				if (period >= usectimeout) {
799 					return 0;
800 				}
801 				if (usecto > usectimeout - period) {
802 					usecto = usectimeout - period;
803 				}
804 			}
805 			portable_usleep(usecto);
806 			continue;
807 		}
808 		//syslog(LOG_NOTICE,"threc(%"PRIu32") - sending ...",rec->packetid);
809 		pthread_mutex_lock(&(rec->mutex));	// make helgrind happy
810 		if (tcptowrite(fd,rec->obuff,rec->odataleng,1000)!=(int32_t)(rec->odataleng)) {
811 			syslog(LOG_WARNING,"tcp send error: %s",strerr(errno));
812 #ifdef HAVE___SYNC_FETCH_AND_OP
813 			__sync_fetch_and_or(&disconnect,1);
814 #else
815 			disconnect = 1;
816 #endif
817 			pthread_mutex_unlock(&(rec->mutex));
818 			pthread_mutex_unlock(&fdlock);
819 			usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
820 			if (usectimeout>0) {
821 				period = monotonic_useconds() - start;
822 				if (period >= usectimeout) {
823 					return 0;
824 				}
825 				if (usecto > usectimeout - period) {
826 					usecto = usectimeout - period;
827 				}
828 			}
829 			portable_usleep(usecto);
830 			continue;
831 		}
832 		rec->rcvd = 0;
833 		rec->sent = 1;
834 		pthread_mutex_unlock(&(rec->mutex));	// make helgrind happy
835 		master_stats_add(MASTER_BYTESSENT,rec->odataleng);
836 		master_stats_inc(MASTER_PACKETSSENT);
837 		lastwrite = monotonic_seconds();
838 		pthread_mutex_unlock(&fdlock);
839 		return 1;
840 	}
841 	return 0;
842 }
843 
844 const uint8_t* fs_receive_only(threc *rec,uint32_t expected_cmd,uint32_t *answer_leng) {
845 	uint32_t cnt;
846 	static uint8_t notsup = MFS_ERROR_ENOTSUP;
847 	uint64_t start,period,usecto;
848 	uint8_t first;
849 //	uint32_t size = rec->size;
850 
851 	start = 0; // make static code analysers happy
852 	first = 1;
853 	if (usectimeout>0) {
854 		start = monotonic_useconds();
855 	}
856 	for (cnt=1 ; cnt<=maxretries ; cnt++) {
857 		if (first==0) { // resend data after error
858 			pthread_mutex_lock(&fdlock);
859 			if (sessionlost==1) {
860 				pthread_mutex_unlock(&fdlock);
861 				return NULL;
862 			}
863 			if (fd==-1) {
864 				pthread_mutex_unlock(&fdlock);
865 				usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
866 				if (usectimeout>0) {
867 					period = monotonic_useconds() - start;
868 					if (period >= usectimeout) {
869 						return NULL;
870 					}
871 					if (usecto > usectimeout - period) {
872 						usecto = usectimeout - period;
873 					}
874 				}
875 				portable_usleep(usecto);
876 				continue;
877 			}
878 			//syslog(LOG_NOTICE,"threc(%"PRIu32") - sending ...",rec->packetid);
879 			pthread_mutex_lock(&(rec->mutex));	// make helgrind happy
880 			if (tcptowrite(fd,rec->obuff,rec->odataleng,1000)!=(int32_t)(rec->odataleng)) {
881 				syslog(LOG_WARNING,"tcp send error: %s",strerr(errno));
882 #ifdef HAVE___SYNC_FETCH_AND_OP
883 				__sync_fetch_and_or(&disconnect,1);
884 #else
885 				disconnect = 1;
886 #endif
887 				pthread_mutex_unlock(&(rec->mutex));
888 				pthread_mutex_unlock(&fdlock);
889 				usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
890 				if (usectimeout>0) {
891 					period = monotonic_useconds() - start;
892 					if (period >= usectimeout) {
893 						return NULL;
894 					}
895 					if (usecto > usectimeout - period) {
896 						usecto = usectimeout - period;
897 					}
898 				}
899 				portable_usleep(usecto);
900 				continue;
901 			}
902 			rec->rcvd = 0;
903 			rec->sent = 1;
904 			pthread_mutex_unlock(&(rec->mutex));	// make helgrind happy
905 			master_stats_add(MASTER_BYTESSENT,rec->odataleng);
906 			master_stats_inc(MASTER_PACKETSSENT);
907 			lastwrite = monotonic_seconds();
908 			pthread_mutex_unlock(&fdlock);
909 		} else {
910 			first = 0;
911 		}
912 		// syslog(LOG_NOTICE,"master: lock: %"PRIu32,rec->packetid);
913 		pthread_mutex_lock(&(rec->mutex));
914 		while (rec->rcvd==0) {
915 //			rec->waiting = 1;
916 			if (usectimeout>0) {
917 				struct timespec ts;
918 				struct timeval tv;
919 				period = monotonic_useconds() - start;
920 				if (period >= usectimeout) {
921 					pthread_mutex_unlock(&(rec->mutex));
922 					return NULL;
923 				}
924 				period = usectimeout - period;
925 				gettimeofday(&tv, NULL);
926 				usecto = tv.tv_sec;
927 				usecto *= 1000000;
928 				usecto += tv.tv_usec;
929 				usecto += period;
930 				ts.tv_sec = usecto / 1000000;
931 				ts.tv_nsec = (usecto % 1000000) * 1000;
932 				if (pthread_cond_timedwait(&(rec->cond),&(rec->mutex),&ts)==ETIMEDOUT) {
933 //					rec->waiting = 0;
934 					pthread_mutex_unlock(&(rec->mutex));
935 					return NULL;
936 				}
937 			} else {
938 				pthread_cond_wait(&(rec->cond),&(rec->mutex));
939 			}
940 //			rec->waiting = 0;
941 		}
942 		*answer_leng = rec->idataleng;
943 		// syslog(LOG_NOTICE,"master: unlocked: %"PRIu32,rec->packetid);
944 		// syslog(LOG_NOTICE,"master: command_info: %"PRIu32" ; reccmd: %"PRIu32,command_info,rec->cmd);
945 		if (rec->status!=0) {
946 			pthread_mutex_unlock(&(rec->mutex));
947 			usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
948 			if (usectimeout>0) {
949 				period = monotonic_useconds() - start;
950 				if (period >= usectimeout) {
951 					return NULL;
952 				}
953 				if (usecto > usectimeout - period) {
954 					usecto = usectimeout - period;
955 				}
956 			}
957 			portable_usleep(usecto);
958 			continue;
959 		}
960 		if (rec->rcvd_cmd==ANTOAN_UNKNOWN_COMMAND || rec->rcvd_cmd==ANTOAN_BAD_COMMAND_SIZE) {
961 			pthread_mutex_unlock(&(rec->mutex));
962 			*answer_leng = 1; // simulate error
963 			return &notsup; // return MFS_ERROR_ENOTSUP in this case
964 		}
965 		if (rec->rcvd_cmd!=expected_cmd) {
966 			pthread_mutex_unlock(&(rec->mutex));
967 			fs_disconnect();
968 			usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
969 			if (usectimeout>0) {
970 				period = monotonic_useconds() - start;
971 				if (period >= usectimeout) {
972 					return NULL;
973 				}
974 				if (usecto > usectimeout - period) {
975 					usecto = usectimeout - period;
976 				}
977 			}
978 			portable_usleep(usecto);
979 			continue;
980 		}
981 		pthread_mutex_unlock(&(rec->mutex));
982 		//syslog(LOG_NOTICE,"threc(%"PRIu32") - received",rec->packetid);
983 		return rec->ibuff;
984 	}
985 	return NULL;
986 }
987 */
fs_sendandreceive(threc * rec,uint32_t expected_cmd,uint32_t * answer_leng)988 const uint8_t* fs_sendandreceive(threc *rec,uint32_t expected_cmd,uint32_t *answer_leng) {
989 	uint32_t cnt;
990 	static uint8_t notsup = MFS_ERROR_ENOTSUP;
991 	uint64_t start,period,usecto;
992 //	uint32_t size = rec->size;
993 
994 	start = 0; // make static code analysers happy
995 	if (usectimeout>0) {
996 		start = monotonic_useconds();
997 	}
998 	for (cnt=1 ; cnt<=maxretries ; cnt++) {
999 		pthread_mutex_lock(&fdlock);
1000 		if (sessionlost==1) {
1001 			pthread_mutex_unlock(&fdlock);
1002 			return NULL;
1003 		}
1004 		if (fd==-1) {
1005 			pthread_mutex_unlock(&fdlock);
1006 			usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
1007 			if (usectimeout>0) {
1008 				period = monotonic_useconds() - start;
1009 				if (period >= usectimeout) {
1010 					return NULL;
1011 				}
1012 				if (usecto > usectimeout - period) {
1013 					usecto = usectimeout - period;
1014 				}
1015 			}
1016 			portable_usleep(usecto);
1017 			continue;
1018 		}
1019 		//syslog(LOG_NOTICE,"threc(%"PRIu32") - sending ...",rec->packetid);
1020 		pthread_mutex_lock(&(rec->mutex));	// make helgrind happy
1021 		if (tcptowrite(fd,rec->obuff,rec->odataleng,1000)!=(int32_t)(rec->odataleng)) {
1022 			syslog(LOG_WARNING,"tcp send error: %s",strerr(errno));
1023 #ifdef HAVE___SYNC_FETCH_AND_OP
1024 			(void)__sync_fetch_and_or(&disconnect,1);
1025 #else
1026 			disconnect = 1;
1027 #endif
1028 			pthread_mutex_unlock(&(rec->mutex));
1029 			pthread_mutex_unlock(&fdlock);
1030 			usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
1031 			if (usectimeout>0) {
1032 				period = monotonic_useconds() - start;
1033 				if (period >= usectimeout) {
1034 					return NULL;
1035 				}
1036 				if (usecto > usectimeout - period) {
1037 					usecto = usectimeout - period;
1038 				}
1039 			}
1040 			portable_usleep(usecto);
1041 			continue;
1042 		}
1043 		rec->rcvd = 0;
1044 		rec->sent = 1;
1045 		pthread_mutex_unlock(&(rec->mutex));	// make helgrind happy
1046 		master_stats_add(MASTER_BYTESSENT,rec->odataleng);
1047 		master_stats_inc(MASTER_PACKETSSENT);
1048 		lastwrite = monotonic_seconds();
1049 		pthread_mutex_unlock(&fdlock);
1050 		// syslog(LOG_NOTICE,"master: lock: %"PRIu32,rec->packetid);
1051 		pthread_mutex_lock(&(rec->mutex));
1052 		while (rec->rcvd==0) {
1053 //			rec->waiting = 1;
1054 			if (usectimeout>0) {
1055 				struct timespec ts;
1056 				struct timeval tv;
1057 				period = monotonic_useconds() - start;
1058 				if (period >= usectimeout) {
1059 					pthread_mutex_unlock(&(rec->mutex));
1060 					return NULL;
1061 				}
1062 				period = usectimeout - period;
1063 				gettimeofday(&tv, NULL);
1064 				usecto = tv.tv_sec;
1065 				usecto *= 1000000;
1066 				usecto += tv.tv_usec;
1067 				usecto += period;
1068 				ts.tv_sec = usecto / 1000000;
1069 				ts.tv_nsec = (usecto % 1000000) * 1000;
1070 				if (pthread_cond_timedwait(&(rec->cond),&(rec->mutex),&ts)==ETIMEDOUT) {
1071 //					rec->waiting = 0;
1072 					pthread_mutex_unlock(&(rec->mutex));
1073 					return NULL;
1074 				}
1075 			} else {
1076 				pthread_cond_wait(&(rec->cond),&(rec->mutex));
1077 			}
1078 //			rec->waiting = 0;
1079 		}
1080 		*answer_leng = rec->idataleng;
1081 		// syslog(LOG_NOTICE,"master: unlocked: %"PRIu32,rec->packetid);
1082 		// syslog(LOG_NOTICE,"master: command_info: %"PRIu32" ; reccmd: %"PRIu32,command_info,rec->cmd);
1083 		if (rec->status!=0) {
1084 			pthread_mutex_unlock(&(rec->mutex));
1085 			usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
1086 			if (usectimeout>0) {
1087 				period = monotonic_useconds() - start;
1088 				if (period >= usectimeout) {
1089 					return NULL;
1090 				}
1091 				if (usecto > usectimeout - period) {
1092 					usecto = usectimeout - period;
1093 				}
1094 			}
1095 			portable_usleep(usecto);
1096 			continue;
1097 		}
1098 		if (rec->rcvd_cmd==ANTOAN_UNKNOWN_COMMAND || rec->rcvd_cmd==ANTOAN_BAD_COMMAND_SIZE) {
1099 			pthread_mutex_unlock(&(rec->mutex));
1100 			*answer_leng = 1; // simulate error
1101 			return &notsup; // return MFS_ERROR_ENOTSUP in this case
1102 		}
1103 		if (rec->rcvd_cmd!=expected_cmd) {
1104 			pthread_mutex_unlock(&(rec->mutex));
1105 			fs_disconnect();
1106 			usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
1107 			if (usectimeout>0) {
1108 				period = monotonic_useconds() - start;
1109 				if (period >= usectimeout) {
1110 					return NULL;
1111 				}
1112 				if (usecto > usectimeout - period) {
1113 					usecto = usectimeout - period;
1114 				}
1115 			}
1116 			portable_usleep(usecto);
1117 			continue;
1118 		}
1119 		pthread_mutex_unlock(&(rec->mutex));
1120 		//syslog(LOG_NOTICE,"threc(%"PRIu32") - received",rec->packetid);
1121 		return rec->ibuff;
1122 	}
1123 	return NULL;
1124 }
1125 
fs_sendandreceive_any(threc * rec,uint32_t * received_cmd,uint32_t * answer_leng)1126 const uint8_t* fs_sendandreceive_any(threc *rec,uint32_t *received_cmd,uint32_t *answer_leng) {
1127 	uint32_t cnt;
1128 	uint64_t start,period,usecto;
1129 //	uint32_t size = rec->size;
1130 
1131 	start = 0; // make static code analysers happy
1132 	if (usectimeout>0) {
1133 		start = monotonic_useconds();
1134 	}
1135 	for (cnt=1 ; cnt<=maxretries ; cnt++) {
1136 		pthread_mutex_lock(&fdlock);
1137 		if (sessionlost==1) {
1138 			pthread_mutex_unlock(&fdlock);
1139 			return NULL;
1140 		}
1141 		if (fd==-1) {
1142 			pthread_mutex_unlock(&fdlock);
1143 			usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
1144 			if (usectimeout>0) {
1145 				period = monotonic_useconds() - start;
1146 				if (period >= usectimeout) {
1147 					return NULL;
1148 				}
1149 				if (usecto > usectimeout - period) {
1150 					usecto = usectimeout - period;
1151 				}
1152 			}
1153 			portable_usleep(usecto);
1154 			continue;
1155 		}
1156 		//syslog(LOG_NOTICE,"threc(%"PRIu32") - sending ...",rec->packetid);
1157 		pthread_mutex_lock(&(rec->mutex));	// make helgrind happy
1158 		if (tcptowrite(fd,rec->obuff,rec->odataleng,1000)!=(int32_t)(rec->odataleng)) {
1159 			syslog(LOG_WARNING,"tcp send error: %s",strerr(errno));
1160 #ifdef HAVE___SYNC_FETCH_AND_OP
1161 			(void)__sync_fetch_and_or(&disconnect,1);
1162 #else
1163 			disconnect = 1;
1164 #endif
1165 			pthread_mutex_unlock(&(rec->mutex));
1166 			pthread_mutex_unlock(&fdlock);
1167 			usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
1168 			if (usectimeout>0) {
1169 				period = monotonic_useconds() - start;
1170 				if (period >= usectimeout) {
1171 					return NULL;
1172 				}
1173 				if (usecto > usectimeout - period) {
1174 					usecto = usectimeout - period;
1175 				}
1176 			}
1177 			portable_usleep(usecto);
1178 			continue;
1179 		}
1180 		rec->rcvd = 0;
1181 		rec->sent = 1;
1182 		pthread_mutex_unlock(&(rec->mutex));	// make helgrind happy
1183 		master_stats_add(MASTER_BYTESSENT,rec->odataleng);
1184 		master_stats_inc(MASTER_PACKETSSENT);
1185 		lastwrite = monotonic_seconds();
1186 		pthread_mutex_unlock(&fdlock);
1187 		// syslog(LOG_NOTICE,"master: lock: %"PRIu32,rec->packetid);
1188 		pthread_mutex_lock(&(rec->mutex));
1189 		while (rec->rcvd==0) {
1190 //			rec->waiting = 1;
1191 			if (usectimeout>0) {
1192 				struct timespec ts;
1193 				struct timeval tv;
1194 				period = monotonic_useconds() - start;
1195 				if (period >= usectimeout) {
1196 					pthread_mutex_unlock(&(rec->mutex));
1197 					return NULL;
1198 				}
1199 				period = usectimeout - period;
1200 				gettimeofday(&tv, NULL);
1201 				usecto = tv.tv_sec;
1202 				usecto *= 1000000;
1203 				usecto += tv.tv_usec;
1204 				usecto += period;
1205 				ts.tv_sec = usecto / 1000000;
1206 				ts.tv_nsec = (usecto % 1000000) * 1000;
1207 				if (pthread_cond_timedwait(&(rec->cond),&(rec->mutex),&ts)==ETIMEDOUT) {
1208 //					rec->waiting = 0;
1209 					pthread_mutex_unlock(&(rec->mutex));
1210 					return NULL;
1211 				}
1212 			} else {
1213 				pthread_cond_wait(&(rec->cond),&(rec->mutex));
1214 			}
1215 //			rec->waiting = 0;
1216 		}
1217 		*answer_leng = rec->idataleng;
1218 		// syslog(LOG_NOTICE,"master: unlocked: %"PRIu32,rec->packetid);
1219 		// syslog(LOG_NOTICE,"master: command_info: %"PRIu32" ; reccmd: %"PRIu32,command_info,rec->cmd);
1220 		if (rec->status!=0) {
1221 			pthread_mutex_unlock(&(rec->mutex));
1222 			usecto = 1000+((cnt<30)?((cnt-1)*300000):10000000);
1223 			if (usectimeout>0) {
1224 				period = monotonic_useconds() - start;
1225 				if (period >= usectimeout) {
1226 					return NULL;
1227 				}
1228 				if (usecto > usectimeout - period) {
1229 					usecto = usectimeout - period;
1230 				}
1231 			}
1232 			portable_usleep(usecto);
1233 			continue;
1234 		}
1235 		*received_cmd = rec->rcvd_cmd;
1236 		pthread_mutex_unlock(&(rec->mutex));
1237 		//syslog(LOG_NOTICE,"threc(%"PRIu32") - received",rec->packetid);
1238 		return rec->ibuff;
1239 	}
1240 	return NULL;
1241 }
1242 
1243 //static inline const uint8_t* fs_sendandreceive(threc *rec,uint32_t expected_cmd,uint32_t *answer_leng) {
1244 //	uint32_t *rcmd;
1245 //	const uint8_t *rptr;
1246 //	rptr = fs_commwithmaster(rec,&rcmd,answer_leng);
1247 //	if (
1248 //}
1249 
1250 /*
1251 int fs_direct_connect() {
1252 	int rfd;
1253 	rfd = tcpsocket();
1254 	if (tcpnumconnect(rfd,masterip,masterport)<0) {
1255 		tcpclose(rfd);
1256 		return -1;
1257 	}
1258 	master_stats_inc(MASTER_TCONNECTS);
1259 	return rfd;
1260 }
1261 
1262 void fs_direct_close(int rfd) {
1263 	tcpclose(rfd);
1264 }
1265 
1266 int fs_direct_write(int rfd,const uint8_t *buff,uint32_t size) {
1267 	int rsize = tcptowrite(rfd,buff,size,60000);
1268 	if (rsize==(int)size) {
1269 		master_stats_add(MASTER_BYTESSENT,size);
1270 	}
1271 	return rsize;
1272 }
1273 
1274 int fs_direct_read(int rfd,uint8_t *buff,uint32_t size) {
1275 	int rsize = tcptoread(rfd,buff,size,60000);
1276 	if (rsize>0) {
1277 		master_stats_add(MASTER_BYTESRCVD,rsize);
1278 	}
1279 	return rsize;
1280 }
1281 */
1282 
fs_resolve(uint8_t oninit,const char * bindhostname,const char * masterhostname,const char * masterportname)1283 int fs_resolve(uint8_t oninit,const char *bindhostname,const char *masterhostname,const char *masterportname) {
1284 	if (bindhostname) {
1285 		if (tcpresolve(bindhostname,NULL,&srcip,NULL,1)<0) {
1286 			if (oninit) {
1287 				fprintf(stderr,"can't resolve source hostname (%s)\n",bindhostname);
1288 			} else {
1289 				syslog(LOG_WARNING,"can't resolve source hostname (%s)",bindhostname);
1290 			}
1291 			return -1;
1292 		}
1293 	} else {
1294 		srcip=0;
1295 	}
1296 	snprintf(srcstrip,17,"%"PRIu32".%"PRIu32".%"PRIu32".%"PRIu32,(srcip>>24)&0xFF,(srcip>>16)&0xFF,(srcip>>8)&0xFF,srcip&0xFF);
1297 	srcstrip[16]=0;
1298 
1299 	if (tcpresolve(masterhostname,masterportname,&masterip,&masterport,0)<0) {
1300 		if (oninit) {
1301 			fprintf(stderr,"can't resolve master hostname and/or portname (%s:%s)\n",masterhostname,masterportname);
1302 		} else {
1303 			syslog(LOG_WARNING,"can't resolve master hostname and/or portname (%s:%s)",masterhostname,masterportname);
1304 		}
1305 		return -1;
1306 	}
1307 	snprintf(masterstrip,17,"%"PRIu32".%"PRIu32".%"PRIu32".%"PRIu32,(masterip>>24)&0xFF,(masterip>>16)&0xFF,(masterip>>8)&0xFF,masterip&0xFF);
1308 	masterstrip[16]=0;
1309 
1310 	return 0;
1311 }
1312 
1313 // int fs_connect(uint8_t oninit,const char *bindhostname,const char *masterhostname,const char *masterportname,uint8_t meta,const char *info,const char *subfolder,const uint8_t passworddigest[16],uint8_t *sesflags,uint32_t *rootuid,uint32_t *rootgid,uint32_t *mapalluid,uint32_t *mapallgid,uint8_t *mingoal,uint8_t *maxgoal,uint32_t *mintrashtime,uint32_t *maxtrashtime) {
fs_connect(uint8_t oninit,struct connect_args_t * cargs)1314 int fs_connect(uint8_t oninit,struct connect_args_t *cargs) {
1315 	uint32_t i,j;
1316 	uint8_t *wptr,*regbuff;
1317 	md5ctx ctx;
1318 	uint8_t digest[16];
1319 	const uint8_t *rptr;
1320 	uint32_t newmasterip;
1321 	uint8_t havepassword;
1322 	uint32_t pleng,ileng;
1323 	uint8_t sesflags;
1324 	uint16_t umaskval;
1325 	uint32_t rootuid,rootgid,mapalluid,mapallgid;
1326 	uint8_t mingoal,maxgoal;
1327 	uint32_t mintrashtime,maxtrashtime;
1328 	uint32_t disables;
1329 	int32_t rleng;
1330 	const char* disablestr[]={DISABLE_STRINGS};
1331 	const char *sesflagposstrtab[]={SESFLAG_POS_STRINGS};
1332 	const char *sesflagnegstrtab[]={SESFLAG_NEG_STRINGS};
1333 #ifndef WIN32
1334 	struct passwd pwd,*pw;
1335 	struct group grp,*gr;
1336 	char pwdgrpbuff[16384];
1337 #endif
1338 	static uint32_t trycnt=0;
1339 
1340 	if (fs_resolve(oninit,cargs->bindhostname,cargs->masterhostname,cargs->masterportname)<0) {
1341 		return -1;
1342 	}
1343 
1344 	havepassword = (cargs->passworddigest==NULL)?0:1;
1345 	ileng = strlen(cargs->info)+1;
1346 	if (cargs->meta) {
1347 		pleng = 0;
1348 		rleng = 9;
1349 	} else {
1350 		pleng = strlen(cargs->subfolder)+1;
1351 		rleng = 13;
1352 	}
1353 	rleng += 8+64+pleng+ileng;
1354 	if (havepassword) {
1355 		rleng += 16;
1356 	}
1357 	if (sessionlost==2) {
1358 		rleng += 4;
1359 		if (metaid!=0) {
1360 			rleng += 8;
1361 		}
1362 	}
1363 	regbuff = malloc(rleng);
1364 
1365 	do {
1366 		fd = tcpsocket();
1367 		if (fd<0) {
1368 			free(regbuff);
1369 			return -1;
1370 		}
1371 		if (tcpnodelay(fd)<0) {
1372 			if (oninit) {
1373 				fprintf(stderr,"can't set TCP_NODELAY\n");
1374 			} else {
1375 				syslog(LOG_WARNING,"can't set TCP_NODELAY");
1376 			}
1377 		}
1378 		if (srcip>0) {
1379 			if (tcpnumbind(fd,srcip,0)<0) {
1380 				if (oninit) {
1381 					fprintf(stderr,"can't bind socket to given ip (\"%s\")\n",srcstrip);
1382 				} else {
1383 					syslog(LOG_WARNING,"can't bind socket to given ip (\"%s\")",srcstrip);
1384 				}
1385 				tcpclose(fd);
1386 				fd=-1;
1387 				free(regbuff);
1388 				return -1;
1389 			}
1390 		}
1391 		if (tcpnumtoconnect(fd,masterip,masterport,CONNECT_TIMEOUT)<0) {
1392 			tcpclose(fd);
1393 			fd=-1;
1394 			if (oninit) {
1395 				if (trycnt<10) {
1396 					trycnt++;
1397 					if (fs_resolve(oninit,cargs->bindhostname,cargs->masterhostname,cargs->masterportname)<0) {
1398 						free(regbuff);
1399 						return -1;
1400 					}
1401 					i=4;
1402 					continue;
1403 				} else {
1404 					fprintf(stderr,"can't connect to mfsmaster (\"%s\":\"%"PRIu16"\")\n",masterstrip,masterport);
1405 					free(regbuff);
1406 					return -1;
1407 				}
1408 			} else {
1409 				syslog(LOG_WARNING,"can't connect to mfsmaster (\"%s\":\"%"PRIu16"\")",masterstrip,masterport);
1410 				free(regbuff);
1411 				return -1;
1412 			}
1413 		}
1414 		if (havepassword) {
1415 			wptr = regbuff;
1416 			put32bit(&wptr,CLTOMA_FUSE_REGISTER);
1417 			put32bit(&wptr,65);
1418 			memcpy(wptr,FUSE_REGISTER_BLOB_ACL,64);
1419 			wptr+=64;
1420 			put8bit(&wptr,REGISTER_GETRANDOM);
1421 			if (tcptowrite(fd,regbuff,8+65,1000)!=8+65) {
1422 				if (oninit) {
1423 					fprintf(stderr,"error sending data to mfsmaster\n");
1424 				} else {
1425 					syslog(LOG_WARNING,"error sending data to mfsmaster");
1426 				}
1427 				tcpclose(fd);
1428 				fd=-1;
1429 				free(regbuff);
1430 				return -1;
1431 			}
1432 			if (tcptoread(fd,regbuff,8,1000)!=8) {
1433 				if (oninit) {
1434 					fprintf(stderr,"error receiving data from mfsmaster\n");
1435 				} else {
1436 					syslog(LOG_WARNING,"error receiving data from mfsmaster");
1437 				}
1438 				tcpclose(fd);
1439 				fd=-1;
1440 				free(regbuff);
1441 				return -1;
1442 			}
1443 			rptr = regbuff;
1444 			i = get32bit(&rptr);
1445 			if (i!=MATOCL_FUSE_REGISTER) {
1446 				if (oninit) {
1447 					fprintf(stderr,"got incorrect answer from mfsmaster\n");
1448 				} else {
1449 					syslog(LOG_WARNING,"got incorrect answer from mfsmaster");
1450 				}
1451 				tcpclose(fd);
1452 				fd=-1;
1453 				free(regbuff);
1454 				return -1;
1455 			}
1456 			i = get32bit(&rptr);
1457 			if (i!=32) {
1458 				if (oninit) {
1459 					fprintf(stderr,"got incorrect answer from mfsmaster\n");
1460 				} else {
1461 					syslog(LOG_WARNING,"got incorrect answer from mfsmaster");
1462 				}
1463 				tcpclose(fd);
1464 				fd=-1;
1465 				free(regbuff);
1466 				return -1;
1467 			}
1468 			if (tcptoread(fd,regbuff,32,1000)!=32) {
1469 				if (oninit) {
1470 					fprintf(stderr,"error receiving data from mfsmaster\n");
1471 				} else {
1472 					syslog(LOG_WARNING,"error receiving data from mfsmaster");
1473 				}
1474 				tcpclose(fd);
1475 				fd=-1;
1476 				free(regbuff);
1477 				return -1;
1478 			}
1479 			md5_init(&ctx);
1480 			md5_update(&ctx,regbuff,16);
1481 			md5_update(&ctx,cargs->passworddigest,16);
1482 			md5_update(&ctx,regbuff+16,16);
1483 			md5_final(digest,&ctx);
1484 		}
1485 		wptr = regbuff;
1486 		put32bit(&wptr,CLTOMA_FUSE_REGISTER);
1487 		put32bit(&wptr,rleng-8);
1488 		memcpy(wptr,FUSE_REGISTER_BLOB_ACL,64);
1489 		wptr+=64;
1490 		put8bit(&wptr,(cargs->meta)?REGISTER_NEWMETASESSION:REGISTER_NEWSESSION);
1491 		put16bit(&wptr,VERSMAJ);
1492 		put8bit(&wptr,VERSMID);
1493 		put8bit(&wptr,VERSMIN);
1494 		put32bit(&wptr,ileng);
1495 		memcpy(wptr,cargs->info,ileng);
1496 		wptr+=ileng;
1497 		if (!cargs->meta) {
1498 			put32bit(&wptr,pleng);
1499 			memcpy(wptr,cargs->subfolder,pleng);
1500 			wptr+=pleng;
1501 		}
1502 		if (sessionlost==2) {
1503 			put32bit(&wptr,sessionid);
1504 			if (metaid!=0) {
1505 				put64bit(&wptr,metaid);
1506 			}
1507 		}
1508 		if (havepassword) {
1509 			memcpy(wptr,digest,16);
1510 		}
1511 		if (tcptowrite(fd,regbuff,rleng,1000)!=rleng) {
1512 			if (oninit) {
1513 				fprintf(stderr,"error sending data to mfsmaster: %s\n",strerr(errno));
1514 			} else {
1515 				syslog(LOG_WARNING,"error sending data to mfsmaster: %s",strerr(errno));
1516 			}
1517 			tcpclose(fd);
1518 			fd=-1;
1519 			free(regbuff);
1520 			return -1;
1521 		}
1522 		if (tcptoread(fd,regbuff,8,1000)!=8) {
1523 			if (oninit) {
1524 				fprintf(stderr,"error receiving data from mfsmaster: %s\n",strerr(errno));
1525 			} else {
1526 				syslog(LOG_WARNING,"error receiving data from mfsmaster: %s",strerr(errno));
1527 			}
1528 			tcpclose(fd);
1529 			fd=-1;
1530 			free(regbuff);
1531 			return -1;
1532 		}
1533 		rptr = regbuff;
1534 		i = get32bit(&rptr);
1535 		if (i!=MATOCL_FUSE_REGISTER) {
1536 			if (oninit) {
1537 				fprintf(stderr,"got incorrect answer from mfsmaster\n");
1538 			} else {
1539 				syslog(LOG_WARNING,"got incorrect answer from mfsmaster");
1540 			}
1541 			tcpclose(fd);
1542 			fd=-1;
1543 			free(regbuff);
1544 			return -1;
1545 		}
1546 		i = get32bit(&rptr);
1547 		if (!(i==1 || i==4 || (cargs->meta && (i==19 || i==27)) || (cargs->meta==0 && (i==35 || i==43 || i==45 || i==49)))) {
1548 			if (oninit) {
1549 				fprintf(stderr,"got incorrect answer from mfsmaster\n");
1550 			} else {
1551 				syslog(LOG_WARNING,"got incorrect answer from mfsmaster");
1552 			}
1553 			tcpclose(fd);
1554 			fd=-1;
1555 			free(regbuff);
1556 			return -1;
1557 		}
1558 		if (tcptoread(fd,regbuff,i,1000)!=(int32_t)i) {
1559 			if (oninit) {
1560 				fprintf(stderr,"error receiving data from mfsmaster: %s\n",strerr(errno));
1561 			} else {
1562 				syslog(LOG_WARNING,"error receiving data from mfsmaster: %s",strerr(errno));
1563 			}
1564 			tcpclose(fd);
1565 			fd=-1;
1566 			free(regbuff);
1567 			return -1;
1568 		}
1569 		rptr = regbuff;
1570 		if (i==1) {
1571 			if (oninit) {
1572 				fprintf(stderr,"mfsmaster register error: %s\n",mfs_strerror(rptr[0]));
1573 			} else {
1574 				syslog(LOG_WARNING,"mfsmaster register error: %s",mfs_strerror(rptr[0]));
1575 			}
1576 			tcpclose(fd);
1577 			fd=-1;
1578 			free(regbuff);
1579 			return -1;
1580 		}
1581 		if (i==4) {
1582 			// redirect
1583 			newmasterip = get32bit(&rptr);
1584 			if (newmasterip==0 || newmasterip==masterip) {
1585 				if (trycnt<10) {
1586 					trycnt++;
1587 					if (oninit) {
1588 						fprintf(stderr,"mfsmaster %s - this is ELECT waitng for being LEADER, waiting a moment and retrying\n",masterstrip);
1589 					} else {
1590 						syslog(LOG_WARNING,"mfsmaster %s - this is ELECT waitng for being LEADER, waiting a moment and retrying",masterstrip);
1591 					}
1592 					tcpclose(fd);
1593 					fd = -1;
1594 					if (oninit) {
1595 						sleep(2);
1596 					} else {
1597 						free(regbuff);
1598 						return -1;
1599 					}
1600 				} else {
1601 					trycnt=0;
1602 					if (oninit) {
1603 						fprintf(stderr,"mfsmaster %s - this is ELECT waitng for being LEADER, waiting a moment and retrying using different IP\n",masterstrip);
1604 					} else {
1605 						syslog(LOG_WARNING,"mfsmaster %s - this is ELECT waitng for being LEADER, waiting a moment and retrying using different IP",masterstrip);
1606 					}
1607 					if (oninit) {
1608 						if (fs_resolve(oninit,cargs->bindhostname,cargs->masterhostname,cargs->masterportname)<0) {
1609 							free(regbuff);
1610 							return -1;
1611 						}
1612 						sleep(2);
1613 					} else {
1614 						free(regbuff);
1615 						return -1;
1616 					}
1617 				}
1618 			} else {
1619 				char newmasterstrip[17];
1620 				snprintf(newmasterstrip,17,"%"PRIu32".%"PRIu32".%"PRIu32".%"PRIu32,(newmasterip>>24)&0xFF,(newmasterip>>16)&0xFF,(newmasterip>>8)&0xFF,newmasterip&0xFF);
1621 				newmasterstrip[16]=0;
1622 				if (oninit) {
1623 					fprintf(stderr,"mfsmaster %s - found leader: %s\n",masterstrip,newmasterstrip);
1624 				} else {
1625 					syslog(LOG_WARNING,"mfsmaster %s - found leader: %s",masterstrip,newmasterstrip);
1626 				}
1627 				masterip = newmasterip;
1628 				strcpy(masterstrip,newmasterstrip);
1629 				tcpclose(fd);
1630 				fd = -1;
1631 			}
1632 		}
1633 	} while (i==4);
1634 	masterversion = get32bit(&rptr);
1635 	if (masterversion < VERSION2INT(2,1,7)) {
1636 		if (oninit) {
1637 			fprintf(stderr,"incompatible mfsmaster version\n");
1638 		} else {
1639 			syslog(LOG_WARNING,"incompatible mfsmaster version");
1640 		}
1641 		tcpclose(fd);
1642 		fd=-1;
1643 		free(regbuff);
1644 		return -1;
1645 	}
1646 	attrsize = (masterversion>=VERSION2INT(3,0,93))?ATTR_RECORD_SIZE:35;
1647 	sessionid = get32bit(&rptr);
1648 	if ((cargs->meta && i==27) || (cargs->meta==0 && (i==43 || i==45 || i==49))) {
1649 		metaid = get64bit(&rptr);
1650 	}
1651 	sesflags = get8bit(&rptr);
1652 	if (!cargs->meta) {
1653 		if (i==45 || i==49) {
1654 			umaskval = get16bit(&rptr);
1655 		} else {
1656 			umaskval = 0;
1657 		}
1658 		rootuid = get32bit(&rptr);
1659 		rootgid = get32bit(&rptr);
1660 		mapalluid = get32bit(&rptr);
1661 		mapallgid = get32bit(&rptr);
1662 	} else {
1663 		umaskval = 0;
1664 		rootuid = 0;
1665 		rootgid = 0;
1666 		mapalluid = 0;
1667 		mapallgid = 0;
1668 	}
1669 	mingoal = get8bit(&rptr);
1670 	maxgoal = get8bit(&rptr);
1671 	mintrashtime = get32bit(&rptr);
1672 	maxtrashtime = get32bit(&rptr);
1673 	if (i==49) {
1674 		disables = get32bit(&rptr);
1675 	} else {
1676 		disables = 0;
1677 	}
1678 	free(regbuff);
1679 	lastwrite = monotonic_seconds();
1680 #ifdef MFSMOUNT
1681 	mfs_setdisables(disables);
1682 	main_setparams(sesflags,umaskval,rootuid,rootgid,mapalluid,mapallgid,mingoal,maxgoal,mintrashtime,maxtrashtime,disables);
1683 #endif
1684 	if (oninit==0) {
1685 		if (sessionlost==2) {
1686 			syslog(LOG_NOTICE,"registered to master using previous session");
1687 		} else {
1688 			syslog(LOG_NOTICE,"registered to master with new session");
1689 		}
1690 	}
1691 	if (cargs->clearpassword && cargs->passworddigest!=NULL) {
1692 		memset(cargs->passworddigest,0,16);
1693 		free(cargs->passworddigest);
1694 		cargs->passworddigest = NULL;
1695 	}
1696 	if (oninit==1) {
1697 		fprintf(stderr,"mfsmaster accepted connection with parameters: ");
1698 		j=0;
1699 		for (i=0 ; i<8 ; i++) {
1700 			if (sesflags&(1<<i)) {
1701 				fprintf(stderr,"%s%s",j?",":"",sesflagposstrtab[i]);
1702 				j=1;
1703 			} else if (sesflagnegstrtab[i]) {
1704 				fprintf(stderr,"%s%s",j?",":"",sesflagnegstrtab[i]);
1705 				j=1;
1706 			}
1707 		}
1708 		if (j==0) {
1709 			fprintf(stderr,"-");
1710 		}
1711 		if (!cargs->meta) {
1712 #ifdef WIN32
1713 			fprintf(stderr," ; root mapped to %"PRIu32":%"PRIu32,rootuid,rootgid);
1714 			if (sesflags&SESFLAG_MAPALL) {
1715 				fprintf(stderr," ; users mapped to %"PRIu32":%"PRIu32,mapalluid,mapallgid);
1716 			}
1717 #else
1718 			if (umaskval!=0) {
1719 				fprintf(stderr," ; global umask set to 0%03"PRIo16,umaskval);
1720 			}
1721 			fprintf(stderr," ; root mapped to ");
1722 			getpwuid_r(rootuid,&pwd,pwdgrpbuff,16384,&pw);
1723 	//		pw = getpwuid(rootuid);
1724 			if (pw) {
1725 				fprintf(stderr,"%s:",pw->pw_name);
1726 			} else {
1727 				fprintf(stderr,"%"PRIu32":",rootuid);
1728 			}
1729 			getgrgid_r(rootgid,&grp,pwdgrpbuff,16384,&gr);
1730 	//		gr = getgrgid(rootgid);
1731 			if (gr) {
1732 				fprintf(stderr,"%s",gr->gr_name);
1733 			} else {
1734 				fprintf(stderr,"%"PRIu32,rootgid);
1735 			}
1736 			if (sesflags&SESFLAG_MAPALL) {
1737 				fprintf(stderr," ; users mapped to ");
1738 				pw = getpwuid(mapalluid);
1739 				if (pw) {
1740 					fprintf(stderr,"%s:",pw->pw_name);
1741 				} else {
1742 					fprintf(stderr,"%"PRIu32":",mapalluid);
1743 				}
1744 				gr = getgrgid(mapallgid);
1745 				if (gr) {
1746 					fprintf(stderr,"%s",gr->gr_name);
1747 				} else {
1748 					fprintf(stderr,"%"PRIu32,mapallgid);
1749 				}
1750 			}
1751 #endif
1752 		}
1753 		if (mingoal>0 && maxgoal>0) {
1754 			if (mingoal>1 || maxgoal<9) {
1755 				fprintf(stderr," ; setgoal limited to (%u:%u)",mingoal,maxgoal);
1756 			}
1757 			if (mintrashtime>0 || maxtrashtime<UINT32_C(0xFFFFFFFF)) {
1758 				fprintf(stderr," ; settrashtime limited to (");
1759 				if (mintrashtime>0) {
1760 					if (mintrashtime>604800) {
1761 						fprintf(stderr,"%uw",mintrashtime/604800);
1762 						mintrashtime %= 604800;
1763 					}
1764 					if (mintrashtime>86400) {
1765 						fprintf(stderr,"%ud",mintrashtime/86400);
1766 						mintrashtime %= 86400;
1767 					}
1768 					if (mintrashtime>3600) {
1769 						fprintf(stderr,"%uh",mintrashtime/3600);
1770 						mintrashtime %= 3600;
1771 					}
1772 					if (mintrashtime>60) {
1773 						fprintf(stderr,"%um",mintrashtime/60);
1774 						mintrashtime %= 60;
1775 					}
1776 					if (mintrashtime>0) {
1777 						fprintf(stderr,"%us",mintrashtime);
1778 					}
1779 				} else {
1780 					fprintf(stderr,"0s");
1781 				}
1782 				fprintf(stderr,":");
1783 				if (maxtrashtime>0) {
1784 					if (maxtrashtime>604800) {
1785 						fprintf(stderr,"%uw",maxtrashtime/604800);
1786 						maxtrashtime %= 604800;
1787 					}
1788 					if (maxtrashtime>86400) {
1789 						fprintf(stderr,"%ud",maxtrashtime/86400);
1790 						maxtrashtime %= 86400;
1791 					}
1792 					if (maxtrashtime>3600) {
1793 						fprintf(stderr,"%uh",maxtrashtime/3600);
1794 						maxtrashtime %= 3600;
1795 					}
1796 					if (maxtrashtime>60) {
1797 						fprintf(stderr,"%um",maxtrashtime/60);
1798 						maxtrashtime %= 60;
1799 					}
1800 					if (maxtrashtime>0) {
1801 						fprintf(stderr,"%us",maxtrashtime);
1802 					}
1803 				} else {
1804 					fprintf(stderr,"0s");
1805 				}
1806 				fprintf(stderr,")");
1807 			}
1808 		}
1809 		if (disables>0) {
1810 			int s;
1811 			fprintf(stderr," ; disabled commands: ");
1812 			s = 0;
1813 			for (i=0,j=1 ; disablestr[i]!=NULL ; i++,j<<=1) {
1814 				if (disables&j) {
1815 					fprintf(stderr,"%s%s",s?",":"",disablestr[i]);
1816 					s = 1;
1817 				}
1818 			}
1819 		}
1820 		fprintf(stderr,"\n");
1821 	}
1822 	return 0;
1823 }
1824 
fs_reconnect()1825 void fs_reconnect() {
1826 	uint32_t newmasterip;
1827 	uint32_t i;
1828 	uint8_t *wptr,regbuff[8+64+17];
1829 	int32_t rleng;
1830 	const uint8_t *rptr;
1831 
1832 	if (sessionid==0) {
1833 		syslog(LOG_WARNING,"can't register: session not created");
1834 		return;
1835 	}
1836 
1837 	snprintf(masterstrip,17,"%"PRIu32".%"PRIu32".%"PRIu32".%"PRIu32,(masterip>>24)&0xFF,(masterip>>16)&0xFF,(masterip>>8)&0xFF,masterip&0xFF);
1838 	masterstrip[16]=0;
1839 
1840 	do {
1841 		fd = tcpsocket();
1842 		if (fd<0) {
1843 			return;
1844 		}
1845 		if (tcpnodelay(fd)<0) {
1846 			syslog(LOG_WARNING,"can't set TCP_NODELAY: %s",strerr(errno));
1847 		}
1848 		if (srcip>0) {
1849 			if (tcpnumbind(fd,srcip,0)<0) {
1850 				syslog(LOG_WARNING,"can't bind socket to given ip (\"%s\")",srcstrip);
1851 				tcpclose(fd);
1852 				fd=-1;
1853 				return;
1854 			}
1855 		}
1856 		if (tcpnumtoconnect(fd,masterip,masterport,CONNECT_TIMEOUT)<0) {
1857 			syslog(LOG_WARNING,"can't connect to master (\"%s\":\"%"PRIu16"\")",masterstrip,masterport);
1858 			tcpclose(fd);
1859 			fd=-1;
1860 			return;
1861 		}
1862 		master_stats_inc(MASTER_CONNECTS);
1863 		wptr = regbuff;
1864 		put32bit(&wptr,CLTOMA_FUSE_REGISTER);
1865 		if (masterversion>=VERSION2INT(3,0,11) && metaid!=0) {
1866 			put32bit(&wptr,81);
1867 			rleng = 8+81;
1868 		} else {
1869 			put32bit(&wptr,73);
1870 			rleng = 8+73;
1871 		}
1872 		memcpy(wptr,FUSE_REGISTER_BLOB_ACL,64);
1873 		wptr+=64;
1874 		put8bit(&wptr,REGISTER_RECONNECT);
1875 		put32bit(&wptr,sessionid);
1876 		put16bit(&wptr,VERSMAJ);
1877 		put8bit(&wptr,VERSMID);
1878 		put8bit(&wptr,VERSMIN);
1879 		put64bit(&wptr,metaid);
1880 		if (tcptowrite(fd,regbuff,rleng,1000)!=rleng) {
1881 			syslog(LOG_WARNING,"master: register error (write: %s)",strerr(errno));
1882 			tcpclose(fd);
1883 			fd=-1;
1884 			return;
1885 		}
1886 		master_stats_add(MASTER_BYTESSENT,rleng);
1887 		master_stats_inc(MASTER_PACKETSSENT);
1888 		if (tcptoread(fd,regbuff,8,1000)!=8) {
1889 			syslog(LOG_WARNING,"master: register error (read header: %s)",strerr(errno));
1890 			tcpclose(fd);
1891 			fd=-1;
1892 			return;
1893 		}
1894 		master_stats_add(MASTER_BYTESRCVD,8);
1895 		rptr = regbuff;
1896 		i = get32bit(&rptr);
1897 		if (i!=MATOCL_FUSE_REGISTER) {
1898 			syslog(LOG_WARNING,"master: register error (bad answer: %"PRIu32")",i);
1899 			tcpclose(fd);
1900 			fd=-1;
1901 			return;
1902 		}
1903 		i = get32bit(&rptr);
1904 		if (i!=1 && i!=4 && i!=5) {
1905 			syslog(LOG_WARNING,"master: register error (bad length: %"PRIu32")",i);
1906 			tcpclose(fd);
1907 			fd=-1;
1908 			return;
1909 		}
1910 		if (tcptoread(fd,regbuff,i,1000)!=(int32_t)i) {
1911 			syslog(LOG_WARNING,"master: register error (read data: %s)",strerr(errno));
1912 			tcpclose(fd);
1913 			fd=-1;
1914 			return;
1915 		}
1916 		master_stats_add(MASTER_BYTESRCVD,i);
1917 		master_stats_inc(MASTER_PACKETSRCVD);
1918 		rptr = regbuff;
1919 		if (i==4) {
1920 			// redirect
1921 			newmasterip = get32bit(&rptr);
1922 			if (newmasterip==0) {
1923 				syslog(LOG_WARNING,"mfsmaster %s - doesn't know his leader, waiting a moment and retrying using different IP",masterstrip);
1924 				tcpclose(fd);
1925 				fd = -1;
1926 				return;
1927 			} else {
1928 				if (newmasterip==masterip) { // this is ELECT
1929 					syslog(LOG_WARNING,"mfsmaster %s - this is ELECT waitng for being LEADER, waiting a moment and retrying",masterstrip);
1930 					tcpclose(fd);
1931 					fd = -1;
1932 					return;
1933 				} else {
1934 					masterip = newmasterip;
1935 					snprintf(masterstrip,17,"%"PRIu32".%"PRIu32".%"PRIu32".%"PRIu32,(masterip>>24)&0xFF,(masterip>>16)&0xFF,(masterip>>8)&0xFF,masterip&0xFF);
1936 					masterstrip[16]=0;
1937 					syslog(LOG_WARNING,"mfsmaster found leader: %s",masterstrip);
1938 					tcpclose(fd);
1939 					fd = -1;
1940 				}
1941 			}
1942 		}
1943 	} while (i==4);
1944 	if (i==5) {
1945 		masterversion = get32bit(&rptr);
1946 		attrsize = ATTR_RECORD_SIZE;
1947 	}
1948 	if (rptr[0]!=0) {
1949 		sessionlost=(rptr[0]==MFS_ERROR_EPERM)?2:1;
1950 		syslog(LOG_WARNING,"master: register status: %s",mfs_strerror(rptr[0]));
1951 		tcpclose(fd);
1952 		fd=-1;
1953 		return;
1954 	}
1955 	lastwrite = monotonic_seconds();
1956 	lastsyncsend = 0;
1957 	syslog(LOG_NOTICE,"registered to master");
1958 }
1959 
fs_close_session(void)1960 void fs_close_session(void) {
1961 	uint8_t *wptr,regbuff[8+64+5+8];
1962 	int32_t rleng;
1963 
1964 	if (sessionid==0) {
1965 		return;
1966 	}
1967 
1968 	wptr = regbuff;
1969 	put32bit(&wptr,CLTOMA_FUSE_REGISTER);
1970 	if (masterversion>=VERSION2INT(3,0,11) && metaid!=0) {
1971 		put32bit(&wptr,77);
1972 		rleng = 8+77;
1973 	} else {
1974 		put32bit(&wptr,69);
1975 		rleng = 8+69;
1976 	}
1977 	memcpy(wptr,FUSE_REGISTER_BLOB_ACL,64);
1978 	wptr+=64;
1979 	put8bit(&wptr,REGISTER_CLOSESESSION);
1980 	put32bit(&wptr,sessionid);
1981 	put64bit(&wptr,metaid);
1982 	if (tcptowrite(fd,regbuff,rleng,1000)!=rleng) {
1983 		syslog(LOG_WARNING,"master: close session error (write: %s)",strerr(errno));
1984 	}
1985 	if (masterversion>=VERSION2INT(1,7,29)) {
1986 		if (tcptoread(fd,regbuff,9,500)!=9) {
1987 			syslog(LOG_WARNING,"master: close session error (read: %s)",strerr(errno));
1988 		} else if (regbuff[8]!=0) {
1989 			syslog(LOG_NOTICE,"master: closes session error: %s",mfs_strerror(regbuff[8]));
1990 		}
1991 	}
1992 }
1993 
fs_send_amtime_inodes(void)1994 void fs_send_amtime_inodes(void) {
1995 	uint8_t *ptr,*inodespacket;
1996 	int32_t inodesleng;
1997 	amtime_file *amfptr,**amfpptr;
1998 	uint32_t amhash;
1999 
2000 	pthread_mutex_lock(&amtimelock);
2001 	//inodesleng=24;
2002 	if (masterversion>=VERSION2INT(3,0,74)) {
2003 		inodesleng=0;
2004 		for (amhash=0 ; amhash < AMTIME_HASH_SIZE ; amhash++) {
2005 			for (amfptr = amtime_hash[amhash] ; amfptr ; amfptr = amfptr->next) {
2006 				if (amfptr->atime > 0 || amfptr->mtime > 0) {
2007 					inodesleng+=12;
2008 				}
2009 			}
2010 		}
2011 		if (inodesleng>0) {
2012 			inodesleng+=8;
2013 			inodespacket = malloc(inodesleng);
2014 			ptr = inodespacket;
2015 			put32bit(&ptr,CLTOMA_FUSE_AMTIME_INODES);
2016 			put32bit(&ptr,inodesleng-8);
2017 			for (amhash=0 ; amhash < AMTIME_HASH_SIZE ; amhash++) {
2018 				amfpptr = amtime_hash + amhash;
2019 				while ((amfptr = *amfpptr)) {
2020 					if (amfptr->atime > 0 || amfptr->mtime > 0) {
2021 						put32bit(&ptr,amfptr->inode);
2022 						put32bit(&ptr,amfptr->atime/1000000);
2023 						put32bit(&ptr,amfptr->mtime/1000000);
2024 					}
2025 					if (amfptr->atimeage>=1) { // sending second time - clear it
2026 						amfptr->atime = 0;
2027 					}
2028 					if (amfptr->mtimeage>=1) { // sending second time - clear it
2029 						amfptr->mtime = 0;
2030 					}
2031 					if (amfptr->atimeage < AMTIME_MAX_AGE || amfptr->mtimeage < AMTIME_MAX_AGE) {
2032 						if (amfptr->atimeage < AMTIME_MAX_AGE) {
2033 							amfptr->atimeage++;
2034 						}
2035 						if (amfptr->mtimeage < AMTIME_MAX_AGE) {
2036 							amfptr->mtimeage++;
2037 						}
2038 						amfpptr = &(amfptr->next);
2039 					} else {
2040 						*amfpptr = amfptr->next;
2041 						free(amfptr);
2042 					}
2043 				}
2044 			}
2045 			if (tcptowrite(fd,inodespacket,inodesleng,1000)!=inodesleng) {
2046 #ifdef HAVE___SYNC_FETCH_AND_OP
2047 				(void)__sync_fetch_and_or(&disconnect,1);
2048 #else
2049 				disconnect = 1;
2050 #endif
2051 			} else {
2052 				master_stats_add(MASTER_BYTESSENT,inodesleng);
2053 				master_stats_inc(MASTER_PACKETSSENT);
2054 			}
2055 			free(inodespacket);
2056 			pthread_mutex_unlock(&amtimelock);
2057 			return;
2058 		}
2059 	}
2060 	for (amhash=0 ; amhash < AMTIME_HASH_SIZE ; amhash++) {
2061 		amfpptr = amtime_hash + amhash;
2062 		while ((amfptr = *amfpptr)) {
2063 			if (amfptr->atimeage < AMTIME_MAX_AGE || amfptr->mtimeage < AMTIME_MAX_AGE) {
2064 				if (amfptr->atimeage < AMTIME_MAX_AGE) {
2065 					amfptr->atimeage++;
2066 				}
2067 				if (amfptr->mtimeage < AMTIME_MAX_AGE) {
2068 					amfptr->mtimeage++;
2069 				}
2070 				amfpptr = &(amfptr->next);
2071 			} else {
2072 				*amfpptr = amfptr->next;
2073 				free(amfptr);
2074 			}
2075 		}
2076 	}
2077 	pthread_mutex_unlock(&amtimelock);
2078 }
2079 
fs_send_open_inodes(void)2080 void fs_send_open_inodes(void) {
2081 	uint8_t *ptr,*inodespacket;
2082 	uint32_t i,inodes;
2083 	uint32_t hash;
2084 	acquired_file *afptr,**afpptr;
2085 #ifdef MFSDEBUG
2086 	uint32_t inode;
2087 #endif
2088 
2089 	pthread_mutex_lock(&aflock);
2090 	//inodesleng=24;
2091 	heap_cleanup();
2092 	for (hash=0 ; hash<ACQFILES_HASH_SIZE ; hash++) {
2093 		afpptr = af_hash + hash;
2094 		while ((afptr = *afpptr)) {
2095 			if (afptr->cnt==0 && afptr->dentry==0) {
2096 				afptr->age++;
2097 				if (afptr->age>ACQFILES_MAX_AGE) {
2098 					*afpptr = afptr->next;
2099 					chunksdatacache_clear_inode(afptr->inode,0);
2100 					fs_af_remove_from_lru(afptr);
2101 					free(afptr);
2102 					continue;
2103 				}
2104 			}
2105 			//syslog(LOG_NOTICE,"sustained inode: %"PRIu32,afptr->inode);
2106 			afpptr = &(afptr->next);
2107 			heap_push(afptr->inode);
2108 		}
2109 	}
2110 	inodes = heap_elements();
2111 
2112 	inodespacket = malloc(inodes*4+8);
2113 	ptr = inodespacket;
2114 	if (masterversion>=VERSION2INT(3,0,74)) {
2115 		put32bit(&ptr,CLTOMA_FUSE_SUSTAINED_INODES);
2116 	} else {
2117 		put32bit(&ptr,CLTOMA_FUSE_SUSTAINED_INODES_DEPRECATED);
2118 	}
2119 	put32bit(&ptr,inodes*4);
2120 	//put32bit(&ptr,inodesleng-24);
2121 	//put64bit(&ptr,0);	// readbytes
2122 	//put64bit(&ptr,0);	// writebytes
2123 	// readbytes = 0;
2124 	// writebytes = 0;
2125 	for (i=0 ; i<inodes ; i++) {
2126 #ifdef MFSDEBUG
2127 		inode = heap_pop();
2128 		syslog(LOG_NOTICE,"open inode(%"PRIu32"): %"PRIu32,i,inode);
2129 		put32bit(&ptr,inode);
2130 #else
2131 		put32bit(&ptr,heap_pop());
2132 #endif
2133 	}
2134 	i = inodes * 4 + 8;
2135 	if (tcptowrite(fd,inodespacket,i,1000)!=(int32_t)i) {
2136 #ifdef HAVE___SYNC_FETCH_AND_OP
2137 		(void)__sync_fetch_and_or(&disconnect,1);
2138 #else
2139 		disconnect = 1;
2140 #endif
2141 	} else {
2142 		master_stats_add(MASTER_BYTESSENT,i);
2143 		master_stats_inc(MASTER_PACKETSSENT);
2144 	}
2145 	free(inodespacket);
2146 	pthread_mutex_unlock(&aflock);
2147 }
2148 
fs_nop_thread(void * arg)2149 void* fs_nop_thread(void *arg) {
2150 	uint8_t *ptr,hdr[12];
2151 	uint64_t usec;
2152 	int now;
2153 	int inodeswritecnt=0;
2154 	(void)arg;
2155 	for (;;) {
2156 		pthread_mutex_lock(&fdlock);
2157 		if (fterm==2 && donotsendsustainedinodes==0) {
2158 			if (fd>=0) {
2159 				fs_send_amtime_inodes();
2160 				fs_send_open_inodes();
2161 				fs_close_session();
2162 				tcpclose(fd);
2163 				fd = -1;
2164 			}
2165 			pthread_mutex_unlock(&fdlock);
2166 			return NULL;
2167 		}
2168 #ifdef HAVE___SYNC_FETCH_AND_OP
2169 		if (__sync_fetch_and_or(&disconnect,0)==0 && fd>=0) {
2170 #else
2171 		if (disconnect==0 && fd>=0) {
2172 #endif
2173 			now = monotonic_seconds();
2174 			if (lastwrite+2.0<now) {	// NOP
2175 				ptr = hdr;
2176 				put32bit(&ptr,ANTOAN_NOP);
2177 				put32bit(&ptr,4);
2178 				put32bit(&ptr,0);
2179 				if (tcptowrite(fd,hdr,12,1000)!=12) {
2180 #ifdef HAVE___SYNC_FETCH_AND_OP
2181 					(void)__sync_fetch_and_or(&disconnect,1);
2182 #else
2183 					disconnect=1;
2184 #endif
2185 				} else {
2186 					master_stats_add(MASTER_BYTESSENT,12);
2187 					master_stats_inc(MASTER_PACKETSSENT);
2188 				}
2189 				lastwrite = now;
2190 			}
2191 			usec = monotonic_useconds();
2192 			if (masterversion>=VERSION2INT(3,0,74) && (lastsyncsend==0 || lastsyncsend+60000000<usec)) { // time sync
2193 				ptr = hdr;
2194 				put32bit(&ptr,CLTOMA_FUSE_TIME_SYNC);
2195 				put32bit(&ptr,4);
2196 				put32bit(&ptr,0);
2197 				if (tcptowrite(fd,hdr,12,1000)!=12) {
2198 #ifdef HAVE___SYNC_FETCH_AND_OP
2199 					(void)__sync_fetch_and_or(&disconnect,1);
2200 #else
2201 					disconnect=1;
2202 #endif
2203 				} else {
2204 					master_stats_add(MASTER_BYTESSENT,12);
2205 					master_stats_inc(MASTER_PACKETSSENT);
2206 				}
2207 				lastsyncsend = usec;
2208 			}
2209 			if (inodeswritecnt<=0 || inodeswritecnt>60) {
2210 				inodeswritecnt=60;
2211 			} else {
2212 				inodeswritecnt--;
2213 			}
2214 			if (inodeswritecnt==0) {	// HELD INODES
2215 				if (donotsendsustainedinodes) {
2216 					inodeswritecnt=1;
2217 				} else {
2218 					fs_send_open_inodes();
2219 				}
2220 			}
2221 			fs_send_amtime_inodes();
2222 		}
2223 		pthread_mutex_unlock(&fdlock);
2224 		sleep(1);
2225 	}
2226 }
2227 
2228 void* fs_receive_thread(void *arg) {
2229 	const uint8_t *ptr;
2230 	uint8_t hdr[12];
2231 	uint8_t msgbuff[29];
2232 	uint8_t internal;
2233 	threc *rec;
2234 	uint32_t cmd,size,packetid;
2235 	uint32_t rcvd,toread;
2236 	uint32_t rechash;
2237 //	static uint8_t *notify_buff=NULL;
2238 //	static uint32_t notify_buff_size=0;
2239 	int32_t r;
2240 
2241 	(void)arg;
2242 	for (;;) {
2243 		pthread_mutex_lock(&fdlock);
2244 		if (fterm) {
2245 			fterm=2;
2246 			pthread_mutex_unlock(&fdlock);
2247 			return NULL;
2248 		}
2249 #ifdef HAVE___SYNC_FETCH_AND_OP
2250 		if (__sync_fetch_and_and(&disconnect,0)) {
2251 #else
2252 		if (disconnect) {
2253 			disconnect = 0;
2254 #endif
2255 //			dir_cache_remove_all();
2256 			chunksdatacache_cleanup();
2257 			tcpclose(fd);
2258 			fd = -1;
2259 			// send to any threc status error and unlock them
2260 			pthread_mutex_lock(&reclock);
2261 			for (rechash=0 ; rechash<THRECHASHSIZE ; rechash++) {
2262 				for (rec=threchash[rechash] ; rec ; rec=rec->next) {
2263 					pthread_mutex_lock(&(rec->mutex));
2264 					if (rec->sent) {
2265 						rec->status = 1;
2266 						rec->rcvd = 1;
2267 						pthread_cond_signal(&(rec->cond));
2268 					}
2269 					pthread_mutex_unlock(&(rec->mutex));
2270 				}
2271 			}
2272 			pthread_mutex_unlock(&reclock);
2273 		}
2274 		if (fd==-1 && sessionid!=0) {
2275 			fs_reconnect();		// try to register using the same session id
2276 		}
2277 		if (fd==-1) {	// still not connected
2278 			if (sessionlost) {	// if previous session is lost then try to register as a new session
2279 				if (fs_connect(0,&connect_args)==0) {
2280 					sessionlost=0;
2281 				}
2282 			} else {	// if other problem occurred then try to resolve hostname and portname then try to reconnect using the same session id
2283 				if (fs_resolve(0,connect_args.bindhostname,connect_args.masterhostname,connect_args.masterportname)==0) {
2284 					fs_reconnect();
2285 				}
2286 			}
2287 		}
2288 		if (fd==-1) {
2289 			pthread_mutex_unlock(&fdlock);
2290 			sleep(2);	// reconnect every 2 seconds
2291 			continue;
2292 		}
2293 		pthread_mutex_unlock(&fdlock);
2294 		r = tcptoread(fd,hdr,12,RECEIVE_TIMEOUT*1000);	// read timeout
2295 		// syslog(LOG_NOTICE,"master: header size: %d",r);
2296 		if (r==0) {
2297 			syslog(LOG_WARNING,"master: connection lost (header)");
2298 			fs_disconnect();
2299 			continue;
2300 		}
2301 		if (r!=12) {
2302 			syslog(LOG_WARNING,"master: tcp recv error: %s (header)",strerr(errno));
2303 			fs_disconnect();
2304 			continue;
2305 		}
2306 		master_stats_add(MASTER_BYTESRCVD,12);
2307 		master_stats_inc(MASTER_PACKETSRCVD);
2308 
2309 		ptr = hdr;
2310 		cmd = get32bit(&ptr);
2311 		size = get32bit(&ptr);
2312 		packetid = get32bit(&ptr);
2313 		if (size<4) {
2314 			syslog(LOG_WARNING,"master: packet too small");
2315 			fs_disconnect();
2316 			continue;
2317 		}
2318 //		printf("got packet from master: cmd:%"PRIu32" ; size:%"PRIu32" ; packetid:%"PRIu32"\n",cmd,size,packetid);
2319 		size -= 4;
2320 		if (packetid==0) {
2321 			if (cmd==ANTOAN_NOP && size==0) {
2322 				// syslog(LOG_NOTICE,"master: got nop");
2323 				continue;
2324 			}
2325 			if (cmd==ANTOAN_UNKNOWN_COMMAND || cmd==ANTOAN_BAD_COMMAND_SIZE) { // just ignore these packets with packetid==0
2326 				continue;
2327 			}
2328 			internal = 0;
2329 //			if (cmd==MATOCL_FUSE_INVALIDATE_DATA_CACHE) {
2330 //				if (size==4) {
2331 //					internal = 1;
2332 //				} else {
2333 //					syslog(LOG_WARNING,"master: unexpected msg size (msg:MATOCL_FUSE_INVALIDATE_DATA_CACHE ; size:%"PRIu32"/4)",size);
2334 //					fs_disconnect();
2335 //					continue;
2336 //				}
2337 //			}
2338 			if (cmd==MATOCL_FUSE_CHUNK_HAS_CHANGED) {
2339 				if (size==29) {
2340 					internal = 1;
2341 				} else {
2342 					syslog(LOG_WARNING,"master: unexpected msg size (msg:MATOCL_FUSE_CHUNK_HAS_CHANGED ; size:%"PRIu32"/33)",size+4);
2343 					fs_disconnect();
2344 					continue;
2345 				}
2346 			}
2347 			if (cmd==MATOCL_FUSE_FLENG_HAS_CHANGED) {
2348 				if (size==12) {
2349 					internal = 1;
2350 				} else {
2351 					syslog(LOG_WARNING,"master: unexpected msg size (msg:MATOCL_FUSE_FLENG_HAS_CHANGED ; size:%"PRIu32"/16)",size+4);
2352 					fs_disconnect();
2353 					continue;
2354 				}
2355 			}
2356 			if (cmd==MATOCL_FUSE_TIME_SYNC) {
2357 				if (size==8) {
2358 					internal = 1;
2359 				} else {
2360 					syslog(LOG_WARNING,"master: unexpected msg size (msg:MATOCL_FUSE_TIME_SYNC ; size:%"PRIu32"/12)",size+4);
2361 					fs_disconnect();
2362 					continue;
2363 				}
2364 			}
2365 			if (cmd==MATOCL_FUSE_INVALIDATE_CHUNK_CACHE) {
2366 				if (size==0) {
2367 					internal = 1;
2368 				} else {
2369 					syslog(LOG_WARNING,"master: unexpected msg size (msg:MATOCL_FUSE_INVALIDATE_CHUNK_CACHE ; size:%"PRIu32"/4)",size+4);
2370 					fs_disconnect();
2371 					continue;
2372 				}
2373 			}
2374 			if (internal) {
2375 				if (size>0) {
2376 					r = tcptoread(fd,msgbuff,size,RECEIVE_TIMEOUT*1000);
2377 					if (r==0) {
2378 						syslog(LOG_WARNING,"master: connection lost (data)");
2379 						fs_disconnect();
2380 						continue;
2381 					}
2382 					if (r!=(int32_t)size) {
2383 						syslog(LOG_WARNING,"master: tcp recv error: %s (data)",strerr(errno));
2384 						fs_disconnect();
2385 						continue;
2386 					}
2387 					master_stats_add(MASTER_BYTESRCVD,size);
2388 				}
2389 				ptr = msgbuff;
2390 //				if (cmd==MATOCL_FUSE_INVALIDATE_DATA_CACHE) {
2391 //#ifndef WIN32
2392 //					uint32_t inode;
2393 //					inode = get32bit(&ptr);
2394 //					mfs_invalidate_data_cache(inode);
2395 //#endif
2396 //					continue;
2397 //				}
2398 				if (cmd==MATOCL_FUSE_FLENG_HAS_CHANGED) {
2399 					uint32_t inode;
2400 					uint64_t fleng;
2401 					inode = get32bit(&ptr);
2402 					fleng = get64bit(&ptr);
2403 //					printf("FLENG_HAS_CHANGED inode:%"PRIu32" ; fleng:%"PRIu64"\n",inode,fleng);
2404 					ep_fleng_has_changed(inode,fleng);
2405 					continue;
2406 				}
2407 				if (cmd==MATOCL_FUSE_CHUNK_HAS_CHANGED) {
2408 					uint32_t inode;
2409 					uint32_t chindx;
2410 					uint64_t chunkid;
2411 					uint32_t version;
2412 					uint64_t fleng;
2413 					uint8_t truncflag;
2414 					inode = get32bit(&ptr);
2415 					chindx = get32bit(&ptr);
2416 					chunkid = get64bit(&ptr);
2417 					version = get32bit(&ptr);
2418 					fleng = get64bit(&ptr);
2419 					truncflag = get8bit(&ptr);
2420 //					printf("CHUNK_HAS_CHANGED inode:%"PRIu32" ; chindx:%"PRIu32" ; chunkid:%"PRIu64" ; version:%"PRIu32" ; fleng:%"PRIu64" ; truncate:%"PRIu8"\n",inode,chindx,chunkid,version,fleng,truncflag);
2421 					ep_chunk_has_changed(inode,chindx,chunkid,version,fleng,truncflag);
2422 					continue;
2423 				}
2424 				if (cmd==MATOCL_FUSE_TIME_SYNC) {
2425 					uint64_t lusectime,rusectime,usec,usecping;
2426 					struct timeval tv;
2427 					usec = monotonic_useconds();
2428 					pthread_mutex_lock(&fdlock);
2429 					if (usec>=lastsyncsend) {
2430 						usecping = usec - lastsyncsend;
2431 						pthread_mutex_unlock(&fdlock);
2432 						master_stats_set(MASTER_PING,usecping);
2433 					} else {
2434 						pthread_mutex_unlock(&fdlock);
2435 						syslog(LOG_NOTICE,"negative packet travel time between client and master - ignoring in time sync");
2436 						usecping = 0;
2437 					}
2438 					if (usecping>100000) { // ignore too high differences
2439 						syslog(LOG_NOTICE,"high packet travel time between client and master (%u.%06us) - ignoring in time sync",(unsigned int)(usecping/1000000),(unsigned int)(usecping%1000000));
2440 						usecping = 0;
2441 					}
2442 					rusectime = get64bit(&ptr);
2443 					rusectime += usecping/2;
2444 					// usectime here should have master's wall clock
2445 					fs_amtime_reference_clock(usec,rusectime);
2446 					gettimeofday(&tv,NULL);
2447 					lusectime = tv.tv_sec;
2448 					lusectime *= 1000000;
2449 					lusectime += tv.tv_usec;
2450 					if (rusectime + 1000000 < lusectime || lusectime + 1000000 < rusectime) {
2451 						syslog(LOG_WARNING,"time desync between client and master is higher than a second - it might lead to strange atime/mtime behaviour - consider time synchronization in your moosefs cluster");
2452 					}
2453 					if (rusectime > lusectime) {
2454 						master_stats_set(MASTER_TIMEDIFF,rusectime-lusectime);
2455 					} else {
2456 						master_stats_set(MASTER_TIMEDIFF,lusectime-rusectime);
2457 					}
2458 #ifdef MFSDEBUG
2459 					syslog(LOG_NOTICE,"ping time: %u.%06u ; remote time: %u.%06u ; local time: %u.%06u ; monotonic time: %u.%06u",(unsigned int)(usecping/1000000),(unsigned int)(usecping%1000000),(unsigned int)(rusectime/1000000),(unsigned int)(rusectime%1000000),(unsigned int)(lusectime/1000000),(unsigned int)(lusectime%1000000),(unsigned int)(usec/1000000),(unsigned int)(usec%1000000));
2460 #endif
2461 					continue;
2462 				}
2463 				if (cmd==MATOCL_FUSE_INVALIDATE_CHUNK_CACHE) {
2464 					chunksdatacache_cleanup();
2465 					continue;
2466 				}
2467 			}
2468 		}
2469 		rec = fs_get_threc_by_id(packetid);
2470 		if (rec==NULL) {
2471 			syslog(LOG_WARNING,"master: got unexpected queryid (%"PRIu32" ; cmd:%"PRIu32" ; size:%"PRIu32")",packetid,cmd,size+4);
2472 			fs_disconnect();
2473 			continue;
2474 		}
2475 		pthread_mutex_lock(&(rec->mutex));	// make helgrind happy
2476 		if (rec->receiving) {
2477 			pthread_mutex_unlock(&(rec->mutex));
2478 			fs_disconnect();
2479 			continue;
2480 		}
2481 		fs_input_buffer_init(rec,size);
2482 		if (rec->ibuff==NULL) {
2483 			pthread_mutex_unlock(&(rec->mutex));
2484 			fs_disconnect();
2485 			continue;
2486 		}
2487 		rec->receiving = 1;
2488 		pthread_mutex_unlock(&(rec->mutex));
2489 		// syslog(LOG_NOTICE,"master: expected data size: %"PRIu32,size);
2490 		rcvd = 0;
2491 		while (size-rcvd>0) {
2492 			toread = size-rcvd;
2493 			if (toread>65536) {
2494 				toread = 65536;
2495 			}
2496 			r = tcptoread(fd,rec->ibuff+rcvd,toread,RECEIVE_TIMEOUT*1000);
2497 			// syslog(LOG_NOTICE,"master: data size: %d",r);
2498 			if (r==0) {
2499 				syslog(LOG_WARNING,"master: connection lost (data)");
2500 				break;
2501 			}
2502 			if (r!=(int32_t)toread) {
2503 				syslog(LOG_WARNING,"master: tcp recv error: %s (data)",strerr(errno));
2504 				break;
2505 			}
2506 			master_stats_add(MASTER_BYTESRCVD,toread);
2507 			rcvd += toread;
2508 		}
2509 		if (size-rcvd>0) { // exit from previous loop by break = error
2510 			pthread_mutex_lock(&(rec->mutex));
2511 			rec->receiving = 0;
2512 			pthread_mutex_unlock(&(rec->mutex));
2513 			fs_disconnect();
2514 			continue;
2515 		}
2516 		pthread_mutex_lock(&(rec->mutex));
2517 		rec->sent = 0;
2518 		rec->status = 0;
2519 		rec->idataleng = size;
2520 		rec->rcvd_cmd = cmd;
2521 		rec->receiving = 0;
2522 		// syslog(LOG_NOTICE,"master: unlock: %"PRIu32,rec->packetid);
2523 		rec->rcvd = 1;
2524 //		if (rec->waiting) {
2525 		pthread_cond_signal(&(rec->cond));
2526 //		}
2527 		pthread_mutex_unlock(&(rec->mutex));
2528 	}
2529 }
2530 
2531 // called before fork
2532 int fs_init_master_connection(const char *bindhostname,const char *masterhostname,const char *masterportname,uint8_t meta,const char *info,const char *subfolder,const uint8_t passworddigest[16],uint8_t donotrememberpassword,uint8_t bgregister) {
2533 	master_statsptr_init();
2534 
2535 	fd = -1;
2536 	sessionlost = bgregister?1:0;
2537 	lastsyncsend = 0;
2538 	sessionid = 0;
2539 	metaid = 0;
2540 #ifdef HAVE___SYNC_FETCH_AND_OP
2541 	(void)__sync_fetch_and_and(&disconnect,0);
2542 #else
2543 	disconnect = 0;
2544 #endif
2545 	donotsendsustainedinodes = 0;
2546 
2547 	if (bindhostname) {
2548 		connect_args.bindhostname = strdup(bindhostname);
2549 	} else {
2550 		connect_args.bindhostname = NULL;
2551 	}
2552 	connect_args.masterhostname = strdup(masterhostname);
2553 	connect_args.masterportname = strdup(masterportname);
2554 	connect_args.meta = meta;
2555 	connect_args.clearpassword = donotrememberpassword;
2556 	connect_args.info = strdup(info);
2557 	connect_args.subfolder = strdup(subfolder);
2558 	if (passworddigest==NULL) {
2559 		connect_args.passworddigest = NULL;
2560 	} else {
2561 		connect_args.passworddigest = malloc(16);
2562 		memcpy(connect_args.passworddigest,passworddigest,16);
2563 	}
2564 
2565 	if (bgregister) {
2566 		return 1;
2567 	}
2568 	return fs_connect(1,&connect_args);
2569 }
2570 
2571 // called after fork
2572 void fs_init_threads(uint32_t retries,uint32_t timeout) {
2573 	uint32_t i;
2574 	pthread_attr_t thattr;
2575 	struct timeval tv;
2576 	uint64_t usectime;
2577 	maxretries = retries;
2578 	usectimeout = timeout;
2579 	usectimeout *= 1000000; // sec -> usec
2580 	fterm = 0;
2581 	ep_init();
2582 	for (i=0 ; i<AMTIME_HASH_SIZE ; i++) {
2583 		amtime_hash[i] = NULL;
2584 	}
2585 	for (i=0 ; i<ACQFILES_HASH_SIZE ; i++) {
2586 		af_hash[i] = NULL;
2587 	}
2588 	af_lruhead = NULL;
2589 	af_lrutail = &(af_lruhead);
2590 	af_lru_cnt = 0;
2591 	gettimeofday(&tv,NULL);
2592 	usectime = tv.tv_sec;
2593 	usectime *= 1000000;
2594 	usectime += tv.tv_usec;
2595 	timediffusec = usectime - monotonic_useconds(); // before receiving packets from master start with own wall clock
2596 	zassert(pthread_key_create(&reckey,fs_free_threc));
2597 	zassert(pthread_mutex_init(&reclock,NULL));
2598 	zassert(pthread_mutex_init(&fdlock,NULL));
2599 	zassert(pthread_mutex_init(&aflock,NULL));
2600 	zassert(pthread_mutex_init(&amtimelock,NULL));
2601 	zassert(pthread_attr_init(&thattr));
2602 	zassert(pthread_attr_setstacksize(&thattr,0x100000));
2603 	zassert(pthread_create(&rpthid,&thattr,fs_receive_thread,NULL));
2604 	zassert(pthread_create(&npthid,&thattr,fs_nop_thread,NULL));
2605 	zassert(pthread_attr_destroy(&thattr));
2606 }
2607 
2608 void fs_term(void) {
2609 	threc *rec,*recn;
2610 	uint32_t i;
2611 	uint32_t rechash;
2612 	amtime_file *amf,*amfn;
2613 	acquired_file *af,*afn;
2614 
2615 	zassert(pthread_mutex_lock(&fdlock));
2616 	fterm = 1;
2617 	zassert(pthread_mutex_unlock(&fdlock));
2618 	zassert(pthread_join(npthid,NULL));
2619 	zassert(pthread_join(rpthid,NULL));
2620 	zassert(pthread_mutex_destroy(&amtimelock));
2621 	zassert(pthread_mutex_destroy(&aflock));
2622 	zassert(pthread_mutex_destroy(&fdlock));
2623 	zassert(pthread_mutex_lock(&reclock));
2624 	for (rechash=0 ; rechash<THRECHASHSIZE ; rechash++) {
2625 		for (rec = threchash[rechash] ; rec!=NULL ; rec=recn) {
2626 			syslog(LOG_WARNING,"thread specific memory (id:%"PRIu32") hasn't been freed",rec->packetid);
2627 			recn = rec->next;
2628 			if (rec->obuff) {
2629 #ifdef MMAP_ALLOC
2630 				munmap((void*)(rec->obuff),rec->obuffsize);
2631 #else
2632 				free(rec->obuff);
2633 #endif
2634 			}
2635 			if (rec->ibuff) {
2636 #ifdef MMAP_ALLOC
2637 				munmap((void*)(rec->ibuff),rec->ibuffsize);
2638 #else
2639 				free(rec->ibuff);
2640 #endif
2641 			}
2642 			pthread_mutex_destroy(&(rec->mutex));
2643 			pthread_cond_destroy(&(rec->cond));
2644 			free(rec);
2645 		}
2646 	}
2647 	for (rec = threcfree ; rec ; rec = recn) {
2648 		recn = rec->next;
2649 		if (rec->obuff) {
2650 #ifdef MMAP_ALLOC
2651 			munmap((void*)(rec->obuff),rec->obuffsize);
2652 #else
2653 			free(rec->obuff);
2654 #endif
2655 		}
2656 		if (rec->ibuff) {
2657 #ifdef MMAP_ALLOC
2658 			munmap((void*)(rec->ibuff),rec->ibuffsize);
2659 #else
2660 			free(rec->ibuff);
2661 #endif
2662 		}
2663 		pthread_mutex_destroy(&(rec->mutex));
2664 		pthread_cond_destroy(&(rec->cond));
2665 		free(rec);
2666 	}
2667 	zassert(pthread_mutex_unlock(&reclock));
2668 	zassert(pthread_mutex_destroy(&reclock));
2669 	zassert(pthread_key_delete(reckey));
2670 	for (i=0 ; i<ACQFILES_HASH_SIZE ; i++) {
2671 		for (af = af_hash[i] ; af ; af = afn) {
2672 			afn = af->next;
2673 			free(af);
2674 		}
2675 	}
2676 	for (i=0 ; i<AMTIME_HASH_SIZE ; i++) {
2677 		for (amf = amtime_hash[i] ; amf ; amf = amfn) {
2678 			amfn = amf->next;
2679 			free(amfn);
2680 		}
2681 	}
2682 	if (fd>=0) {
2683 		tcpclose(fd);
2684 	}
2685 	if (connect_args.bindhostname) {
2686 		free(connect_args.bindhostname);
2687 	}
2688 	free(connect_args.masterhostname);
2689 	free(connect_args.masterportname);
2690 	free(connect_args.info);
2691 	free(connect_args.subfolder);
2692 	if (connect_args.passworddigest) {
2693 		free(connect_args.passworddigest);
2694 	}
2695 	heap_term();
2696 	ep_term();
2697 }
2698 
2699 uint8_t fs_get_cfg(const char *opt_name,char opt_value[256]) {
2700 	uint8_t *wptr;
2701 	const uint8_t *rptr;
2702 	uint32_t i;
2703 	uint32_t nleng;
2704 	threc *rec = fs_get_my_threc();
2705 
2706 	nleng = strlen(opt_name);
2707 	if (nleng>255) {
2708 		return MFS_ERROR_EINVAL;
2709 	}
2710 	wptr = fs_createpacket(rec,ANTOAN_GET_CONFIG,1+nleng);
2711 	if (wptr==NULL) {
2712 		return MFS_ERROR_IO;
2713 	}
2714 	put8bit(&wptr,nleng);
2715 	memcpy(wptr,opt_name,nleng);
2716 	rptr = fs_sendandreceive(rec,ANTOAN_CONFIG_VALUE,&i);
2717 	if (rptr==NULL) {
2718 		return MFS_ERROR_IO;
2719 	} else if (i==0 || i>255) {
2720 		fs_disconnect();
2721 		return MFS_ERROR_IO;
2722 	} else {
2723 		nleng = get8bit(&rptr);
2724 		if (i!=(1U+nleng)) {
2725 			fs_disconnect();
2726 			return MFS_ERROR_IO;
2727 		}
2728 		memcpy(opt_value,rptr,nleng);
2729 		opt_value[nleng]=0;
2730 		return MFS_STATUS_OK;
2731 	}
2732 }
2733 
2734 void fs_statfs(uint64_t *totalspace,uint64_t *availspace,uint64_t *freespace,uint64_t *trashspace,uint64_t *sustainedspace,uint32_t *inodes) {
2735 	uint8_t *wptr;
2736 	const uint8_t *rptr;
2737 	uint32_t i;
2738 	threc *rec = fs_get_my_threc();
2739 	wptr = fs_createpacket(rec,CLTOMA_FUSE_STATFS,0);
2740 	if (wptr==NULL) {
2741 		*totalspace = 0;
2742 		*availspace = 0;
2743 		*freespace = 0;
2744 		*trashspace = 0;
2745 		*sustainedspace = 0;
2746 		*inodes = 0;
2747 		return;
2748 	}
2749 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_STATFS,&i);
2750 	if (rptr==NULL || (i!=36 && i!=44)) {
2751 		*totalspace = 0;
2752 		*availspace = 0;
2753 		*freespace = 0;
2754 		*trashspace = 0;
2755 		*sustainedspace = 0;
2756 		*inodes = 0;
2757 	} else {
2758 		*totalspace = get64bit(&rptr);
2759 		*availspace = get64bit(&rptr);
2760 		if (i==44) {
2761 			*freespace = get64bit(&rptr);
2762 		} else {
2763 			*freespace = *availspace;
2764 		}
2765 		*trashspace = get64bit(&rptr);
2766 		*sustainedspace = get64bit(&rptr);
2767 		*inodes = get32bit(&rptr);
2768 	}
2769 }
2770 
2771 uint8_t fs_access(uint32_t inode,uint32_t uid,uint32_t gids,uint32_t *gid,uint16_t modemask) {
2772 	uint8_t *wptr;
2773 	const uint8_t *rptr;
2774 	uint32_t i;
2775 	uint8_t ret;
2776 	threc *rec = fs_get_my_threc();
2777 	if (master_version()<VERSION2INT(2,0,0) || gids==0) {
2778 		wptr = fs_createpacket(rec,CLTOMA_FUSE_ACCESS,13);
2779 		if (wptr==NULL) {
2780 			return MFS_ERROR_IO;
2781 		}
2782 		put32bit(&wptr,inode);
2783 		put32bit(&wptr,uid);
2784 		if (gids>0) {
2785 			put32bit(&wptr,gid[0]);
2786 		} else {
2787 			put32bit(&wptr,0xFFFFFFFF);
2788 		}
2789 		put8bit(&wptr,modemask);
2790 	} else {
2791 		wptr = fs_createpacket(rec,CLTOMA_FUSE_ACCESS,14+4*gids);
2792 		if (wptr==NULL) {
2793 			return MFS_ERROR_IO;
2794 		}
2795 		put32bit(&wptr,inode);
2796 		put32bit(&wptr,uid);
2797 		put32bit(&wptr,gids);
2798 		for (i=0 ; i<gids ; i++) {
2799 			put32bit(&wptr,gid[i]);
2800 		}
2801 		put16bit(&wptr,modemask);
2802 	}
2803 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_ACCESS,&i);
2804 	if (!rptr || i!=1) {
2805 		ret = MFS_ERROR_IO;
2806 	} else {
2807 		ret = rptr[0];
2808 	}
2809 	return ret;
2810 }
2811 
2812 uint8_t fs_simple_lookup(uint32_t parent,uint8_t nleng,const uint8_t *name,uint32_t uid,uint32_t gids,uint32_t *gid,uint32_t *inode,uint8_t attr[ATTR_RECORD_SIZE]) {
2813 	uint8_t *wptr;
2814 	const uint8_t *rptr;
2815 	uint32_t i;
2816 	uint8_t ret;
2817 	uint8_t packetver;
2818 	threc *rec = fs_get_my_threc();
2819 	uint8_t asize = master_attrsize();
2820 
2821 	if (master_version()<VERSION2INT(2,0,0)) {
2822 		wptr = fs_createpacket(rec,CLTOMA_FUSE_LOOKUP,13+nleng);
2823 		packetver = 0;
2824 	} else {
2825 		wptr = fs_createpacket(rec,CLTOMA_FUSE_LOOKUP,13+4*gids+nleng);
2826 		packetver = 1;
2827 	}
2828 	if (wptr==NULL) {
2829 		return MFS_ERROR_IO;
2830 	}
2831 	put32bit(&wptr,parent);
2832 	put8bit(&wptr,nleng);
2833 	memcpy(wptr,name,nleng);
2834 	wptr+=nleng;
2835 	put32bit(&wptr,uid);
2836 	if (packetver==0) {
2837 		if (gids>0) {
2838 			put32bit(&wptr,gid[0]);
2839 		} else {
2840 			put32bit(&wptr,0xFFFFFFFF);
2841 		}
2842 	} else {
2843 		if (gids>0) {
2844 			put32bit(&wptr,gids);
2845 			for (i=0 ; i<gids ; i++) {
2846 				put32bit(&wptr,gid[i]);
2847 			}
2848 		} else {
2849 			put32bit(&wptr,0xFFFFFFFF);
2850 		}
2851 	}
2852 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_LOOKUP,&i);
2853 	if (rptr==NULL) {
2854 		ret = MFS_ERROR_IO;
2855 	} else if (i==1) {
2856 		ret = rptr[0];
2857 	} else if (i==(uint32_t)(4+asize) || i>=(uint32_t)(6+asize)) {
2858 		*inode = get32bit(&rptr);
2859 		copy_attr(rptr,attr,asize);
2860 		ret = MFS_STATUS_OK;
2861 	} else {
2862 		fs_disconnect();
2863 		ret = MFS_ERROR_IO;
2864 	}
2865 	return ret;
2866 }
2867 
2868 uint8_t fs_lookup(uint32_t parent,uint8_t nleng,const uint8_t *name,uint32_t uid,uint32_t gids,uint32_t *gid,uint32_t *inode,uint8_t attr[ATTR_RECORD_SIZE],uint16_t *lflags,uint8_t *csdataver,uint64_t *chunkid,uint32_t *version,const uint8_t **csdata,uint32_t *csdatasize) {
2869 	uint8_t *wptr;
2870 	const uint8_t *rptr;
2871 	uint32_t i;
2872 	uint8_t ret;
2873 	uint8_t packetver;
2874 	threc *rec = fs_get_my_threc();
2875 	uint8_t asize = master_attrsize();
2876 
2877 	if (master_version()<VERSION2INT(2,0,0)) {
2878 		wptr = fs_createpacket(rec,CLTOMA_FUSE_LOOKUP,13+nleng);
2879 		packetver = 0;
2880 	} else {
2881 		wptr = fs_createpacket(rec,CLTOMA_FUSE_LOOKUP,13+4*gids+nleng);
2882 		packetver = 1;
2883 	}
2884 	if (wptr==NULL) {
2885 		return MFS_ERROR_IO;
2886 	}
2887 	put32bit(&wptr,parent);
2888 	put8bit(&wptr,nleng);
2889 	memcpy(wptr,name,nleng);
2890 	wptr+=nleng;
2891 	put32bit(&wptr,uid);
2892 	if (packetver==0) {
2893 		if (gids>0) {
2894 			put32bit(&wptr,gid[0]);
2895 		} else {
2896 			put32bit(&wptr,0xFFFFFFFF);
2897 		}
2898 	} else {
2899 		if (gids>0) {
2900 			put32bit(&wptr,gids);
2901 			for (i=0 ; i<gids ; i++) {
2902 				put32bit(&wptr,gid[i]);
2903 			}
2904 		} else {
2905 			put32bit(&wptr,0xFFFFFFFF);
2906 		}
2907 	}
2908 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_LOOKUP,&i);
2909 	if (rptr==NULL) {
2910 		ret = MFS_ERROR_IO;
2911 	} else if (i==1) {
2912 		ret = rptr[0];
2913 	} else if (i==(uint32_t)(4+asize)) {
2914 		*inode = get32bit(&rptr);
2915 		copy_attr(rptr,attr,asize);
2916 		*lflags = 0xFFFF;
2917 		ret = MFS_STATUS_OK;
2918 	} else if (i>=(uint32_t)(6+asize)) {
2919 		*inode = get32bit(&rptr);
2920 		copy_attr(rptr,attr,asize);
2921 		rptr+=asize;
2922 		*lflags = get16bit(&rptr);
2923 		ret = MFS_STATUS_OK;
2924 		if ((*lflags) & LOOKUP_CHUNK_ZERO_DATA) {
2925 			if (i>=(uint32_t)(19+asize)) {
2926 				*csdataver = get8bit(&rptr);
2927 				*chunkid = get64bit(&rptr);
2928 				*version = get32bit(&rptr);
2929 				*csdata = rptr;
2930 				*csdatasize = i-(19+asize);
2931 				if ((*csdataver)!=2 || ((i-(19+asize))%14)!=0) {
2932 					ret = MFS_ERROR_IO;
2933 				}
2934 			} else {
2935 				ret = MFS_ERROR_IO;
2936 			}
2937 		} else {
2938 			*csdataver = 0;
2939 			*chunkid = 0;
2940 			*version = 0;
2941 			*csdata = NULL;
2942 			*csdatasize = 0;
2943 		}
2944 		if (ret == MFS_ERROR_IO) {
2945 			fs_disconnect();
2946 		}
2947 	} else {
2948 		fs_disconnect();
2949 		ret = MFS_ERROR_IO;
2950 	}
2951 	return ret;
2952 }
2953 
2954 uint8_t fs_getattr(uint32_t inode,uint8_t opened,uint32_t uid,uint32_t gid,uint8_t attr[ATTR_RECORD_SIZE]) {
2955 	uint8_t *wptr;
2956 	const uint8_t *rptr;
2957 	uint32_t i;
2958 	uint8_t ret;
2959 	uint8_t packetver;
2960 	threc *rec = fs_get_my_threc();
2961 	uint8_t asize = master_attrsize();
2962 
2963 	if (master_version()<VERSION2INT(1,6,28)) {
2964 		wptr = fs_createpacket(rec,CLTOMA_FUSE_GETATTR,12);
2965 		packetver = 0;
2966 	} else {
2967 		wptr = fs_createpacket(rec,CLTOMA_FUSE_GETATTR,13);
2968 		packetver = 1;
2969 	}
2970 	if (wptr==NULL) {
2971 		return MFS_ERROR_IO;
2972 	}
2973 	put32bit(&wptr,inode);
2974 	if (packetver>=1) {
2975 		put8bit(&wptr,opened);
2976 	}
2977 	put32bit(&wptr,uid);
2978 	put32bit(&wptr,gid);
2979 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_GETATTR,&i);
2980 	if (rptr==NULL) {
2981 		ret = MFS_ERROR_IO;
2982 	} else if (i==1) {
2983 		ret = rptr[0];
2984 	} else if (i!=asize) {
2985 		fs_disconnect();
2986 		ret = MFS_ERROR_IO;
2987 	} else {
2988 		copy_attr(rptr,attr,asize);
2989 		ret = MFS_STATUS_OK;
2990 	}
2991 	return ret;
2992 }
2993 
2994 uint8_t fs_setattr(uint32_t inode,uint8_t opened,uint32_t uid,uint32_t gids,uint32_t *gid,uint8_t setmask,uint16_t attrmode,uint32_t attruid,uint32_t attrgid,uint32_t attratime,uint32_t attrmtime,uint8_t winattr,uint8_t sugidclearmode,uint8_t attr[ATTR_RECORD_SIZE]) {
2995 	uint8_t *wptr;
2996 	const uint8_t *rptr;
2997 	uint32_t i;
2998 	uint8_t ret;
2999 	uint8_t packetver;
3000 	threc *rec = fs_get_my_threc();
3001 	uint8_t asize = master_attrsize();
3002 	uint32_t mv = master_version();
3003 
3004 	if (mv<VERSION2INT(1,6,25)) {
3005 		wptr = fs_createpacket(rec,CLTOMA_FUSE_SETATTR,31);
3006 		packetver = 0;
3007 	} else if (mv<VERSION2INT(1,6,28)) {
3008 		wptr = fs_createpacket(rec,CLTOMA_FUSE_SETATTR,32);
3009 		packetver = 1;
3010 	} else if (mv<VERSION2INT(2,0,0)) {
3011 		wptr = fs_createpacket(rec,CLTOMA_FUSE_SETATTR,33);
3012 		packetver = 2;
3013 	} else if (mv<VERSION2INT(3,0,93)) {
3014 		wptr = fs_createpacket(rec,CLTOMA_FUSE_SETATTR,33+gids*4);
3015 		packetver = 3;
3016 	} else {
3017 		wptr = fs_createpacket(rec,CLTOMA_FUSE_SETATTR,34+gids*4);
3018 		packetver = 4;
3019 	}
3020 	if (wptr==NULL) {
3021 		return MFS_ERROR_IO;
3022 	}
3023 	put32bit(&wptr,inode);
3024 	if (packetver>=2) {
3025 		put8bit(&wptr,opened);
3026 	}
3027 	put32bit(&wptr,uid);
3028 	if (packetver<=2) {
3029 		if (gids>0) {
3030 			put32bit(&wptr,gid[0]);
3031 		} else {
3032 			put32bit(&wptr,0xFFFFFFFF);
3033 		}
3034 	} else {
3035 		if (gids>0) {
3036 			put32bit(&wptr,gids);
3037 			for (i=0 ; i<gids ; i++) {
3038 				put32bit(&wptr,gid[i]);
3039 			}
3040 		} else {
3041 			put32bit(&wptr,0xFFFFFFFF);
3042 		}
3043 	}
3044 	put8bit(&wptr,setmask);
3045 	put16bit(&wptr,attrmode);
3046 	put32bit(&wptr,attruid);
3047 	put32bit(&wptr,attrgid);
3048 	put32bit(&wptr,attratime);
3049 	put32bit(&wptr,attrmtime);
3050 	if (packetver>=4) {
3051 		put8bit(&wptr,winattr);
3052 	}
3053 	if (packetver>=1) {
3054 		put8bit(&wptr,sugidclearmode);
3055 	}
3056 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_SETATTR,&i);
3057 	if (rptr==NULL) {
3058 		ret = MFS_ERROR_IO;
3059 	} else if (i==1) {
3060 		ret = rptr[0];
3061 	} else if (i!=asize) {
3062 		fs_disconnect();
3063 		ret = MFS_ERROR_IO;
3064 	} else {
3065 		copy_attr(rptr,attr,asize);
3066 		ret = MFS_STATUS_OK;
3067 	}
3068 	return ret;
3069 }
3070 
3071 uint8_t fs_truncate(uint32_t inode,uint8_t flags,uint32_t uid,uint32_t gids,uint32_t *gid,uint64_t attrlength,uint8_t attr[ATTR_RECORD_SIZE],uint64_t *prevlength) {
3072 	uint8_t *wptr;
3073 	const uint8_t *rptr;
3074 	uint32_t i;
3075 	uint8_t ret;
3076 	uint8_t packetver;
3077 	threc *rec = fs_get_my_threc();
3078 	uint8_t asize = master_attrsize();
3079 
3080 	if (master_version()<VERSION2INT(2,0,0)) {
3081 		wptr = fs_createpacket(rec,CLTOMA_FUSE_TRUNCATE,21);
3082 		packetver = 0;
3083 	} else {
3084 		wptr = fs_createpacket(rec,CLTOMA_FUSE_TRUNCATE,21+4*gids);
3085 		packetver = 1;
3086 	}
3087 	if (wptr==NULL) {
3088 		return MFS_ERROR_IO;
3089 	}
3090 	put32bit(&wptr,inode);
3091 	put8bit(&wptr,flags);
3092 	put32bit(&wptr,uid);
3093 	if (packetver==0) {
3094 		if (gids>0) {
3095 			put32bit(&wptr,gid[0]);
3096 		} else {
3097 			put32bit(&wptr,0xFFFFFFFF);
3098 		}
3099 	} else {
3100 		if (gids>0) {
3101 			put32bit(&wptr,gids);
3102 			for (i=0 ; i<gids ; i++) {
3103 				put32bit(&wptr,gid[i]);
3104 			}
3105 		} else {
3106 			put32bit(&wptr,0xFFFFFFFF);
3107 		}
3108 	}
3109 	put64bit(&wptr,attrlength);
3110 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_TRUNCATE,&i);
3111 	if (rptr==NULL) {
3112 		ret = MFS_ERROR_IO;
3113 	} else if (i==1) {
3114 		ret = rptr[0];
3115 	} else if (i==asize) {
3116 		if (attr!=NULL) {
3117 			copy_attr(rptr,attr,asize);
3118 		}
3119 		ret = MFS_STATUS_OK;
3120 	} else if (i==(uint32_t)(asize+8)) {
3121 		if (prevlength!=NULL) {
3122 			*prevlength = get64bit(&rptr);
3123 		} else {
3124 			rptr+=8;
3125 		}
3126 		if (attr!=NULL) {
3127 			copy_attr(rptr,attr,asize);
3128 		}
3129 		ret = MFS_STATUS_OK;
3130 	} else {
3131 		fs_disconnect();
3132 		ret = MFS_ERROR_IO;
3133 	}
3134 	return ret;
3135 }
3136 
3137 uint8_t fs_readlink(uint32_t inode,const uint8_t **path) {
3138 	uint8_t *wptr;
3139 	const uint8_t *rptr;
3140 	uint32_t i;
3141 	uint32_t pleng;
3142 	uint8_t ret;
3143 	threc *rec = fs_get_my_threc();
3144 	wptr = fs_createpacket(rec,CLTOMA_FUSE_READLINK,4);
3145 	if (wptr==NULL) {
3146 		return MFS_ERROR_IO;
3147 	}
3148 	put32bit(&wptr,inode);
3149 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_READLINK,&i);
3150 	if (rptr==NULL) {
3151 		ret = MFS_ERROR_IO;
3152 	} else if (i==1) {
3153 		ret = rptr[0];
3154 	} else if (i<4) {
3155 		fs_disconnect();
3156 		ret = MFS_ERROR_IO;
3157 	} else {
3158 		pleng = get32bit(&rptr);
3159 		if (i!=4+pleng || pleng==0 || rptr[pleng-1]!=0) {
3160 			fs_disconnect();
3161 			ret = MFS_ERROR_IO;
3162 		} else {
3163 			*path = rptr;
3164 			//*path = malloc(pleng);
3165 			//memcpy(*path,ptr,pleng);
3166 			ret = MFS_STATUS_OK;
3167 		}
3168 	}
3169 	return ret;
3170 }
3171 
3172 uint8_t fs_symlink(uint32_t parent,uint8_t nleng,const uint8_t *name,const uint8_t *path,uint32_t uid,uint32_t gids,uint32_t *gid,uint32_t *inode,uint8_t attr[ATTR_RECORD_SIZE]) {
3173 	uint8_t *wptr;
3174 	const uint8_t *rptr;
3175 	uint32_t i;
3176 	uint32_t pleng;
3177 	uint8_t ret;
3178 	uint8_t packetver;
3179 	threc *rec = fs_get_my_threc();
3180 	uint8_t asize = master_attrsize();
3181 
3182 	pleng = strlen((const char *)path)+1;
3183 	if (master_version()<VERSION2INT(2,0,0)) {
3184 		wptr = fs_createpacket(rec,CLTOMA_FUSE_SYMLINK,pleng+nleng+17);
3185 		packetver = 0;
3186 	} else {
3187 		wptr = fs_createpacket(rec,CLTOMA_FUSE_SYMLINK,pleng+nleng+17+4*gids);
3188 		packetver = 1;
3189 	}
3190 	if (wptr==NULL) {
3191 		return MFS_ERROR_IO;
3192 	}
3193 	put32bit(&wptr,parent);
3194 	put8bit(&wptr,nleng);
3195 	memcpy(wptr,name,nleng);
3196 	wptr+=nleng;
3197 	put32bit(&wptr,pleng);
3198 	memcpy(wptr,path,pleng);
3199 	wptr+=pleng;
3200 	put32bit(&wptr,uid);
3201 	if (packetver==0) {
3202 		if (gids>0) {
3203 			put32bit(&wptr,gid[0]);
3204 		} else {
3205 			put32bit(&wptr,0xFFFFFFFF);
3206 		}
3207 	} else {
3208 		if (gids>0) {
3209 			put32bit(&wptr,gids);
3210 			for (i=0 ; i<gids ; i++) {
3211 				put32bit(&wptr,gid[i]);
3212 			}
3213 		} else {
3214 			put32bit(&wptr,0xFFFFFFFF);
3215 		}
3216 	}
3217 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_SYMLINK,&i);
3218 	if (rptr==NULL) {
3219 		ret = MFS_ERROR_IO;
3220 	} else if (i==1) {
3221 		ret = rptr[0];
3222 	} else if (i!=(uint32_t)(4+asize)) {
3223 		fs_disconnect();
3224 		ret = MFS_ERROR_IO;
3225 	} else {
3226 		*inode = get32bit(&rptr);
3227 		copy_attr(rptr,attr,asize);
3228 		ret = MFS_STATUS_OK;
3229 	}
3230 	return ret;
3231 }
3232 
3233 static inline uint8_t fsnodes_type_back_convert(uint8_t type) {
3234 	switch (type) {
3235 		case TYPE_FILE:
3236 			return DISP_TYPE_FILE;
3237 		case TYPE_DIRECTORY:
3238 			return DISP_TYPE_DIRECTORY;
3239 		case TYPE_SYMLINK:
3240 			return DISP_TYPE_SYMLINK;
3241 		case TYPE_FIFO:
3242 			return DISP_TYPE_FIFO;
3243 		case TYPE_BLOCKDEV:
3244 			return DISP_TYPE_BLOCKDEV;
3245 		case TYPE_CHARDEV:
3246 			return DISP_TYPE_CHARDEV;
3247 		case TYPE_SOCKET:
3248 			return DISP_TYPE_SOCKET;
3249 		case TYPE_TRASH:
3250 			return DISP_TYPE_TRASH;
3251 		case TYPE_SUSTAINED:
3252 			return DISP_TYPE_SUSTAINED;
3253 	}
3254 	return type;
3255 }
3256 
3257 uint8_t fs_mknod(uint32_t parent,uint8_t nleng,const uint8_t *name,uint8_t type,uint16_t mode,uint16_t cumask,uint32_t uid,uint32_t gids,uint32_t *gid,uint32_t rdev,uint32_t *inode,uint8_t attr[ATTR_RECORD_SIZE]) {
3258 	uint8_t *wptr;
3259 	const uint8_t *rptr;
3260 	uint32_t i;
3261 	uint8_t ret;
3262 	uint8_t packetver;
3263 	threc *rec = fs_get_my_threc();
3264 	uint8_t asize = master_attrsize();
3265 
3266 	if (master_version()<VERSION2INT(2,0,0)) {
3267 		mode &= ~cumask;
3268 		wptr = fs_createpacket(rec,CLTOMA_FUSE_MKNOD,20+nleng);
3269 		packetver = 0;
3270 	} else {
3271 		wptr = fs_createpacket(rec,CLTOMA_FUSE_MKNOD,22+nleng+4*gids);
3272 		packetver = 1;
3273 	}
3274 	if (wptr==NULL) {
3275 		return MFS_ERROR_IO;
3276 	}
3277 	put32bit(&wptr,parent);
3278 	put8bit(&wptr,nleng);
3279 	memcpy(wptr,name,nleng);
3280 	wptr+=nleng;
3281 	if (master_version()<VERSION2INT(1,7,32)) {
3282 		type = fsnodes_type_back_convert(type);
3283 	}
3284 	put8bit(&wptr,type);
3285 	put16bit(&wptr,mode);
3286 	if (packetver>=1) {
3287 		put16bit(&wptr,cumask);
3288 	}
3289 	put32bit(&wptr,uid);
3290 	if (packetver==0) {
3291 		if (gids>0) {
3292 			put32bit(&wptr,gid[0]);
3293 		} else {
3294 			put32bit(&wptr,0xFFFFFFFF);
3295 		}
3296 	} else {
3297 		if (gids>0) {
3298 			put32bit(&wptr,gids);
3299 			for (i=0 ; i<gids ; i++) {
3300 				put32bit(&wptr,gid[i]);
3301 			}
3302 		} else {
3303 			put32bit(&wptr,0xFFFFFFFF);
3304 		}
3305 	}
3306 	put32bit(&wptr,rdev);
3307 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_MKNOD,&i);
3308 	if (rptr==NULL) {
3309 		ret = MFS_ERROR_IO;
3310 	} else if (i==1) {
3311 		ret = rptr[0];
3312 	} else if (i!=(uint32_t)(4+asize)) {
3313 		fs_disconnect();
3314 		ret = MFS_ERROR_IO;
3315 	} else {
3316 		*inode = get32bit(&rptr);
3317 		copy_attr(rptr,attr,asize);
3318 		ret = MFS_STATUS_OK;
3319 	}
3320 	return ret;
3321 }
3322 
3323 uint8_t fs_mkdir(uint32_t parent,uint8_t nleng,const uint8_t *name,uint16_t mode,uint16_t cumask,uint32_t uid,uint32_t gids,uint32_t *gid,uint8_t copysgid,uint32_t *inode,uint8_t attr[ATTR_RECORD_SIZE]) {
3324 	uint8_t *wptr;
3325 	const uint8_t *rptr;
3326 	uint32_t i;
3327 	uint8_t ret;
3328 	uint8_t packetver;
3329 	threc *rec = fs_get_my_threc();
3330 	uint8_t asize = master_attrsize();
3331 
3332 	if (master_version()<VERSION2INT(1,6,25)) {
3333 		mode &= ~cumask;
3334 		wptr = fs_createpacket(rec,CLTOMA_FUSE_MKDIR,15+nleng);
3335 		packetver = 0;
3336 	} else if (master_version()<VERSION2INT(2,0,0)) {
3337 		mode &= ~cumask;
3338 		wptr = fs_createpacket(rec,CLTOMA_FUSE_MKDIR,16+nleng);
3339 		packetver = 1;
3340 	} else {
3341 		wptr = fs_createpacket(rec,CLTOMA_FUSE_MKDIR,18+nleng+4*gids);
3342 		packetver = 2;
3343 	}
3344 	if (wptr==NULL) {
3345 		return MFS_ERROR_IO;
3346 	}
3347 	put32bit(&wptr,parent);
3348 	put8bit(&wptr,nleng);
3349 	memcpy(wptr,name,nleng);
3350 	wptr+=nleng;
3351 	put16bit(&wptr,mode);
3352 	if (packetver>=2) {
3353 		put16bit(&wptr,cumask);
3354 	}
3355 	put32bit(&wptr,uid);
3356 	if (packetver<2) {
3357 		if (gids>0) {
3358 			put32bit(&wptr,gid[0]);
3359 		} else {
3360 			put32bit(&wptr,0xFFFFFFFF);
3361 		}
3362 	} else {
3363 		if (gids>0) {
3364 			put32bit(&wptr,gids);
3365 			for (i=0 ; i<gids ; i++) {
3366 				put32bit(&wptr,gid[i]);
3367 			}
3368 		} else {
3369 			put32bit(&wptr,0xFFFFFFFF);
3370 		}
3371 	}
3372 	if (packetver>=1) {
3373 		put8bit(&wptr,copysgid);
3374 	}
3375 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_MKDIR,&i);
3376 	if (rptr==NULL) {
3377 		ret = MFS_ERROR_IO;
3378 	} else if (i==1) {
3379 		ret = rptr[0];
3380 	} else if (i!=(uint32_t)(4+asize)) {
3381 		fs_disconnect();
3382 		ret = MFS_ERROR_IO;
3383 	} else {
3384 		*inode = get32bit(&rptr);
3385 		copy_attr(rptr,attr,asize);
3386 		ret = MFS_STATUS_OK;
3387 	}
3388 	return ret;
3389 }
3390 
3391 uint8_t fs_unlink(uint32_t parent,uint8_t nleng,const uint8_t *name,uint32_t uid,uint32_t gids,uint32_t *gid,uint32_t *inode) {
3392 	uint8_t *wptr;
3393 	const uint8_t *rptr;
3394 	uint32_t i;
3395 	uint8_t ret;
3396 	uint8_t packetver;
3397 	threc *rec = fs_get_my_threc();
3398 	if (master_version()<VERSION2INT(2,0,0)) {
3399 		wptr = fs_createpacket(rec,CLTOMA_FUSE_UNLINK,13+nleng);
3400 		packetver = 0;
3401 	} else {
3402 		wptr = fs_createpacket(rec,CLTOMA_FUSE_UNLINK,13+nleng+4*gids);
3403 		packetver = 1;
3404 	}
3405 	if (wptr==NULL) {
3406 		return MFS_ERROR_IO;
3407 	}
3408 	put32bit(&wptr,parent);
3409 	put8bit(&wptr,nleng);
3410 	memcpy(wptr,name,nleng);
3411 	wptr+=nleng;
3412 	put32bit(&wptr,uid);
3413 	if (packetver==0) {
3414 		if (gids>0) {
3415 			put32bit(&wptr,gid[0]);
3416 		} else {
3417 			put32bit(&wptr,0xFFFFFFFF);
3418 		}
3419 	} else {
3420 		if (gids>0) {
3421 			put32bit(&wptr,gids);
3422 			for (i=0 ; i<gids ; i++) {
3423 				put32bit(&wptr,gid[i]);
3424 			}
3425 		} else {
3426 			put32bit(&wptr,0xFFFFFFFF);
3427 		}
3428 	}
3429 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_UNLINK,&i);
3430 	if (rptr==NULL) {
3431 		ret = MFS_ERROR_IO;
3432 	} else if (i==1) {
3433 		ret = rptr[0];
3434 		*inode = 0;
3435 	} else if (i==4) {
3436 		ret = MFS_STATUS_OK;
3437 		*inode = get32bit(&rptr);
3438 	} else {
3439 		fs_disconnect();
3440 		ret = MFS_ERROR_IO;
3441 	}
3442 	return ret;
3443 }
3444 
3445 uint8_t fs_rmdir(uint32_t parent,uint8_t nleng,const uint8_t *name,uint32_t uid,uint32_t gids,uint32_t *gid,uint32_t *inode) {
3446 	uint8_t *wptr;
3447 	const uint8_t *rptr;
3448 	uint32_t i;
3449 	uint8_t ret;
3450 	uint8_t packetver;
3451 	threc *rec = fs_get_my_threc();
3452 	if (master_version()<VERSION2INT(2,0,0)) {
3453 		wptr = fs_createpacket(rec,CLTOMA_FUSE_RMDIR,13+nleng);
3454 		packetver = 0;
3455 	} else {
3456 		wptr = fs_createpacket(rec,CLTOMA_FUSE_RMDIR,13+nleng+4*gids);
3457 		packetver = 1;
3458 	}
3459 	if (wptr==NULL) {
3460 		return MFS_ERROR_IO;
3461 	}
3462 	put32bit(&wptr,parent);
3463 	put8bit(&wptr,nleng);
3464 	memcpy(wptr,name,nleng);
3465 	wptr+=nleng;
3466 	put32bit(&wptr,uid);
3467 	if (packetver==0) {
3468 		if (gids>0) {
3469 			put32bit(&wptr,gid[0]);
3470 		} else {
3471 			put32bit(&wptr,0xFFFFFFFF);
3472 		}
3473 	} else {
3474 		if (gids>0) {
3475 			put32bit(&wptr,gids);
3476 			for (i=0 ; i<gids ; i++) {
3477 				put32bit(&wptr,gid[i]);
3478 			}
3479 		} else {
3480 			put32bit(&wptr,0xFFFFFFFF);
3481 		}
3482 	}
3483 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_RMDIR,&i);
3484 	if (rptr==NULL) {
3485 		ret = MFS_ERROR_IO;
3486 	} else if (i==1) {
3487 		ret = rptr[0];
3488 		*inode = 0;
3489 	} else if (i==4) {
3490 		ret = MFS_STATUS_OK;
3491 		*inode = get32bit(&rptr);
3492 	} else {
3493 		fs_disconnect();
3494 		ret = MFS_ERROR_IO;
3495 	}
3496 	return ret;
3497 }
3498 
3499 uint8_t fs_rename(uint32_t parent_src,uint8_t nleng_src,const uint8_t *name_src,uint32_t parent_dst,uint8_t nleng_dst,const uint8_t *name_dst,uint32_t uid,uint32_t gids,uint32_t *gid,uint32_t *inode,uint8_t attr[ATTR_RECORD_SIZE]) {
3500 	uint8_t *wptr;
3501 	const uint8_t *rptr;
3502 	uint32_t i;
3503 	uint8_t ret;
3504 	uint8_t packetver;
3505 	threc *rec = fs_get_my_threc();
3506 	uint8_t asize = master_attrsize();
3507 
3508 	if (master_version()<VERSION2INT(2,0,0)) {
3509 		wptr = fs_createpacket(rec,CLTOMA_FUSE_RENAME,18+nleng_src+nleng_dst);
3510 		packetver = 0;
3511 	} else {
3512 		wptr = fs_createpacket(rec,CLTOMA_FUSE_RENAME,18+nleng_src+nleng_dst+4*gids);
3513 		packetver = 1;
3514 	}
3515 	if (wptr==NULL) {
3516 		return MFS_ERROR_IO;
3517 	}
3518 	put32bit(&wptr,parent_src);
3519 	put8bit(&wptr,nleng_src);
3520 	memcpy(wptr,name_src,nleng_src);
3521 	wptr+=nleng_src;
3522 	put32bit(&wptr,parent_dst);
3523 	put8bit(&wptr,nleng_dst);
3524 	memcpy(wptr,name_dst,nleng_dst);
3525 	wptr+=nleng_dst;
3526 	put32bit(&wptr,uid);
3527 	if (packetver==0) {
3528 		if (gids>0) {
3529 			put32bit(&wptr,gid[0]);
3530 		} else {
3531 			put32bit(&wptr,0xFFFFFFFF);
3532 		}
3533 	} else {
3534 		if (gids>0) {
3535 			put32bit(&wptr,gids);
3536 			for (i=0 ; i<gids ; i++) {
3537 				put32bit(&wptr,gid[i]);
3538 			}
3539 		} else {
3540 			put32bit(&wptr,0xFFFFFFFF);
3541 		}
3542 	}
3543 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_RENAME,&i);
3544 	if (rptr==NULL) {
3545 		ret = MFS_ERROR_IO;
3546 	} else if (i==1) {
3547 		ret = rptr[0];
3548 		*inode = 0;
3549 		memset(attr,0,ATTR_RECORD_SIZE);
3550 	} else if (i!=(uint32_t)(4+asize)) {
3551 		fs_disconnect();
3552 		ret = MFS_ERROR_IO;
3553 	} else {
3554 		*inode = get32bit(&rptr);
3555 		copy_attr(rptr,attr,asize);
3556 		ret = MFS_STATUS_OK;
3557 	}
3558 	return ret;
3559 }
3560 
3561 uint8_t fs_link(uint32_t inode_src,uint32_t parent_dst,uint8_t nleng_dst,const uint8_t *name_dst,uint32_t uid,uint32_t gids,uint32_t *gid,uint32_t *inode,uint8_t attr[ATTR_RECORD_SIZE]) {
3562 	uint8_t *wptr;
3563 	const uint8_t *rptr;
3564 	uint32_t i;
3565 	uint8_t ret;
3566 	uint8_t packetver;
3567 	threc *rec = fs_get_my_threc();
3568 	uint8_t asize = master_attrsize();
3569 
3570 	if (master_version()<VERSION2INT(2,0,0)) {
3571 		wptr = fs_createpacket(rec,CLTOMA_FUSE_LINK,17+nleng_dst);
3572 		packetver = 0;
3573 	} else {
3574 		wptr = fs_createpacket(rec,CLTOMA_FUSE_LINK,17+nleng_dst+4*gids);
3575 		packetver = 1;
3576 	}
3577 	if (wptr==NULL) {
3578 		return MFS_ERROR_IO;
3579 	}
3580 	put32bit(&wptr,inode_src);
3581 	put32bit(&wptr,parent_dst);
3582 	put8bit(&wptr,nleng_dst);
3583 	memcpy(wptr,name_dst,nleng_dst);
3584 	wptr+=nleng_dst;
3585 	put32bit(&wptr,uid);
3586 	if (packetver==0) {
3587 		if (gids>0) {
3588 			put32bit(&wptr,gid[0]);
3589 		} else {
3590 			put32bit(&wptr,0xFFFFFFFF);
3591 		}
3592 	} else {
3593 		if (gids>0) {
3594 			put32bit(&wptr,gids);
3595 			for (i=0 ; i<gids ; i++) {
3596 				put32bit(&wptr,gid[i]);
3597 			}
3598 		} else {
3599 			put32bit(&wptr,0xFFFFFFFF);
3600 		}
3601 	}
3602 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_LINK,&i);
3603 	if (rptr==NULL) {
3604 		ret = MFS_ERROR_IO;
3605 	} else if (i==1) {
3606 		ret = rptr[0];
3607 	} else if (i!=(uint32_t)(4+asize)) {
3608 		fs_disconnect();
3609 		ret = MFS_ERROR_IO;
3610 	} else {
3611 		*inode = get32bit(&rptr);
3612 		copy_attr(rptr,attr,asize);
3613 		ret = MFS_STATUS_OK;
3614 	}
3615 	return ret;
3616 }
3617 /*
3618 uint8_t fs_getdir(uint32_t inode,uint32_t uid,uint32_t gid,const uint8_t **dbuff,uint32_t *dbuffsize) {
3619 	uint8_t *wptr;
3620 	const uint8_t *rptr;
3621 	uint32_t i;
3622 	uint8_t ret;
3623 	threc *rec = fs_get_my_threc();
3624 	wptr = fs_createpacket(rec,CLTOMA_FUSE_READDIR,12);
3625 	if (wptr==NULL) {
3626 		return MFS_ERROR_IO;
3627 	}
3628 	put32bit(&wptr,inode);
3629 	put32bit(&wptr,uid);
3630 	put32bit(&wptr,gid);
3631 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_READDIR,&i);
3632 	if (rptr==NULL) {
3633 		ret = MFS_ERROR_IO;
3634 	} else if (i==1) {
3635 		ret = rptr[0];
3636 	} else {
3637 		*dbuff = rptr;
3638 		*dbuffsize = i;
3639 		ret = MFS_STATUS_OK;
3640 	}
3641 	return ret;
3642 }
3643 */
3644 
3645 uint8_t fs_readdir(uint32_t inode,uint32_t uid,uint32_t gids,uint32_t *gid,uint8_t wantattr,uint8_t addtocache,const uint8_t **dbuff,uint32_t *dbuffsize) {
3646 	uint8_t *wptr;
3647 	const uint8_t *rptr;
3648 	uint32_t i;
3649 	uint8_t ret;
3650 	uint8_t packetver;
3651 	uint8_t flags;
3652 	threc *rec = fs_get_my_threc();
3653 	if (master_version()<VERSION2INT(2,0,0)) {
3654 		wptr = fs_createpacket(rec,CLTOMA_FUSE_READDIR,13);
3655 		packetver = 0;
3656 	} else {
3657 		wptr = fs_createpacket(rec,CLTOMA_FUSE_READDIR,25+4*gids);
3658 		packetver = 1;
3659 	}
3660 	if (wptr==NULL) {
3661 		return MFS_ERROR_IO;
3662 	}
3663 	put32bit(&wptr,inode);
3664 	put32bit(&wptr,uid);
3665 	if (packetver==0) {
3666 		if (gids>0) {
3667 			put32bit(&wptr,gid[0]);
3668 		} else {
3669 			put32bit(&wptr,0xFFFFFFFF);
3670 		}
3671 	} else {
3672 		if (gids>0) {
3673 			put32bit(&wptr,gids);
3674 			for (i=0 ; i<gids ; i++) {
3675 				put32bit(&wptr,gid[i]);
3676 			}
3677 		} else {
3678 			put32bit(&wptr,0xFFFFFFFF);
3679 		}
3680 	}
3681 	flags = 0;
3682 	if (wantattr) {
3683 		flags |= GETDIR_FLAG_WITHATTR;
3684 	}
3685 	if (addtocache) {
3686 		flags |= GETDIR_FLAG_ADDTOCACHE;
3687 	}
3688 	put8bit(&wptr,flags);
3689 	if (packetver>=1) {
3690 		put32bit(&wptr,0xFFFFFFFFU);
3691 		put64bit(&wptr,0);
3692 	}
3693 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_READDIR,&i);
3694 	if (rptr==NULL) {
3695 		ret = MFS_ERROR_IO;
3696 	} else if (i==1) {
3697 		ret = rptr[0];
3698 	} else {
3699 		if (packetver>=1) { // skip 'read-next' handler
3700 			rptr+=8;
3701 			i-=8;
3702 		}
3703 		*dbuff = rptr;
3704 		*dbuffsize = i;
3705 		ret = MFS_STATUS_OK;
3706 	}
3707 	return ret;
3708 }
3709 
3710 uint8_t fs_create(uint32_t parent,uint8_t nleng,const uint8_t *name,uint16_t mode,uint16_t cumask,uint32_t uid,uint32_t gids,uint32_t *gid,uint32_t *inode,uint8_t attr[ATTR_RECORD_SIZE],uint8_t *oflags) {
3711 	uint8_t *wptr;
3712 	const uint8_t *rptr;
3713 	uint32_t i;
3714 	uint8_t ret;
3715 	uint8_t packetver;
3716 	threc *rec = fs_get_my_threc();
3717 	uint8_t asize = master_attrsize();
3718 
3719 	if (master_version()<VERSION2INT(1,7,25)) {
3720 		return MFS_ERROR_ENOTSUP;
3721 	}
3722 	if (master_version()<VERSION2INT(2,0,0)) {
3723 		mode &= ~cumask;
3724 		wptr = fs_createpacket(rec,CLTOMA_FUSE_CREATE,15+nleng);
3725 		packetver = 0;
3726 	} else {
3727 		wptr = fs_createpacket(rec,CLTOMA_FUSE_CREATE,17+nleng+4*gids);
3728 		packetver = 1;
3729 	}
3730 	if (wptr==NULL) {
3731 		return MFS_ERROR_IO;
3732 	}
3733 	put32bit(&wptr,parent);
3734 	put8bit(&wptr,nleng);
3735 	memcpy(wptr,name,nleng);
3736 	wptr+=nleng;
3737 	put16bit(&wptr,mode);
3738 	if (packetver>=1) {
3739 		put16bit(&wptr,cumask);
3740 	}
3741 	put32bit(&wptr,uid);
3742 	if (packetver==0) {
3743 		if (gids>0) {
3744 			put32bit(&wptr,gid[0]);
3745 		} else {
3746 			put32bit(&wptr,0xFFFFFFFF);
3747 		}
3748 	} else {
3749 		if (gids>0) {
3750 			put32bit(&wptr,gids);
3751 			for (i=0 ; i<gids ; i++) {
3752 				put32bit(&wptr,gid[i]);
3753 			}
3754 		} else {
3755 			put32bit(&wptr,0xFFFFFFFF);
3756 		}
3757 	}
3758 	pthread_mutex_lock(&fdlock);
3759 	donotsendsustainedinodes = 1;
3760 	pthread_mutex_unlock(&fdlock);
3761 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_CREATE,&i);
3762 	if (rptr==NULL) {
3763 		ret = MFS_ERROR_IO;
3764 	} else if (i==1) {
3765 		ret = rptr[0];
3766 	} else if (i==(uint32_t)(4+asize)) {
3767 		*oflags = 0xFF;
3768 		*inode = get32bit(&rptr);
3769 		copy_attr(rptr,attr,asize);
3770 		ret = MFS_STATUS_OK;
3771 	} else if (i==(uint32_t)(5+asize)) {
3772 		*oflags = get8bit(&rptr);
3773 		*inode = get32bit(&rptr);
3774 		copy_attr(rptr,attr,asize);
3775 		ret = MFS_STATUS_OK;
3776 	} else {
3777 		fs_disconnect();
3778 		ret = MFS_ERROR_IO;
3779 	}
3780 	pthread_mutex_lock(&fdlock);
3781 	donotsendsustainedinodes = 0;
3782 	pthread_mutex_unlock(&fdlock);
3783 	return ret;
3784 }
3785 
3786 uint8_t fs_opencheck(uint32_t inode,uint32_t uid,uint32_t gids,uint32_t *gid,uint8_t flags,uint8_t attr[ATTR_RECORD_SIZE],uint8_t *oflags) {
3787 	uint8_t *wptr;
3788 	const uint8_t *rptr;
3789 	uint32_t i;
3790 	uint8_t ret;
3791 	uint8_t packetver;
3792 	threc *rec = fs_get_my_threc();
3793 	uint8_t asize = master_attrsize();
3794 
3795 	if (master_version()<VERSION2INT(2,0,0) || gids==0) {
3796 		wptr = fs_createpacket(rec,CLTOMA_FUSE_OPEN,13);
3797 		packetver = 0;
3798 	} else {
3799 		wptr = fs_createpacket(rec,CLTOMA_FUSE_OPEN,13+4*gids);
3800 		packetver = 1;
3801 	}
3802 	if (wptr==NULL) {
3803 		return MFS_ERROR_IO;
3804 	}
3805 	put32bit(&wptr,inode);
3806 	put32bit(&wptr,uid);
3807 	if (packetver==0) {
3808 		if (gids>0) { // should be always true)
3809 			put32bit(&wptr,gid[0]);
3810 		} else {
3811 			put32bit(&wptr,0xFFFFFFFF);
3812 		}
3813 	} else {
3814 		put32bit(&wptr,gids);
3815 		for (i=0 ; i<gids ; i++) {
3816 			put32bit(&wptr,gid[i]);
3817 		}
3818 	}
3819 	put8bit(&wptr,flags);
3820 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_OPEN,&i);
3821 	if (rptr==NULL) {
3822 		ret = MFS_ERROR_IO;
3823 	} else if (i==1) {
3824 		if (attr) {
3825 			memset(attr,0,ATTR_RECORD_SIZE);
3826 		}
3827 		if (oflags) {
3828 			*oflags = 0xFF;
3829 		}
3830 		ret = rptr[0];
3831 	} else if (i==asize) {
3832 		if (attr) {
3833 			copy_attr(rptr,attr,asize);
3834 		}
3835 		if (oflags) {
3836 			*oflags = 0xFF;
3837 		}
3838 		ret = MFS_STATUS_OK;
3839 	} else if (i==(uint32_t)(1+asize)) {
3840 		if (oflags) {
3841 			*oflags = rptr[0];
3842 		}
3843 		rptr++;
3844 		if (attr) {
3845 			copy_attr(rptr,attr,asize);
3846 		}
3847 		ret = MFS_STATUS_OK;
3848 	} else {
3849 		fs_disconnect();
3850 		ret = MFS_ERROR_IO;
3851 	}
3852 	return ret;
3853 }
3854 
3855 uint8_t fs_readchunk(uint32_t inode,uint32_t indx,uint8_t chunkopflags,uint8_t *csdataver,uint64_t *length,uint64_t *chunkid,uint32_t *version,const uint8_t **csdata,uint32_t *csdatasize) {
3856 	uint8_t *wptr;
3857 	const uint8_t *rptr;
3858 	uint32_t i;
3859 	uint8_t ret;
3860 	uint8_t packetver;
3861 	threc *rec = fs_get_my_threc();
3862 
3863 	*csdata = NULL;
3864 	*csdatasize = 0;
3865 
3866 	if (master_version()>=VERSION2INT(3,0,4)) {
3867 		wptr = fs_createpacket(rec,CLTOMA_FUSE_READ_CHUNK,9);
3868 		packetver = 1;
3869 	} else {
3870 		wptr = fs_createpacket(rec,CLTOMA_FUSE_READ_CHUNK,8);
3871 		packetver = 0;
3872 	}
3873 	if (wptr==NULL) {
3874 		return MFS_ERROR_IO;
3875 	}
3876 	put32bit(&wptr,inode);
3877 	put32bit(&wptr,indx);
3878 	if (packetver>=1) {
3879 		put8bit(&wptr,chunkopflags);
3880 	}
3881 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_READ_CHUNK,&i);
3882 	if (rptr==NULL) {
3883 		ret = MFS_ERROR_IO;
3884 	} else if (i==1) {
3885 		ret = rptr[0];
3886 	} else {
3887 		if (i&1) {
3888 			*csdataver = get8bit(&rptr);
3889 			if (i<21 || ((*csdataver)==1 && ((i-21)%10)!=0) || ((*csdataver)==2 && ((i-21)%14)!=0)) {
3890 				ret = MFS_ERROR_IO;
3891 			} else {
3892 				*csdatasize = i-21;
3893 				ret = MFS_STATUS_OK;
3894 			}
3895 		} else {
3896 			*csdataver = 0;
3897 			if (i<20 || ((i-20)%6)!=0) {
3898 				ret = MFS_ERROR_IO;
3899 			} else {
3900 				*csdatasize = i-20;
3901 				ret = MFS_STATUS_OK;
3902 			}
3903 		}
3904 		if (ret!=MFS_STATUS_OK) {
3905 			fs_disconnect();
3906 		} else {
3907 			*length = get64bit(&rptr);
3908 			*chunkid = get64bit(&rptr);
3909 			*version = get32bit(&rptr);
3910 			*csdata = rptr;
3911 		}
3912 	}
3913 	return ret;
3914 }
3915 
3916 uint8_t fs_writechunk(uint32_t inode,uint32_t indx,uint8_t chunkopflags,uint8_t *csdataver,uint64_t *length,uint64_t *chunkid,uint32_t *version,const uint8_t **csdata,uint32_t *csdatasize) {
3917 	uint8_t *wptr;
3918 	const uint8_t *rptr;
3919 	uint32_t i;
3920 	uint8_t ret;
3921 	threc *rec = fs_get_my_threc();
3922 
3923 	*csdata = NULL;
3924 	*csdatasize = 0;
3925 
3926 	if (master_version()>=VERSION2INT(3,0,4)) {
3927 		wptr = fs_createpacket(rec,CLTOMA_FUSE_WRITE_CHUNK,9);
3928 	} else {
3929 		wptr = fs_createpacket(rec,CLTOMA_FUSE_WRITE_CHUNK,8);
3930 	}
3931 	if (wptr==NULL) {
3932 		return MFS_ERROR_IO;
3933 	}
3934 	put32bit(&wptr,inode);
3935 	put32bit(&wptr,indx);
3936 	if (master_version()>=VERSION2INT(3,0,4)) {
3937 		put8bit(&wptr,chunkopflags);
3938 	}
3939 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_WRITE_CHUNK,&i);
3940 	if (rptr==NULL) {
3941 		ret = MFS_ERROR_IO;
3942 	} else if (i==1) {
3943 		ret = rptr[0];
3944 	} else {
3945 		if (i&1) {
3946 			*csdataver = get8bit(&rptr);
3947 			if (i<21 || ((*csdataver)==1 && ((i-21)%10)!=0) || ((*csdataver)==2 && ((i-21)%14)!=0)) {
3948 				ret = MFS_ERROR_IO;
3949 			} else {
3950 				*csdatasize = i-21;
3951 				ret = MFS_STATUS_OK;
3952 			}
3953 		} else {
3954 			*csdataver = 0;
3955 			if (i<20 || ((i-20)%6)!=0) {
3956 				ret = MFS_ERROR_IO;
3957 			} else {
3958 				*csdatasize = i-20;
3959 				ret = MFS_STATUS_OK;
3960 			}
3961 		}
3962 		if (ret!=MFS_STATUS_OK) {
3963 			fs_disconnect();
3964 		} else {
3965 			*length = get64bit(&rptr);
3966 			*chunkid = get64bit(&rptr);
3967 			*version = get32bit(&rptr);
3968 			*csdata = rptr;
3969 		}
3970 	}
3971 	return ret;
3972 }
3973 
3974 uint8_t fs_writeend(uint64_t chunkid,uint32_t inode,uint32_t indx,uint64_t length,uint8_t chunkopflags) {
3975 	uint8_t *wptr;
3976 	const uint8_t *rptr;
3977 	uint32_t i;
3978 	uint8_t ret;
3979 	threc *rec = fs_get_my_threc();
3980 	if (master_version()>=VERSION2INT(3,0,74)) {
3981 		wptr = fs_createpacket(rec,CLTOMA_FUSE_WRITE_CHUNK_END,25);
3982 	} else if (master_version()>=VERSION2INT(3,0,4)) {
3983 		wptr = fs_createpacket(rec,CLTOMA_FUSE_WRITE_CHUNK_END,21);
3984 	} else {
3985 		wptr = fs_createpacket(rec,CLTOMA_FUSE_WRITE_CHUNK_END,20);
3986 	}
3987 	if (wptr==NULL) {
3988 		return MFS_ERROR_IO;
3989 	}
3990 	put64bit(&wptr,chunkid);
3991 	put32bit(&wptr,inode);
3992 	if (master_version()>=VERSION2INT(3,0,74)) {
3993 		put32bit(&wptr,indx);
3994 	}
3995 	put64bit(&wptr,length);
3996 	if (master_version()>=VERSION2INT(3,0,4)) {
3997 		put8bit(&wptr,chunkopflags);
3998 	}
3999 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_WRITE_CHUNK_END,&i);
4000 	if (rptr==NULL) {
4001 		ret = MFS_ERROR_IO;
4002 	} else if (i==1) {
4003 		ret = rptr[0];
4004 	} else {
4005 		fs_disconnect();
4006 		ret = MFS_ERROR_IO;
4007 	}
4008 	return ret;
4009 }
4010 /*
4011 uint8_t fs_fsync_send(uint32_t inode) {
4012 	uint8_t *wptr;
4013 	threc *rec;
4014 	if (master_version()>=VERSION2INT(3,0,74)) {
4015 		rec = fs_get_my_threc();
4016 		wptr = fs_createpacket(rec,CLTOMA_FUSE_FSYNC,4);
4017 		if (wptr==NULL) {
4018 			return MFS_ERROR_IO;
4019 		}
4020 		put32bit(&wptr,inode);
4021 		if (fs_send_only(rec)) {
4022 			return MFS_STATUS_OK;
4023 		} else {
4024 			return MFS_ERROR_IO;
4025 		}
4026 	} else {
4027 		return MFS_STATUS_OK;
4028 	}
4029 }
4030 
4031 uint8_t fs_fsync_wait(void) {
4032 	const uint8_t *rptr;
4033 	uint32_t i;
4034 	uint8_t ret;
4035 	threc *rec;
4036 	if (master_version()>=VERSION2INT(3,0,74)) {
4037 		rec = fs_get_my_threc();
4038 		rptr = fs_receive_only(rec,MATOCL_FUSE_FSYNC,&i);
4039 		if (rptr==NULL) {
4040 			ret = MFS_ERROR_IO;
4041 		} else if (i==1) {
4042 			ret = rptr[0];
4043 		} else {
4044 			fs_disconnect();
4045 			ret = MFS_ERROR_IO;
4046 		}
4047 	} else {
4048 		ret = MFS_STATUS_OK;
4049 	}
4050 	return ret;
4051 }
4052 */
4053 uint8_t fs_flock(uint32_t inode,uint32_t reqid,uint64_t owner,uint8_t cmd) {
4054 	uint8_t *wptr;
4055 	const uint8_t *rptr;
4056 	uint32_t i;
4057 	uint8_t ret;
4058 	threc *rec = fs_get_my_threc();
4059 	wptr = fs_createpacket(rec,CLTOMA_FUSE_FLOCK,17);
4060 	if (wptr==NULL) {
4061 		return MFS_ERROR_IO;
4062 	}
4063 	put32bit(&wptr,inode);
4064 	put32bit(&wptr,reqid);
4065 	put64bit(&wptr,owner);
4066 	put8bit(&wptr,cmd);
4067 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_FLOCK,&i);
4068 	if (rptr==NULL) {
4069 		ret = MFS_ERROR_IO;
4070 	} else if (i==1) {
4071 		ret = rptr[0];
4072 	} else {
4073 		fs_disconnect();
4074 		ret = MFS_ERROR_IO;
4075 	}
4076 	return ret;
4077 }
4078 
4079 uint8_t fs_posixlock(uint32_t inode,uint32_t reqid,uint64_t owner,uint8_t cmd,uint8_t type,uint64_t start,uint64_t end,uint32_t pid,uint8_t *rtype,uint64_t *rstart,uint64_t *rend,uint32_t *rpid) {
4080 	uint8_t *wptr;
4081 	const uint8_t *rptr;
4082 	uint32_t i;
4083 	uint8_t ret;
4084 	threc *rec = fs_get_my_threc();
4085 	wptr = fs_createpacket(rec,CLTOMA_FUSE_POSIX_LOCK,38);
4086 	if (wptr==NULL) {
4087 		return MFS_ERROR_IO;
4088 	}
4089 	put32bit(&wptr,inode);
4090 	put32bit(&wptr,reqid);
4091 	put64bit(&wptr,owner);
4092 	put32bit(&wptr,pid);
4093 	put8bit(&wptr,cmd);
4094 	put8bit(&wptr,type);
4095 	put64bit(&wptr,start);
4096 	put64bit(&wptr,end);
4097 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_POSIX_LOCK,&i);
4098 	if (rtype!=NULL) {
4099 		*rtype = POSIX_LOCK_UNLCK;
4100 	}
4101 	if (rstart!=NULL) {
4102 		*rstart = 0;
4103 	}
4104 	if (rend!=NULL) {
4105 		*rend = 0;
4106 	}
4107 	if (rpid!=NULL) {
4108 		*rpid = 0;
4109 	}
4110 	if (rptr==NULL) {
4111 		ret = MFS_ERROR_IO;
4112 	} else if (i==1) {
4113 		ret = rptr[0];
4114 	} else if (i==21) {
4115 		if (rpid!=NULL) {
4116 			*rpid = get32bit(&rptr);
4117 		} else {
4118 			rptr += 4;
4119 		}
4120 		if (rtype!=NULL) {
4121 			*rtype = get8bit(&rptr);
4122 		} else {
4123 			rptr++;
4124 		}
4125 		if (rstart!=NULL) {
4126 			*rstart = get64bit(&rptr);
4127 		} else {
4128 			rptr += 8;
4129 		}
4130 		if (rend!=NULL) {
4131 			*rend = get64bit(&rptr);
4132 		} else {
4133 			rptr += 8;
4134 		}
4135 		ret = MFS_STATUS_OK;
4136 	} else {
4137 		fs_disconnect();
4138 		ret = MFS_ERROR_IO;
4139 	}
4140 	return ret;
4141 }
4142 
4143 // FUSE - META
4144 
4145 
4146 uint8_t fs_getsustained(const uint8_t **dbuff,uint32_t *dbuffsize) {
4147 	uint8_t *wptr;
4148 	const uint8_t *rptr;
4149 	uint32_t i;
4150 	uint8_t ret;
4151 	threc *rec = fs_get_my_threc();
4152 	wptr = fs_createpacket(rec,CLTOMA_FUSE_GETSUSTAINED,0);
4153 	if (wptr==NULL) {
4154 		return MFS_ERROR_IO;
4155 	}
4156 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_GETSUSTAINED,&i);
4157 	if (rptr==NULL) {
4158 		ret = MFS_ERROR_IO;
4159 	} else if (i==1) {
4160 		ret = rptr[0];
4161 	} else {
4162 		*dbuff = rptr;
4163 		*dbuffsize = i;
4164 		ret = MFS_STATUS_OK;
4165 	}
4166 	return ret;
4167 }
4168 
4169 uint8_t fs_gettrash(uint32_t tid,const uint8_t **dbuff,uint32_t *dbuffsize) {
4170 	uint8_t *wptr;
4171 	const uint8_t *rptr;
4172 	uint32_t i;
4173 	uint8_t ret;
4174 	threc *rec = fs_get_my_threc();
4175 	if (master_version()>=VERSION2INT(3,0,64)) {
4176 		wptr = fs_createpacket(rec,CLTOMA_FUSE_GETTRASH,4);
4177 	} else {
4178 		wptr = fs_createpacket(rec,CLTOMA_FUSE_GETTRASH,0);
4179 	}
4180 	if (wptr==NULL) {
4181 		return MFS_ERROR_IO;
4182 	}
4183 	if (master_version()>=VERSION2INT(3,0,64)) {
4184 		put32bit(&wptr,tid);
4185 	}
4186 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_GETTRASH,&i);
4187 	if (rptr==NULL) {
4188 		ret = MFS_ERROR_IO;
4189 	} else if (i==1) {
4190 		ret = rptr[0];
4191 	} else {
4192 		*dbuff = rptr;
4193 		*dbuffsize = i;
4194 		ret = MFS_STATUS_OK;
4195 	}
4196 	return ret;
4197 }
4198 
4199 uint8_t fs_getdetachedattr(uint32_t inode,uint8_t attr[ATTR_RECORD_SIZE]) {
4200 	uint8_t *wptr;
4201 	const uint8_t *rptr;
4202 	uint32_t i;
4203 	uint8_t ret;
4204 	threc *rec = fs_get_my_threc();
4205 	uint8_t asize = master_attrsize();
4206 
4207 	wptr = fs_createpacket(rec,CLTOMA_FUSE_GETDETACHEDATTR,4);
4208 	if (wptr==NULL) {
4209 		return MFS_ERROR_IO;
4210 	}
4211 	put32bit(&wptr,inode);
4212 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_GETDETACHEDATTR,&i);
4213 	if (rptr==NULL) {
4214 		ret = MFS_ERROR_IO;
4215 	} else if (i==1) {
4216 		ret = rptr[0];
4217 	} else if (i!=asize) {
4218 		fs_disconnect();
4219 		ret = MFS_ERROR_IO;
4220 	} else {
4221 		copy_attr(rptr,attr,asize);
4222 		ret = MFS_STATUS_OK;
4223 	}
4224 	return ret;
4225 }
4226 
4227 uint8_t fs_gettrashpath(uint32_t inode,const uint8_t **path) {
4228 	uint8_t *wptr;
4229 	const uint8_t *rptr;
4230 	uint32_t i;
4231 	uint32_t pleng;
4232 	uint8_t ret;
4233 	threc *rec = fs_get_my_threc();
4234 	wptr = fs_createpacket(rec,CLTOMA_FUSE_GETTRASHPATH,4);
4235 	if (wptr==NULL) {
4236 		return MFS_ERROR_IO;
4237 	}
4238 	put32bit(&wptr,inode);
4239 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_GETTRASHPATH,&i);
4240 	if (rptr==NULL) {
4241 		ret = MFS_ERROR_IO;
4242 	} else if (i==1) {
4243 		ret = rptr[0];
4244 	} else if (i<4) {
4245 		fs_disconnect();
4246 		ret = MFS_ERROR_IO;
4247 	} else {
4248 		pleng = get32bit(&rptr);
4249 		if (i!=4+pleng || pleng==0 || rptr[pleng-1]!=0) {
4250 			fs_disconnect();
4251 			ret = MFS_ERROR_IO;
4252 		} else {
4253 			*path = rptr;
4254 			ret = MFS_STATUS_OK;
4255 		}
4256 	}
4257 	return ret;
4258 }
4259 
4260 uint8_t fs_settrashpath(uint32_t inode,const uint8_t *path) {
4261 	uint8_t *wptr;
4262 	const uint8_t *rptr;
4263 	uint32_t i;
4264 	uint32_t pleng;
4265 	uint8_t ret;
4266 	threc *rec = fs_get_my_threc();
4267 	pleng = strlen((const char *)path)+1;
4268 	wptr = fs_createpacket(rec,CLTOMA_FUSE_SETTRASHPATH,pleng+8);
4269 	if (wptr==NULL) {
4270 		return MFS_ERROR_IO;
4271 	}
4272 	put32bit(&wptr,inode);
4273 	put32bit(&wptr,pleng);
4274 	memcpy(wptr,path,pleng);
4275 //	ptr+=pleng;
4276 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_SETTRASHPATH,&i);
4277 	if (rptr==NULL) {
4278 		ret = MFS_ERROR_IO;
4279 	} else if (i==1) {
4280 		ret = rptr[0];
4281 	} else {
4282 		fs_disconnect();
4283 		ret = MFS_ERROR_IO;
4284 	}
4285 	return ret;
4286 }
4287 
4288 uint8_t fs_undel(uint32_t inode) {
4289 	uint8_t *wptr;
4290 	const uint8_t *rptr;
4291 	uint32_t i;
4292 	uint8_t ret;
4293 	threc *rec = fs_get_my_threc();
4294 	wptr = fs_createpacket(rec,CLTOMA_FUSE_UNDEL,4);
4295 	if (wptr==NULL) {
4296 		return MFS_ERROR_IO;
4297 	}
4298 	put32bit(&wptr,inode);
4299 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_UNDEL,&i);
4300 	if (rptr==NULL) {
4301 		ret = MFS_ERROR_IO;
4302 	} else if (i==1) {
4303 		ret = rptr[0];
4304 	} else {
4305 		fs_disconnect();
4306 		ret = MFS_ERROR_IO;
4307 	}
4308 	return ret;
4309 }
4310 
4311 uint8_t fs_purge(uint32_t inode) {
4312 	uint8_t *wptr;
4313 	const uint8_t *rptr;
4314 	uint32_t i;
4315 	uint8_t ret;
4316 	threc *rec = fs_get_my_threc();
4317 	wptr = fs_createpacket(rec,CLTOMA_FUSE_PURGE,4);
4318 	if (wptr==NULL) {
4319 		return MFS_ERROR_IO;
4320 	}
4321 	put32bit(&wptr,inode);
4322 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_PURGE,&i);
4323 	if (rptr==NULL) {
4324 		ret = MFS_ERROR_IO;
4325 	} else if (i==1) {
4326 		ret = rptr[0];
4327 	} else {
4328 		fs_disconnect();
4329 		ret = MFS_ERROR_IO;
4330 	}
4331 	return ret;
4332 }
4333 
4334 uint8_t fs_getfacl(uint32_t inode,/*uint8_t opened,uint32_t uid,uint32_t gids,uint32_t *gid,*/uint8_t acltype,uint16_t *userperm,uint16_t *groupperm,uint16_t *otherperm,uint16_t *maskperm,uint16_t *namedusers,uint16_t *namedgroups,const uint8_t **namedacls,uint32_t *namedaclssize) {
4335 	uint8_t *wptr;
4336 	const uint8_t *rptr;
4337 	uint32_t i;
4338 	uint8_t ret;
4339 	threc *rec = fs_get_my_threc();
4340 	*namedacls = NULL;
4341 	*namedaclssize = 0;
4342 	if (master_version()<VERSION2INT(2,0,0)) {
4343 		return MFS_ERROR_ENOTSUP;
4344 	}
4345 	if (master_version()<VERSION2INT(3,0,91)) {
4346 		wptr = fs_createpacket(rec,CLTOMA_FUSE_GETFACL,14);
4347 		if (wptr==NULL) {
4348 			return MFS_ERROR_IO;
4349 		}
4350 		put32bit(&wptr,inode);
4351 		put8bit(&wptr,acltype);
4352 		put8bit(&wptr,1);
4353 		put32bit(&wptr,0);
4354 		put32bit(&wptr,0);
4355 	} else {
4356 		wptr = fs_createpacket(rec,CLTOMA_FUSE_GETFACL,5);
4357 		if (wptr==NULL) {
4358 			return MFS_ERROR_IO;
4359 		}
4360 		put32bit(&wptr,inode);
4361 		put8bit(&wptr,acltype);
4362 	}
4363 //	put8bit(&wptr,opened);
4364 //	put32bit(&wptr,uid);
4365 //	if (gids>0) {
4366 //		put32bit(&wptr,gids);
4367 //		for (i=0 ; i<gids ; i++) {
4368 //			put32bit(&wptr,gid[i]);
4369 //		}
4370 //	} else {
4371 //		put32bit(&wptr,0xFFFFFFFFU);
4372 //	}
4373 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_GETFACL,&i);
4374 	if (rptr==NULL) {
4375 		ret = MFS_ERROR_IO;
4376 	} else if (i==1) {
4377 		ret = rptr[0];
4378 	} else if (i<12) {
4379 		fs_disconnect();
4380 		ret = MFS_ERROR_IO;
4381 	} else {
4382 		*userperm = get16bit(&rptr);
4383 		*groupperm = get16bit(&rptr);
4384 		*otherperm = get16bit(&rptr);
4385 		*maskperm = get16bit(&rptr);
4386 		*namedusers = get16bit(&rptr);
4387 		*namedgroups = get16bit(&rptr);
4388 		*namedacls = rptr;
4389 		*namedaclssize = i-12;
4390 		ret = MFS_STATUS_OK;
4391 	}
4392 	return ret;
4393 }
4394 
4395 uint8_t fs_setfacl(uint32_t inode,uint32_t uid,uint8_t acltype,uint16_t userperm,uint16_t groupperm,uint16_t otherperm,uint16_t maskperm,uint16_t namedusers,uint16_t namedgroups,uint8_t *namedacls,uint32_t namedaclssize) {
4396 	uint8_t *wptr;
4397 	const uint8_t *rptr;
4398 	uint32_t i;
4399 	uint8_t ret;
4400 	threc *rec = fs_get_my_threc();
4401 	if (master_version()<VERSION2INT(2,0,0)) {
4402 		return MFS_ERROR_ENOTSUP;
4403 	}
4404 	wptr = fs_createpacket(rec,CLTOMA_FUSE_SETFACL,21+namedaclssize);
4405 	if (wptr==NULL) {
4406 		return MFS_ERROR_IO;
4407 	}
4408 	put32bit(&wptr,inode);
4409 	put32bit(&wptr,uid);
4410 	put8bit(&wptr,acltype);
4411 	put16bit(&wptr,userperm);
4412 	put16bit(&wptr,groupperm);
4413 	put16bit(&wptr,otherperm);
4414 	put16bit(&wptr,maskperm);
4415 	put16bit(&wptr,namedusers);
4416 	put16bit(&wptr,namedgroups);
4417 	memcpy(wptr,namedacls,namedaclssize);
4418 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_SETFACL,&i);
4419 	if (rptr==NULL) {
4420 		ret = MFS_ERROR_IO;
4421 	} else if (i==1) {
4422 		ret = rptr[0];
4423 	} else {
4424 		fs_disconnect();
4425 		ret = MFS_ERROR_IO;
4426 	}
4427 	return ret;
4428 }
4429 
4430 uint8_t fs_getxattr(uint32_t inode,uint8_t opened,uint32_t uid,uint32_t gids,uint32_t *gid,uint8_t nleng,const uint8_t *name,uint8_t mode,const uint8_t **vbuff,uint32_t *vleng) {
4431 	uint8_t *wptr;
4432 	const uint8_t *rptr;
4433 	uint32_t i;
4434 	uint8_t ret;
4435 	uint8_t packetver;
4436 	threc *rec = fs_get_my_threc();
4437 	*vbuff = NULL;
4438 	*vleng = 0;
4439 	if (master_version()<VERSION2INT(1,7,0)) {
4440 		return MFS_ERROR_ENOTSUP;
4441 	}
4442 	if (master_version()<VERSION2INT(2,0,0)) {
4443 		wptr = fs_createpacket(rec,CLTOMA_FUSE_GETXATTR,15+nleng);
4444 		packetver = 0;
4445 	} else {
4446 		wptr = fs_createpacket(rec,CLTOMA_FUSE_GETXATTR,15+nleng+4*gids);
4447 		packetver = 1;
4448 	}
4449 	if (wptr==NULL) {
4450 		return MFS_ERROR_IO;
4451 	}
4452 	put32bit(&wptr,inode);
4453 	if (packetver==0) {
4454 		put8bit(&wptr,opened);
4455 		put32bit(&wptr,uid);
4456 		if (gids>0) {
4457 			put32bit(&wptr,gid[0]);
4458 		} else {
4459 			put32bit(&wptr,0xFFFFFFFFU);
4460 		}
4461 	}
4462 	put8bit(&wptr,nleng);
4463 	memcpy(wptr,name,nleng);
4464 	wptr+=nleng;
4465 	put8bit(&wptr,mode);
4466 	if (packetver>=1) {
4467 		put8bit(&wptr,opened);
4468 		put32bit(&wptr,uid);
4469 		if (gids>0) {
4470 			put32bit(&wptr,gids);
4471 			for (i=0 ; i<gids ; i++) {
4472 				put32bit(&wptr,gid[i]);
4473 			}
4474 		} else {
4475 			put32bit(&wptr,0xFFFFFFFFU);
4476 		}
4477 	}
4478 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_GETXATTR,&i);
4479 	if (rptr==NULL) {
4480 		ret = MFS_ERROR_IO;
4481 	} else if (i==1) {
4482 		ret = rptr[0];
4483 	} else if (i<4) {
4484 		fs_disconnect();
4485 		ret = MFS_ERROR_IO;
4486 	} else {
4487 		*vleng = get32bit(&rptr);
4488 		*vbuff = (mode==MFS_XATTR_GETA_DATA)?rptr:NULL;
4489 		if ((mode==MFS_XATTR_GETA_DATA && i!=(*vleng)+4) || (mode==MFS_XATTR_LENGTH_ONLY && i!=4)) {
4490 			fs_disconnect();
4491 			ret = MFS_ERROR_IO;
4492 		} else {
4493 			ret = MFS_STATUS_OK;
4494 		}
4495 	}
4496 	return ret;
4497 }
4498 
4499 uint8_t fs_listxattr(uint32_t inode,uint8_t opened,uint32_t uid,uint32_t gids,uint32_t *gid,uint8_t mode,const uint8_t **dbuff,uint32_t *dleng) {
4500 	uint8_t *wptr;
4501 	const uint8_t *rptr;
4502 	uint32_t i;
4503 	uint8_t ret;
4504 	uint8_t packetver;
4505 	threc *rec = fs_get_my_threc();
4506 	if (master_version()<VERSION2INT(1,7,0)) {
4507 		return MFS_ERROR_ENOTSUP;
4508 	}
4509 	if (master_version()<VERSION2INT(2,0,0)) {
4510 		wptr = fs_createpacket(rec,CLTOMA_FUSE_GETXATTR,15);
4511 		packetver = 0;
4512 	} else {
4513 		wptr = fs_createpacket(rec,CLTOMA_FUSE_GETXATTR,15+4*gids);
4514 		packetver = 1;
4515 	}
4516 	if (wptr==NULL) {
4517 		return MFS_ERROR_IO;
4518 	}
4519 	put32bit(&wptr,inode);
4520 	if (packetver==0) {
4521 		put8bit(&wptr,opened);
4522 		put32bit(&wptr,uid);
4523 		if (gids>0) {
4524 			put32bit(&wptr,gid[0]);
4525 		} else {
4526 			put32bit(&wptr,0xFFFFFFFFU);
4527 		}
4528 	}
4529 	put8bit(&wptr,0);
4530 	put8bit(&wptr,mode);
4531 	if (packetver>=1) {
4532 		put8bit(&wptr,opened);
4533 		put32bit(&wptr,uid);
4534 		if (gids>0) {
4535 			put32bit(&wptr,gids);
4536 			for (i=0 ; i<gids ; i++) {
4537 				put32bit(&wptr,gid[i]);
4538 			}
4539 		} else {
4540 			put32bit(&wptr,0xFFFFFFFFU);
4541 		}
4542 	}
4543 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_GETXATTR,&i);
4544 	if (rptr==NULL) {
4545 		ret = MFS_ERROR_IO;
4546 	} else if (i==1) {
4547 		ret = rptr[0];
4548 	} else if (i<4) {
4549 		fs_disconnect();
4550 		ret = MFS_ERROR_IO;
4551 	} else {
4552 		*dleng = get32bit(&rptr);
4553 		*dbuff = (mode==MFS_XATTR_GETA_DATA)?rptr:NULL;
4554 		if ((mode==MFS_XATTR_GETA_DATA && i!=(*dleng)+4) || (mode==MFS_XATTR_LENGTH_ONLY && i!=4)) {
4555 			fs_disconnect();
4556 			ret = MFS_ERROR_IO;
4557 		} else {
4558 			ret = MFS_STATUS_OK;
4559 		}
4560 	}
4561 	return ret;
4562 }
4563 
4564 uint8_t fs_setxattr(uint32_t inode,uint8_t opened,uint32_t uid,uint32_t gids,uint32_t *gid,uint8_t nleng,const uint8_t *name,uint32_t vleng,const uint8_t *value,uint8_t mode) {
4565 	uint8_t *wptr;
4566 	const uint8_t *rptr;
4567 	uint32_t i;
4568 	uint8_t ret;
4569 	uint8_t packetver;
4570 	threc *rec = fs_get_my_threc();
4571 	if (master_version()<VERSION2INT(1,7,0)) {
4572 		return MFS_ERROR_ENOTSUP;
4573 	}
4574 	if (mode>=MFS_XATTR_REMOVE) {
4575 		return MFS_ERROR_EINVAL;
4576 	}
4577 	if (master_version()<VERSION2INT(2,0,0)) {
4578 		wptr = fs_createpacket(rec,CLTOMA_FUSE_SETXATTR,19+nleng+vleng);
4579 		packetver = 0;
4580 	} else {
4581 		wptr = fs_createpacket(rec,CLTOMA_FUSE_SETXATTR,19+nleng+vleng+4*gids);
4582 		packetver = 1;
4583 	}
4584 	if (wptr==NULL) {
4585 		return MFS_ERROR_IO;
4586 	}
4587 	put32bit(&wptr,inode);
4588 	if (packetver==0) {
4589 		put8bit(&wptr,opened);
4590 		put32bit(&wptr,uid);
4591 		if (gids>0) {
4592 			put32bit(&wptr,gid[0]);
4593 		} else {
4594 			put32bit(&wptr,0xFFFFFFFFU);
4595 		}
4596 	}
4597 	put8bit(&wptr,nleng);
4598 	memcpy(wptr,name,nleng);
4599 	wptr+=nleng;
4600 	put32bit(&wptr,vleng);
4601 	memcpy(wptr,value,vleng);
4602 	wptr+=vleng;
4603 	put8bit(&wptr,mode);
4604 	if (packetver>=1) {
4605 		put8bit(&wptr,opened);
4606 		put32bit(&wptr,uid);
4607 		if (gids>0) {
4608 			put32bit(&wptr,gids);
4609 			for (i=0 ; i<gids ; i++) {
4610 				put32bit(&wptr,gid[i]);
4611 			}
4612 		} else {
4613 			put32bit(&wptr,0xFFFFFFFFU);
4614 		}
4615 	}
4616 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_SETXATTR,&i);
4617 	if (rptr==NULL) {
4618 		ret = MFS_ERROR_IO;
4619 	} else if (i==1) {
4620 		ret = rptr[0];
4621 	} else {
4622 		fs_disconnect();
4623 		ret = MFS_ERROR_IO;
4624 	}
4625 	return ret;
4626 }
4627 
4628 uint8_t fs_removexattr(uint32_t inode,uint8_t opened,uint32_t uid,uint32_t gids,uint32_t *gid,uint8_t nleng,const uint8_t *name) {
4629 	uint8_t *wptr;
4630 	const uint8_t *rptr;
4631 	uint32_t i;
4632 	uint8_t ret;
4633 	uint8_t packetver;
4634 	threc *rec = fs_get_my_threc();
4635 	if (master_version()<VERSION2INT(1,7,0)) {
4636 		return MFS_ERROR_ENOTSUP;
4637 	}
4638 	if (master_version()<VERSION2INT(2,0,0)) {
4639 		wptr = fs_createpacket(rec,CLTOMA_FUSE_SETXATTR,19+nleng);
4640 		packetver = 0;
4641 	} else {
4642 		wptr = fs_createpacket(rec,CLTOMA_FUSE_SETXATTR,19+nleng+4*gids);
4643 		packetver = 1;
4644 	}
4645 	if (wptr==NULL) {
4646 		return MFS_ERROR_IO;
4647 	}
4648 	put32bit(&wptr,inode);
4649 	if (packetver==0) {
4650 		put8bit(&wptr,opened);
4651 		put32bit(&wptr,uid);
4652 		if (gids>0) {
4653 			put32bit(&wptr,gid[0]);
4654 		} else {
4655 			put32bit(&wptr,0xFFFFFFFFU);
4656 		}
4657 	}
4658 	put8bit(&wptr,nleng);
4659 	memcpy(wptr,name,nleng);
4660 	wptr+=nleng;
4661 	put32bit(&wptr,0);
4662 	put8bit(&wptr,MFS_XATTR_REMOVE);
4663 	if (packetver>=1) {
4664 		put8bit(&wptr,opened);
4665 		put32bit(&wptr,uid);
4666 		if (gids>0) {
4667 			put32bit(&wptr,gids);
4668 			for (i=0 ; i<gids ; i++) {
4669 				put32bit(&wptr,gid[i]);
4670 			}
4671 		} else {
4672 			put32bit(&wptr,0xFFFFFFFFU);
4673 		}
4674 	}
4675 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_SETXATTR,&i);
4676 	if (rptr==NULL) {
4677 		ret = MFS_ERROR_IO;
4678 	} else if (i==1) {
4679 		ret = rptr[0];
4680 	} else {
4681 		fs_disconnect();
4682 		ret = MFS_ERROR_IO;
4683 	}
4684 	return ret;
4685 }
4686 
4687 uint8_t fs_custom(uint32_t qcmd,const uint8_t *query,uint32_t queryleng,uint32_t *acmd,uint8_t *answer,uint32_t *answerleng) {
4688 	uint8_t *wptr;
4689 	const uint8_t *rptr;
4690 	uint32_t i;
4691 	uint8_t ret;
4692 	threc *rec = fs_get_my_threc();
4693 	wptr = fs_createpacket(rec,qcmd,queryleng);
4694 	if (wptr==NULL) {
4695 		return MFS_ERROR_IO;
4696 	}
4697 	memcpy(wptr,query,queryleng);
4698 	rptr = fs_sendandreceive_any(rec,acmd,&i);
4699 	if (rptr==NULL) {
4700 		ret = MFS_ERROR_IO;
4701 	} else {
4702 		if (*answerleng<i) {
4703 			ret = MFS_ERROR_EINVAL;
4704 		} else {
4705 			*answerleng = i;
4706 			memcpy(answer,rptr,i);
4707 			ret = MFS_STATUS_OK;
4708 		}
4709 	}
4710 	return ret;
4711 }
4712 
4713 /*
4714 uint8_t fs_append(uint32_t inode,uint32_t ainode,uint32_t uid,uint32_t gid) {
4715 	uint8_t *wptr;
4716 	const uint8_t *rptr;
4717 	uint32_t i;
4718 	uint8_t ret;
4719 	threc *rec = fs_get_my_threc();
4720 	wptr = fs_createpacket(rec,CLTOMA_FUSE_APPEND,16);
4721 	if (wptr==NULL) {
4722 		return MFS_ERROR_IO;
4723 	}
4724 	put32bit(&wptr,inode);
4725 	put32bit(&wptr,ainode);
4726 	put32bit(&wptr,uid);
4727 	put32bit(&wptr,gid);
4728 	rptr = fs_sendandreceive(rec,MATOCL_FUSE_APPEND,&i);
4729 	if (rptr==NULL) {
4730 		ret = MFS_ERROR_IO;
4731 	} else if (i==1) {
4732 		ret = rptr[0];
4733 	} else {
4734 		fs_disconnect();
4735 		ret = MFS_ERROR_IO;
4736 	}
4737 	return ret;
4738 }
4739 */
4740