1 /*
2 * Zebra API server.
3 * Portions:
4 * Copyright (C) 1997-1999 Kunihiro Ishiguro
5 * Copyright (C) 2015-2018 Cumulus Networks, Inc.
6 * et al.
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; see the file COPYING; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23 #include <zebra.h>
24
25 /* clang-format off */
26 #include <errno.h> /* for errno */
27 #include <netinet/in.h> /* for sockaddr_in */
28 #include <stdint.h> /* for uint8_t */
29 #include <stdio.h> /* for snprintf */
30 #include <sys/socket.h> /* for sockaddr_storage, AF_UNIX, accept... */
31 #include <sys/stat.h> /* for umask, mode_t */
32 #include <sys/un.h> /* for sockaddr_un */
33 #include <time.h> /* for NULL, tm, gmtime, time_t */
34 #include <unistd.h> /* for close, unlink, ssize_t */
35
36 #include "lib/buffer.h" /* for BUFFER_EMPTY, BUFFER_ERROR, BUFFE... */
37 #include "lib/command.h" /* for vty, install_element, CMD_SUCCESS... */
38 #include "lib/hook.h" /* for DEFINE_HOOK, DEFINE_KOOH, hook_call */
39 #include "lib/linklist.h" /* for ALL_LIST_ELEMENTS_RO, ALL_LIST_EL... */
40 #include "lib/libfrr.h" /* for frr_zclient_addr */
41 #include "lib/log.h" /* for zlog_warn, zlog_debug, safe_strerror */
42 #include "lib/memory.h" /* for MTYPE_TMP, XCALLOC, XFREE */
43 #include "lib/monotime.h" /* for monotime, ONE_DAY_SECOND, ONE_WEE... */
44 #include "lib/network.h" /* for set_nonblocking */
45 #include "lib/privs.h" /* for zebra_privs_t, ZPRIVS_LOWER, ZPRI... */
46 #include "lib/route_types.h" /* for ZEBRA_ROUTE_MAX */
47 #include "lib/sockopt.h" /* for setsockopt_so_recvbuf, setsockopt... */
48 #include "lib/sockunion.h" /* for sockopt_reuseaddr, sockopt_reuseport */
49 #include "lib/stream.h" /* for STREAM_SIZE, stream (ptr only), ... */
50 #include "lib/thread.h" /* for thread (ptr only), THREAD_ARG, ... */
51 #include "lib/vrf.h" /* for vrf_info_lookup, VRF_DEFAULT */
52 #include "lib/vty.h" /* for vty_out, vty (ptr only) */
53 #include "lib/zassert.h" /* for assert */
54 #include "lib/zclient.h" /* for zmsghdr, ZEBRA_HEADER_SIZE, ZEBRA... */
55 #include "lib/frr_pthread.h" /* for frr_pthread_new, frr_pthread_stop... */
56 #include "lib/frratomic.h" /* for atomic_load_explicit, atomic_stor... */
57 #include "lib/lib_errors.h" /* for generic ferr ids */
58
59 #include "zebra/debug.h" /* for various debugging macros */
60 #include "zebra/rib.h" /* for rib_score_proto */
61 #include "zebra/zapi_msg.h" /* for zserv_handle_commands */
62 #include "zebra/zebra_vrf.h" /* for zebra_vrf_lookup_by_id, zvrf */
63 #include "zebra/zserv.h" /* for zserv */
64 #include "zebra/zebra_router.h"
65 #include "zebra/zebra_errors.h" /* for error messages */
66 /* clang-format on */
67
68 /* privileges */
69 extern struct zebra_privs_t zserv_privs;
70
71 /* The listener socket for clients connecting to us */
72 static int zsock;
73
74 /* The lock that protects access to zapi client objects */
75 static pthread_mutex_t client_mutex;
76
77 static struct zserv *find_client_internal(uint8_t proto,
78 unsigned short instance,
79 uint32_t session_id);
80
81
82 /*
83 * Client thread events.
84 *
85 * These are used almost exclusively by client threads to drive their own event
86 * loops. The only exception is in zserv_client_create(), which pushes an
87 * initial ZSERV_CLIENT_READ event to start the API handler loop.
88 */
89 enum zserv_client_event {
90 /* Schedule a socket read */
91 ZSERV_CLIENT_READ,
92 /* Schedule a buffer write */
93 ZSERV_CLIENT_WRITE,
94 };
95
96 /*
97 * Main thread events.
98 *
99 * These are used by client threads to notify the main thread about various
100 * events and to make processing requests.
101 */
102 enum zserv_event {
103 /* Schedule listen job on Zebra API socket */
104 ZSERV_ACCEPT,
105 /* The calling client has packets on its input buffer */
106 ZSERV_PROCESS_MESSAGES,
107 /* The calling client wishes to be killed */
108 ZSERV_HANDLE_CLIENT_FAIL,
109 };
110
111 /*
112 * Zebra server event driver for all client threads.
113 *
114 * This is essentially a wrapper around thread_add_event() that centralizes
115 * those scheduling calls into one place.
116 *
117 * All calls to this function schedule an event on the pthread running the
118 * provided client.
119 *
120 * client
121 * the client in question, and thread target
122 *
123 * event
124 * the event to notify them about
125 */
126 static void zserv_client_event(struct zserv *client,
127 enum zserv_client_event event);
128
129 /*
130 * Zebra server event driver for the main thread.
131 *
132 * This is essentially a wrapper around thread_add_event() that centralizes
133 * those scheduling calls into one place.
134 *
135 * All calls to this function schedule an event on Zebra's main pthread.
136 *
137 * client
138 * the client in question
139 *
140 * event
141 * the event to notify the main thread about
142 */
143 static void zserv_event(struct zserv *client, enum zserv_event event);
144
145
146 /* Client thread lifecycle -------------------------------------------------- */
147
148 /*
149 * Log zapi message to zlog.
150 *
151 * errmsg (optional)
152 * Debugging message
153 *
154 * msg
155 * The message
156 *
157 * hdr (optional)
158 * The message header
159 */
zserv_log_message(const char * errmsg,struct stream * msg,struct zmsghdr * hdr)160 void zserv_log_message(const char *errmsg, struct stream *msg,
161 struct zmsghdr *hdr)
162 {
163 zlog_debug("Rx'd ZAPI message");
164 if (errmsg)
165 zlog_debug("%s", errmsg);
166 if (hdr) {
167 zlog_debug(" Length: %d", hdr->length);
168 zlog_debug("Command: %s", zserv_command_string(hdr->command));
169 zlog_debug(" VRF: %u", hdr->vrf_id);
170 }
171 stream_hexdump(msg);
172 }
173
174 /*
175 * Gracefully shut down a client connection.
176 *
177 * Cancel any pending tasks for the client's thread. Then schedule a task on
178 * the main thread to shut down the calling thread.
179 *
180 * It is not safe to close the client socket in this function. The socket is
181 * owned by the main thread.
182 *
183 * Must be called from the client pthread, never the main thread.
184 */
zserv_client_fail(struct zserv * client)185 static void zserv_client_fail(struct zserv *client)
186 {
187 flog_warn(EC_ZEBRA_CLIENT_IO_ERROR,
188 "Client '%s' encountered an error and is shutting down.",
189 zebra_route_string(client->proto));
190
191 atomic_store_explicit(&client->pthread->running, false,
192 memory_order_relaxed);
193
194 THREAD_OFF(client->t_read);
195 THREAD_OFF(client->t_write);
196 zserv_event(client, ZSERV_HANDLE_CLIENT_FAIL);
197 }
198
199 /*
200 * Write all pending messages to client socket.
201 *
202 * This function first attempts to flush any buffered data. If unsuccessful,
203 * the function reschedules itself and returns. If successful, it pops all
204 * available messages from the output queue and continues to write data
205 * directly to the socket until the socket would block. If the socket never
206 * blocks and all data is written, the function returns without rescheduling
207 * itself. If the socket ends up throwing EWOULDBLOCK, the remaining data is
208 * buffered and the function reschedules itself.
209 *
210 * The utility of the buffer is that it allows us to vastly reduce lock
211 * contention by allowing us to pop *all* messages off the output queue at once
212 * instead of locking and unlocking each time we want to pop a single message
213 * off the queue. The same thing could arguably be accomplished faster by
214 * allowing the main thread to write directly into the buffer instead of
215 * enqueuing packets onto an intermediary queue, but the intermediary queue
216 * allows us to expose information about input and output queues to the user in
217 * terms of number of packets rather than size of data.
218 */
zserv_write(struct thread * thread)219 static int zserv_write(struct thread *thread)
220 {
221 struct zserv *client = THREAD_ARG(thread);
222 struct stream *msg;
223 uint32_t wcmd = 0;
224 struct stream_fifo *cache;
225
226 /* If we have any data pending, try to flush it first */
227 switch (buffer_flush_all(client->wb, client->sock)) {
228 case BUFFER_ERROR:
229 goto zwrite_fail;
230 case BUFFER_PENDING:
231 atomic_store_explicit(&client->last_write_time,
232 (uint32_t)monotime(NULL),
233 memory_order_relaxed);
234 zserv_client_event(client, ZSERV_CLIENT_WRITE);
235 return 0;
236 case BUFFER_EMPTY:
237 break;
238 }
239
240 cache = stream_fifo_new();
241
242 frr_with_mutex(&client->obuf_mtx) {
243 while (stream_fifo_head(client->obuf_fifo))
244 stream_fifo_push(cache,
245 stream_fifo_pop(client->obuf_fifo));
246 }
247
248 if (cache->tail) {
249 msg = cache->tail;
250 stream_set_getp(msg, 0);
251 wcmd = stream_getw_from(msg, ZAPI_HEADER_CMD_LOCATION);
252 }
253
254 while (stream_fifo_head(cache)) {
255 msg = stream_fifo_pop(cache);
256 buffer_put(client->wb, STREAM_DATA(msg), stream_get_endp(msg));
257 stream_free(msg);
258 }
259
260 stream_fifo_free(cache);
261
262 /* If we have any data pending, try to flush it first */
263 switch (buffer_flush_all(client->wb, client->sock)) {
264 case BUFFER_ERROR:
265 goto zwrite_fail;
266 case BUFFER_PENDING:
267 atomic_store_explicit(&client->last_write_time,
268 (uint32_t)monotime(NULL),
269 memory_order_relaxed);
270 zserv_client_event(client, ZSERV_CLIENT_WRITE);
271 return 0;
272 case BUFFER_EMPTY:
273 break;
274 }
275
276 atomic_store_explicit(&client->last_write_cmd, wcmd,
277 memory_order_relaxed);
278
279 atomic_store_explicit(&client->last_write_time,
280 (uint32_t)monotime(NULL), memory_order_relaxed);
281
282 return 0;
283
284 zwrite_fail:
285 flog_warn(EC_ZEBRA_CLIENT_WRITE_FAILED,
286 "%s: could not write to %s [fd = %d], closing.", __func__,
287 zebra_route_string(client->proto), client->sock);
288 zserv_client_fail(client);
289 return 0;
290 }
291
292 /*
293 * Read and process data from a client socket.
294 *
295 * The responsibilities here are to read raw data from the client socket,
296 * validate the header, encapsulate it into a single stream object, push it
297 * onto the input queue and then notify the main thread that there is new data
298 * available.
299 *
300 * This function first looks for any data in the client structure's working
301 * input buffer. If data is present, it is assumed that reading stopped in a
302 * previous invocation of this task and needs to be resumed to finish a message.
303 * Otherwise, the socket data stream is assumed to be at the beginning of a new
304 * ZAPI message (specifically at the header). The header is read and validated.
305 * If the header passed validation then the length field found in the header is
306 * used to compute the total length of the message. That much data is read (but
307 * not inspected), appended to the header, placed into a stream and pushed onto
308 * the client's input queue. A task is then scheduled on the main thread to
309 * process the client's input queue. Finally, if all of this was successful,
310 * this task reschedules itself.
311 *
312 * Any failure in any of these actions is handled by terminating the client.
313 */
zserv_read(struct thread * thread)314 static int zserv_read(struct thread *thread)
315 {
316 struct zserv *client = THREAD_ARG(thread);
317 int sock;
318 size_t already;
319 struct stream_fifo *cache;
320 uint32_t p2p_orig;
321
322 uint32_t p2p;
323 struct zmsghdr hdr;
324
325 p2p_orig = atomic_load_explicit(&zrouter.packets_to_process,
326 memory_order_relaxed);
327 cache = stream_fifo_new();
328 p2p = p2p_orig;
329 sock = THREAD_FD(thread);
330
331 while (p2p) {
332 ssize_t nb;
333 bool hdrvalid;
334 char errmsg[256];
335
336 already = stream_get_endp(client->ibuf_work);
337
338 /* Read length and command (if we don't have it already). */
339 if (already < ZEBRA_HEADER_SIZE) {
340 nb = stream_read_try(client->ibuf_work, sock,
341 ZEBRA_HEADER_SIZE - already);
342 if ((nb == 0 || nb == -1)) {
343 if (IS_ZEBRA_DEBUG_EVENT)
344 zlog_debug("connection closed socket [%d]",
345 sock);
346 goto zread_fail;
347 }
348 if (nb != (ssize_t)(ZEBRA_HEADER_SIZE - already)) {
349 /* Try again later. */
350 break;
351 }
352 already = ZEBRA_HEADER_SIZE;
353 }
354
355 /* Reset to read from the beginning of the incoming packet. */
356 stream_set_getp(client->ibuf_work, 0);
357
358 /* Fetch header values */
359 hdrvalid = zapi_parse_header(client->ibuf_work, &hdr);
360
361 if (!hdrvalid) {
362 snprintf(errmsg, sizeof(errmsg),
363 "%s: Message has corrupt header", __func__);
364 zserv_log_message(errmsg, client->ibuf_work, NULL);
365 goto zread_fail;
366 }
367
368 /* Validate header */
369 if (hdr.marker != ZEBRA_HEADER_MARKER
370 || hdr.version != ZSERV_VERSION) {
371 snprintf(
372 errmsg, sizeof(errmsg),
373 "Message has corrupt header\n%s: socket %d version mismatch, marker %d, version %d",
374 __func__, sock, hdr.marker, hdr.version);
375 zserv_log_message(errmsg, client->ibuf_work, &hdr);
376 goto zread_fail;
377 }
378 if (hdr.length < ZEBRA_HEADER_SIZE) {
379 snprintf(
380 errmsg, sizeof(errmsg),
381 "Message has corrupt header\n%s: socket %d message length %u is less than header size %d",
382 __func__, sock, hdr.length, ZEBRA_HEADER_SIZE);
383 zserv_log_message(errmsg, client->ibuf_work, &hdr);
384 goto zread_fail;
385 }
386 if (hdr.length > STREAM_SIZE(client->ibuf_work)) {
387 snprintf(
388 errmsg, sizeof(errmsg),
389 "Message has corrupt header\n%s: socket %d message length %u exceeds buffer size %lu",
390 __func__, sock, hdr.length,
391 (unsigned long)STREAM_SIZE(client->ibuf_work));
392 zserv_log_message(errmsg, client->ibuf_work, &hdr);
393 goto zread_fail;
394 }
395
396 /* Read rest of data. */
397 if (already < hdr.length) {
398 nb = stream_read_try(client->ibuf_work, sock,
399 hdr.length - already);
400 if ((nb == 0 || nb == -1)) {
401 if (IS_ZEBRA_DEBUG_EVENT)
402 zlog_debug(
403 "connection closed [%d] when reading zebra data",
404 sock);
405 goto zread_fail;
406 }
407 if (nb != (ssize_t)(hdr.length - already)) {
408 /* Try again later. */
409 break;
410 }
411 }
412
413 /* Debug packet information. */
414 if (IS_ZEBRA_DEBUG_PACKET)
415 zlog_debug("zebra message[%s:%u:%u] comes from socket [%d]",
416 zserv_command_string(hdr.command),
417 hdr.vrf_id, hdr.length,
418 sock);
419
420 stream_set_getp(client->ibuf_work, 0);
421 struct stream *msg = stream_dup(client->ibuf_work);
422
423 stream_fifo_push(cache, msg);
424 stream_reset(client->ibuf_work);
425 p2p--;
426 }
427
428 if (p2p < p2p_orig) {
429 /* update session statistics */
430 atomic_store_explicit(&client->last_read_time, monotime(NULL),
431 memory_order_relaxed);
432 atomic_store_explicit(&client->last_read_cmd, hdr.command,
433 memory_order_relaxed);
434
435 /* publish read packets on client's input queue */
436 frr_with_mutex(&client->ibuf_mtx) {
437 while (cache->head)
438 stream_fifo_push(client->ibuf_fifo,
439 stream_fifo_pop(cache));
440 }
441
442 /* Schedule job to process those packets */
443 zserv_event(client, ZSERV_PROCESS_MESSAGES);
444
445 }
446
447 if (IS_ZEBRA_DEBUG_PACKET)
448 zlog_debug("Read %d packets from client: %s", p2p_orig - p2p,
449 zebra_route_string(client->proto));
450
451 /* Reschedule ourselves */
452 zserv_client_event(client, ZSERV_CLIENT_READ);
453
454 stream_fifo_free(cache);
455
456 return 0;
457
458 zread_fail:
459 stream_fifo_free(cache);
460 zserv_client_fail(client);
461 return -1;
462 }
463
zserv_client_event(struct zserv * client,enum zserv_client_event event)464 static void zserv_client_event(struct zserv *client,
465 enum zserv_client_event event)
466 {
467 switch (event) {
468 case ZSERV_CLIENT_READ:
469 thread_add_read(client->pthread->master, zserv_read, client,
470 client->sock, &client->t_read);
471 break;
472 case ZSERV_CLIENT_WRITE:
473 thread_add_write(client->pthread->master, zserv_write, client,
474 client->sock, &client->t_write);
475 break;
476 }
477 }
478
479 /* Main thread lifecycle ---------------------------------------------------- */
480
481 /*
482 * Read and process messages from a client.
483 *
484 * This task runs on the main pthread. It is scheduled by client pthreads when
485 * they have new messages available on their input queues. The client is passed
486 * as the task argument.
487 *
488 * Each message is popped off the client's input queue and the action associated
489 * with the message is executed. This proceeds until there are no more messages,
490 * an error occurs, or the processing limit is reached.
491 *
492 * The client's I/O thread can push at most zrouter.packets_to_process messages
493 * onto the input buffer before notifying us there are packets to read. As long
494 * as we always process zrouter.packets_to_process messages here, then we can
495 * rely on the read thread to handle queuing this task enough times to process
496 * everything on the input queue.
497 */
zserv_process_messages(struct thread * thread)498 static int zserv_process_messages(struct thread *thread)
499 {
500 struct zserv *client = THREAD_ARG(thread);
501 struct stream *msg;
502 struct stream_fifo *cache = stream_fifo_new();
503 uint32_t p2p = zrouter.packets_to_process;
504 bool need_resched = false;
505
506 frr_with_mutex(&client->ibuf_mtx) {
507 uint32_t i;
508 for (i = 0; i < p2p && stream_fifo_head(client->ibuf_fifo);
509 ++i) {
510 msg = stream_fifo_pop(client->ibuf_fifo);
511 stream_fifo_push(cache, msg);
512 }
513
514 msg = NULL;
515
516 /* Need to reschedule processing work if there are still
517 * packets in the fifo.
518 */
519 if (stream_fifo_head(client->ibuf_fifo))
520 need_resched = true;
521 }
522
523 /* Process the batch of messages */
524 if (stream_fifo_head(cache))
525 zserv_handle_commands(client, cache);
526
527 stream_fifo_free(cache);
528
529 /* Reschedule ourselves if necessary */
530 if (need_resched)
531 zserv_event(client, ZSERV_PROCESS_MESSAGES);
532
533 return 0;
534 }
535
zserv_send_message(struct zserv * client,struct stream * msg)536 int zserv_send_message(struct zserv *client, struct stream *msg)
537 {
538 frr_with_mutex(&client->obuf_mtx) {
539 stream_fifo_push(client->obuf_fifo, msg);
540 }
541
542 zserv_client_event(client, ZSERV_CLIENT_WRITE);
543
544 return 0;
545 }
546
547 /*
548 * Send a batch of messages to a connected Zebra API client.
549 */
zserv_send_batch(struct zserv * client,struct stream_fifo * fifo)550 int zserv_send_batch(struct zserv *client, struct stream_fifo *fifo)
551 {
552 struct stream *msg;
553
554 frr_with_mutex(&client->obuf_mtx) {
555 msg = stream_fifo_pop(fifo);
556 while (msg) {
557 stream_fifo_push(client->obuf_fifo, msg);
558 msg = stream_fifo_pop(fifo);
559 }
560 }
561
562 zserv_client_event(client, ZSERV_CLIENT_WRITE);
563
564 return 0;
565 }
566
567 /* Hooks for client connect / disconnect */
568 DEFINE_HOOK(zserv_client_connect, (struct zserv *client), (client));
569 DEFINE_KOOH(zserv_client_close, (struct zserv *client), (client));
570
571 /*
572 * Deinitialize zebra client.
573 *
574 * - Deregister and deinitialize related internal resources
575 * - Gracefully close socket
576 * - Free associated resources
577 * - Free client structure
578 *
579 * This does *not* take any action on the struct thread * fields. These are
580 * managed by the owning pthread and any tasks associated with them must have
581 * been stopped prior to invoking this function.
582 */
zserv_client_free(struct zserv * client)583 static void zserv_client_free(struct zserv *client)
584 {
585 if (client == NULL)
586 return;
587
588 hook_call(zserv_client_close, client);
589
590 /* Close file descriptor. */
591 if (client->sock) {
592 unsigned long nroutes;
593
594 close(client->sock);
595
596 if (DYNAMIC_CLIENT_GR_DISABLED(client)) {
597 nroutes = rib_score_proto(client->proto,
598 client->instance);
599 zlog_notice(
600 "client %d disconnected %lu %s routes removed from the rib",
601 client->sock, nroutes,
602 zebra_route_string(client->proto));
603 }
604 client->sock = -1;
605 }
606
607 /* Free stream buffers. */
608 if (client->ibuf_work)
609 stream_free(client->ibuf_work);
610 if (client->obuf_work)
611 stream_free(client->obuf_work);
612 if (client->ibuf_fifo)
613 stream_fifo_free(client->ibuf_fifo);
614 if (client->obuf_fifo)
615 stream_fifo_free(client->obuf_fifo);
616 if (client->wb)
617 buffer_free(client->wb);
618
619 /* Free buffer mutexes */
620 pthread_mutex_destroy(&client->obuf_mtx);
621 pthread_mutex_destroy(&client->ibuf_mtx);
622
623 /* Free bitmaps. */
624 for (afi_t afi = AFI_IP; afi < AFI_MAX; afi++) {
625 for (int i = 0; i < ZEBRA_ROUTE_MAX; i++) {
626 vrf_bitmap_free(client->redist[afi][i]);
627 redist_del_all_instances(&client->mi_redist[afi][i]);
628 }
629
630 vrf_bitmap_free(client->redist_default[afi]);
631 vrf_bitmap_free(client->ridinfo[afi]);
632 }
633
634 /*
635 * If any instance are graceful restart enabled,
636 * client is not deleted
637 */
638 if (DYNAMIC_CLIENT_GR_DISABLED(client)) {
639 if (IS_ZEBRA_DEBUG_EVENT)
640 zlog_debug("%s: Deleting client %s", __func__,
641 zebra_route_string(client->proto));
642 XFREE(MTYPE_TMP, client);
643 } else {
644 /* Handle cases where client has GR instance. */
645 if (IS_ZEBRA_DEBUG_EVENT)
646 zlog_debug("%s: client %s restart enabled", __func__,
647 zebra_route_string(client->proto));
648 if (zebra_gr_client_disconnect(client) < 0)
649 zlog_err(
650 "%s: GR enabled but could not handle disconnect event",
651 __func__);
652 }
653 }
654
zserv_close_client(struct zserv * client)655 void zserv_close_client(struct zserv *client)
656 {
657 bool free_p = true;
658
659 if (client->pthread) {
660 /* synchronously stop and join pthread */
661 frr_pthread_stop(client->pthread, NULL);
662
663 if (IS_ZEBRA_DEBUG_EVENT)
664 zlog_debug("Closing client '%s'",
665 zebra_route_string(client->proto));
666
667 thread_cancel_event(zrouter.master, client);
668 THREAD_OFF(client->t_cleanup);
669 THREAD_OFF(client->t_process);
670
671 /* destroy pthread */
672 frr_pthread_destroy(client->pthread);
673 client->pthread = NULL;
674 }
675
676 /*
677 * Final check in case the client struct is in use in another
678 * pthread: if not in-use, continue and free the client
679 */
680 frr_with_mutex(&client_mutex) {
681 if (client->busy_count <= 0) {
682 /* remove from client list */
683 listnode_delete(zrouter.client_list, client);
684 } else {
685 /*
686 * The client session object may be in use, although
687 * the associated pthread is gone. Defer final
688 * cleanup.
689 */
690 client->is_closed = true;
691 free_p = false;
692 }
693 }
694
695 /* delete client */
696 if (free_p)
697 zserv_client_free(client);
698 }
699
700 /*
701 * This task is scheduled by a ZAPI client pthread on the main pthread when it
702 * wants to stop itself. When this executes, the client connection should
703 * already have been closed and the thread will most likely have died, but its
704 * resources still need to be cleaned up.
705 */
zserv_handle_client_fail(struct thread * thread)706 static int zserv_handle_client_fail(struct thread *thread)
707 {
708 struct zserv *client = THREAD_ARG(thread);
709
710 zserv_close_client(client);
711 return 0;
712 }
713
714 /*
715 * Create a new client.
716 *
717 * This is called when a new connection is accept()'d on the ZAPI socket. It
718 * initializes new client structure, notifies any subscribers of the connection
719 * event and spawns the client's thread.
720 *
721 * sock
722 * client's socket file descriptor
723 */
zserv_client_create(int sock)724 static struct zserv *zserv_client_create(int sock)
725 {
726 struct zserv *client;
727 size_t stream_size =
728 MAX(ZEBRA_MAX_PACKET_SIZ, sizeof(struct zapi_route));
729 int i;
730 afi_t afi;
731
732 client = XCALLOC(MTYPE_TMP, sizeof(struct zserv));
733
734 /* Make client input/output buffer. */
735 client->sock = sock;
736 client->ibuf_fifo = stream_fifo_new();
737 client->obuf_fifo = stream_fifo_new();
738 client->ibuf_work = stream_new(stream_size);
739 client->obuf_work = stream_new(stream_size);
740 pthread_mutex_init(&client->ibuf_mtx, NULL);
741 pthread_mutex_init(&client->obuf_mtx, NULL);
742 client->wb = buffer_new(0);
743 TAILQ_INIT(&(client->gr_info_queue));
744
745 atomic_store_explicit(&client->connect_time, (uint32_t) monotime(NULL),
746 memory_order_relaxed);
747
748 /* Initialize flags */
749 for (afi = AFI_IP; afi < AFI_MAX; afi++) {
750 for (i = 0; i < ZEBRA_ROUTE_MAX; i++)
751 client->redist[afi][i] = vrf_bitmap_init();
752 client->redist_default[afi] = vrf_bitmap_init();
753 client->ridinfo[afi] = vrf_bitmap_init();
754 }
755
756 /* Add this client to linked list. */
757 frr_with_mutex(&client_mutex) {
758 listnode_add(zrouter.client_list, client);
759 }
760
761 struct frr_pthread_attr zclient_pthr_attrs = {
762 .start = frr_pthread_attr_default.start,
763 .stop = frr_pthread_attr_default.stop
764 };
765 client->pthread =
766 frr_pthread_new(&zclient_pthr_attrs, "Zebra API client thread",
767 "zebra_apic");
768
769 /* start read loop */
770 zserv_client_event(client, ZSERV_CLIENT_READ);
771
772 /* call callbacks */
773 hook_call(zserv_client_connect, client);
774
775 /* start pthread */
776 frr_pthread_run(client->pthread, NULL);
777
778 return client;
779 }
780
781 /*
782 * Retrieve a client object by the complete tuple of
783 * {protocol, instance, session}. This version supports use
784 * from a different pthread: the object will be returned marked
785 * in-use. The caller *must* release the client object with the
786 * release_client() api, to ensure that the in-use marker is cleared properly.
787 */
zserv_acquire_client(uint8_t proto,unsigned short instance,uint32_t session_id)788 struct zserv *zserv_acquire_client(uint8_t proto, unsigned short instance,
789 uint32_t session_id)
790 {
791 struct zserv *client = NULL;
792
793 frr_with_mutex(&client_mutex) {
794 client = find_client_internal(proto, instance, session_id);
795 if (client) {
796 /* Don't return a dead/closed client object */
797 if (client->is_closed)
798 client = NULL;
799 else
800 client->busy_count++;
801 }
802 }
803
804 return client;
805 }
806
807 /*
808 * Release a client object that was acquired with the acquire_client() api.
809 * After this has been called, the caller must not use the client pointer -
810 * it may be freed if the client has closed.
811 */
zserv_release_client(struct zserv * client)812 void zserv_release_client(struct zserv *client)
813 {
814 /*
815 * Once we've decremented the client object's refcount, it's possible
816 * for it to be deleted as soon as we release the lock, so we won't
817 * touch the object again.
818 */
819 frr_with_mutex(&client_mutex) {
820 client->busy_count--;
821
822 if (client->busy_count <= 0) {
823 /*
824 * No more users of the client object. If the client
825 * session is closed, schedule cleanup on the zebra
826 * main pthread.
827 */
828 if (client->is_closed)
829 thread_add_event(zrouter.master,
830 zserv_handle_client_fail,
831 client, 0, &client->t_cleanup);
832 }
833 }
834
835 /*
836 * Cleanup must take place on the zebra main pthread, so we've
837 * scheduled an event.
838 */
839 }
840
841 /*
842 * Accept socket connection.
843 */
zserv_accept(struct thread * thread)844 static int zserv_accept(struct thread *thread)
845 {
846 int accept_sock;
847 int client_sock;
848 struct sockaddr_in client;
849 socklen_t len;
850
851 accept_sock = THREAD_FD(thread);
852
853 /* Reregister myself. */
854 zserv_event(NULL, ZSERV_ACCEPT);
855
856 len = sizeof(struct sockaddr_in);
857 client_sock = accept(accept_sock, (struct sockaddr *)&client, &len);
858
859 if (client_sock < 0) {
860 flog_err_sys(EC_LIB_SOCKET, "Can't accept zebra socket: %s",
861 safe_strerror(errno));
862 return -1;
863 }
864
865 /* Make client socket non-blocking. */
866 set_nonblocking(client_sock);
867
868 /* Create new zebra client. */
869 zserv_client_create(client_sock);
870
871 return 0;
872 }
873
zserv_close(void)874 void zserv_close(void)
875 {
876 /*
877 * On shutdown, let's close the socket down
878 * so that long running processes of killing the
879 * routing table doesn't leave us in a bad
880 * state where a client tries to reconnect
881 */
882 close(zsock);
883 zsock = -1;
884
885 /* Free client list's mutex */
886 pthread_mutex_destroy(&client_mutex);
887 }
888
zserv_start(char * path)889 void zserv_start(char *path)
890 {
891 int ret;
892 mode_t old_mask;
893 struct sockaddr_storage sa;
894 socklen_t sa_len;
895
896 if (!frr_zclient_addr(&sa, &sa_len, path))
897 /* should be caught in zebra main() */
898 return;
899
900 /* Set umask */
901 old_mask = umask(0077);
902
903 /* Make UNIX domain socket. */
904 zsock = socket(sa.ss_family, SOCK_STREAM, 0);
905 if (zsock < 0) {
906 flog_err_sys(EC_LIB_SOCKET, "Can't create zserv socket: %s",
907 safe_strerror(errno));
908 return;
909 }
910
911 if (sa.ss_family != AF_UNIX) {
912 sockopt_reuseaddr(zsock);
913 sockopt_reuseport(zsock);
914 } else {
915 struct sockaddr_un *suna = (struct sockaddr_un *)&sa;
916 if (suna->sun_path[0])
917 unlink(suna->sun_path);
918 }
919
920 setsockopt_so_recvbuf(zsock, 1048576);
921 setsockopt_so_sendbuf(zsock, 1048576);
922
923 frr_with_privs((sa.ss_family != AF_UNIX) ? &zserv_privs : NULL) {
924 ret = bind(zsock, (struct sockaddr *)&sa, sa_len);
925 }
926 if (ret < 0) {
927 flog_err_sys(EC_LIB_SOCKET, "Can't bind zserv socket on %s: %s",
928 path, safe_strerror(errno));
929 close(zsock);
930 zsock = -1;
931 return;
932 }
933
934 ret = listen(zsock, 5);
935 if (ret < 0) {
936 flog_err_sys(EC_LIB_SOCKET,
937 "Can't listen to zserv socket %s: %s", path,
938 safe_strerror(errno));
939 close(zsock);
940 zsock = -1;
941 return;
942 }
943
944 umask(old_mask);
945
946 zserv_event(NULL, ZSERV_ACCEPT);
947 }
948
zserv_event(struct zserv * client,enum zserv_event event)949 void zserv_event(struct zserv *client, enum zserv_event event)
950 {
951 switch (event) {
952 case ZSERV_ACCEPT:
953 thread_add_read(zrouter.master, zserv_accept, NULL, zsock,
954 NULL);
955 break;
956 case ZSERV_PROCESS_MESSAGES:
957 thread_add_event(zrouter.master, zserv_process_messages, client,
958 0, &client->t_process);
959 break;
960 case ZSERV_HANDLE_CLIENT_FAIL:
961 thread_add_event(zrouter.master, zserv_handle_client_fail,
962 client, 0, &client->t_cleanup);
963 }
964 }
965
966
967 /* General purpose ---------------------------------------------------------- */
968
969 #define ZEBRA_TIME_BUF 32
zserv_time_buf(time_t * time1,char * buf,int buflen)970 static char *zserv_time_buf(time_t *time1, char *buf, int buflen)
971 {
972 time_t now;
973
974 assert(buf != NULL);
975 assert(buflen >= ZEBRA_TIME_BUF);
976 assert(time1 != NULL);
977
978 if (!*time1) {
979 snprintf(buf, buflen, "never ");
980 return (buf);
981 }
982
983 now = monotime(NULL);
984 now -= *time1;
985
986 frrtime_to_interval(now, buf, buflen);
987
988 return buf;
989 }
990
991 /* Display client info details */
zebra_show_client_detail(struct vty * vty,struct zserv * client)992 static void zebra_show_client_detail(struct vty *vty, struct zserv *client)
993 {
994 char cbuf[ZEBRA_TIME_BUF], rbuf[ZEBRA_TIME_BUF];
995 char wbuf[ZEBRA_TIME_BUF], nhbuf[ZEBRA_TIME_BUF], mbuf[ZEBRA_TIME_BUF];
996 time_t connect_time, last_read_time, last_write_time;
997 uint32_t last_read_cmd, last_write_cmd;
998 struct client_gr_info *info = NULL;
999
1000 vty_out(vty, "Client: %s", zebra_route_string(client->proto));
1001 if (client->instance)
1002 vty_out(vty, " Instance: %u", client->instance);
1003 if (client->session_id)
1004 vty_out(vty, " [%u]", client->session_id);
1005 vty_out(vty, "\n");
1006
1007 vty_out(vty, "------------------------ \n");
1008 vty_out(vty, "FD: %d \n", client->sock);
1009
1010 connect_time = (time_t) atomic_load_explicit(&client->connect_time,
1011 memory_order_relaxed);
1012
1013 vty_out(vty, "Connect Time: %s \n",
1014 zserv_time_buf(&connect_time, cbuf, ZEBRA_TIME_BUF));
1015 if (client->nh_reg_time) {
1016 vty_out(vty, "Nexthop Registry Time: %s \n",
1017 zserv_time_buf(&client->nh_reg_time, nhbuf,
1018 ZEBRA_TIME_BUF));
1019 if (client->nh_last_upd_time)
1020 vty_out(vty, "Nexthop Last Update Time: %s \n",
1021 zserv_time_buf(&client->nh_last_upd_time, mbuf,
1022 ZEBRA_TIME_BUF));
1023 else
1024 vty_out(vty, "No Nexthop Update sent\n");
1025 } else
1026 vty_out(vty, "Not registered for Nexthop Updates\n");
1027
1028 last_read_time = (time_t)atomic_load_explicit(&client->last_read_time,
1029 memory_order_relaxed);
1030 last_write_time = (time_t)atomic_load_explicit(&client->last_write_time,
1031 memory_order_relaxed);
1032
1033 last_read_cmd = atomic_load_explicit(&client->last_read_cmd,
1034 memory_order_relaxed);
1035 last_write_cmd = atomic_load_explicit(&client->last_write_cmd,
1036 memory_order_relaxed);
1037
1038 vty_out(vty, "Last Msg Rx Time: %s \n",
1039 zserv_time_buf(&last_read_time, rbuf, ZEBRA_TIME_BUF));
1040 vty_out(vty, "Last Msg Tx Time: %s \n",
1041 zserv_time_buf(&last_write_time, wbuf, ZEBRA_TIME_BUF));
1042 if (last_read_cmd)
1043 vty_out(vty, "Last Rcvd Cmd: %s \n",
1044 zserv_command_string(last_read_cmd));
1045 if (last_write_cmd)
1046 vty_out(vty, "Last Sent Cmd: %s \n",
1047 zserv_command_string(last_write_cmd));
1048 vty_out(vty, "\n");
1049
1050 vty_out(vty, "Type Add Update Del \n");
1051 vty_out(vty, "================================================== \n");
1052 vty_out(vty, "IPv4 %-12u%-12u%-12u\n", client->v4_route_add_cnt,
1053 client->v4_route_upd8_cnt, client->v4_route_del_cnt);
1054 vty_out(vty, "IPv6 %-12u%-12u%-12u\n", client->v6_route_add_cnt,
1055 client->v6_route_upd8_cnt, client->v6_route_del_cnt);
1056 vty_out(vty, "Redist:v4 %-12u%-12u%-12u\n", client->redist_v4_add_cnt,
1057 0, client->redist_v4_del_cnt);
1058 vty_out(vty, "Redist:v6 %-12u%-12u%-12u\n", client->redist_v6_add_cnt,
1059 0, client->redist_v6_del_cnt);
1060 vty_out(vty, "Connected %-12u%-12u%-12u\n", client->ifadd_cnt, 0,
1061 client->ifdel_cnt);
1062 vty_out(vty, "BFD peer %-12u%-12u%-12u\n", client->bfd_peer_add_cnt,
1063 client->bfd_peer_upd8_cnt, client->bfd_peer_del_cnt);
1064 vty_out(vty, "NHT v4 %-12u%-12u%-12u\n",
1065 client->v4_nh_watch_add_cnt, 0, client->v4_nh_watch_rem_cnt);
1066 vty_out(vty, "NHT v6 %-12u%-12u%-12u\n",
1067 client->v6_nh_watch_add_cnt, 0, client->v6_nh_watch_rem_cnt);
1068 vty_out(vty, "VxLAN SG %-12u%-12u%-12u\n", client->vxlan_sg_add_cnt,
1069 0, client->vxlan_sg_del_cnt);
1070 vty_out(vty, "Interface Up Notifications: %u\n", client->ifup_cnt);
1071 vty_out(vty, "Interface Down Notifications: %u\n", client->ifdown_cnt);
1072 vty_out(vty, "VNI add notifications: %u\n", client->vniadd_cnt);
1073 vty_out(vty, "VNI delete notifications: %u\n", client->vnidel_cnt);
1074 vty_out(vty, "L3-VNI add notifications: %u\n", client->l3vniadd_cnt);
1075 vty_out(vty, "L3-VNI delete notifications: %u\n", client->l3vnidel_cnt);
1076 vty_out(vty, "MAC-IP add notifications: %u\n", client->macipadd_cnt);
1077 vty_out(vty, "MAC-IP delete notifications: %u\n", client->macipdel_cnt);
1078 vty_out(vty, "ES add notifications: %u\n", client->local_es_add_cnt);
1079 vty_out(vty, "ES delete notifications: %u\n", client->local_es_del_cnt);
1080 vty_out(vty, "ES-EVI add notifications: %u\n",
1081 client->local_es_evi_add_cnt);
1082 vty_out(vty, "ES-EVI delete notifications: %u\n",
1083 client->local_es_evi_del_cnt);
1084
1085 TAILQ_FOREACH (info, &client->gr_info_queue, gr_info) {
1086 vty_out(vty, "VRF : %s\n", vrf_id_to_name(info->vrf_id));
1087 vty_out(vty, "Capabilities : ");
1088 switch (info->capabilities) {
1089 case ZEBRA_CLIENT_GR_CAPABILITIES:
1090 vty_out(vty, "Graceful Restart\n");
1091 break;
1092 case ZEBRA_CLIENT_ROUTE_UPDATE_COMPLETE:
1093 case ZEBRA_CLIENT_ROUTE_UPDATE_PENDING:
1094 case ZEBRA_CLIENT_GR_DISABLE:
1095 case ZEBRA_CLIENT_RIB_STALE_TIME:
1096 vty_out(vty, "None\n");
1097 break;
1098 }
1099 }
1100
1101 #if defined DEV_BUILD
1102 vty_out(vty, "Input Fifo: %zu:%zu Output Fifo: %zu:%zu\n",
1103 client->ibuf_fifo->count, client->ibuf_fifo->max_count,
1104 client->obuf_fifo->count, client->obuf_fifo->max_count);
1105 #endif
1106 vty_out(vty, "\n");
1107 }
1108
1109 /* Display stale client information */
zebra_show_stale_client_detail(struct vty * vty,struct zserv * client)1110 static void zebra_show_stale_client_detail(struct vty *vty,
1111 struct zserv *client)
1112 {
1113 char buf[PREFIX2STR_BUFFER];
1114 time_t uptime;
1115 struct client_gr_info *info = NULL;
1116 struct zserv *s = NULL;
1117 bool first_p = true;
1118
1119 TAILQ_FOREACH (info, &client->gr_info_queue, gr_info) {
1120 if (first_p) {
1121 vty_out(vty, "Stale Client Information\n");
1122 vty_out(vty, "------------------------\n");
1123
1124 if (client->instance)
1125 vty_out(vty, " Instance: %u", client->instance);
1126 if (client->session_id)
1127 vty_out(vty, " [%u]", client->session_id);
1128
1129 first_p = false;
1130 }
1131
1132 vty_out(vty, "VRF : %s\n", vrf_id_to_name(info->vrf_id));
1133 vty_out(vty, "Capabilities : ");
1134 switch (info->capabilities) {
1135 case ZEBRA_CLIENT_GR_CAPABILITIES:
1136 vty_out(vty, "Graceful Restart\n");
1137 break;
1138 case ZEBRA_CLIENT_ROUTE_UPDATE_COMPLETE:
1139 case ZEBRA_CLIENT_ROUTE_UPDATE_PENDING:
1140 case ZEBRA_CLIENT_GR_DISABLE:
1141 case ZEBRA_CLIENT_RIB_STALE_TIME:
1142 vty_out(vty, "None\n");
1143 break;
1144 }
1145
1146 if (ZEBRA_CLIENT_GR_ENABLED(info->capabilities)) {
1147 if (info->stale_client_ptr) {
1148 s = (struct zserv *)(info->stale_client_ptr);
1149 uptime = monotime(NULL);
1150 uptime -= s->restart_time;
1151
1152 frrtime_to_interval(uptime, buf, sizeof(buf));
1153
1154 vty_out(vty, "Last restart time : %s ago\n",
1155 buf);
1156
1157 vty_out(vty, "Stalepath removal time: %d sec\n",
1158 info->stale_removal_time);
1159 if (info->t_stale_removal) {
1160 vty_out(vty,
1161 "Stale delete timer: %ld sec\n",
1162 thread_timer_remain_second(
1163 info->t_stale_removal));
1164 }
1165 }
1166 vty_out(vty, "Current AFI : %d\n", info->current_afi);
1167 if (info->current_prefix) {
1168 prefix2str(info->current_prefix, buf,
1169 sizeof(buf));
1170 vty_out(vty, "Current prefix : %s\n", buf);
1171 }
1172 }
1173 }
1174 vty_out(vty, "\n");
1175 return;
1176 }
1177
zebra_show_client_brief(struct vty * vty,struct zserv * client)1178 static void zebra_show_client_brief(struct vty *vty, struct zserv *client)
1179 {
1180 char cbuf[ZEBRA_TIME_BUF], rbuf[ZEBRA_TIME_BUF];
1181 char wbuf[ZEBRA_TIME_BUF];
1182 time_t connect_time, last_read_time, last_write_time;
1183
1184 connect_time = (time_t)atomic_load_explicit(&client->connect_time,
1185 memory_order_relaxed);
1186 last_read_time = (time_t)atomic_load_explicit(&client->last_read_time,
1187 memory_order_relaxed);
1188 last_write_time = (time_t)atomic_load_explicit(&client->last_write_time,
1189 memory_order_relaxed);
1190
1191 vty_out(vty, "%-10s%12s %12s%12s%8d/%-8d%8d/%-8d\n",
1192 zebra_route_string(client->proto),
1193 zserv_time_buf(&connect_time, cbuf, ZEBRA_TIME_BUF),
1194 zserv_time_buf(&last_read_time, rbuf, ZEBRA_TIME_BUF),
1195 zserv_time_buf(&last_write_time, wbuf, ZEBRA_TIME_BUF),
1196 client->v4_route_add_cnt + client->v4_route_upd8_cnt,
1197 client->v4_route_del_cnt,
1198 client->v6_route_add_cnt + client->v6_route_upd8_cnt,
1199 client->v6_route_del_cnt);
1200 }
1201
1202 /*
1203 * Common logic that searches the client list for a zapi client; this
1204 * MUST be called holding the client list mutex.
1205 */
find_client_internal(uint8_t proto,unsigned short instance,uint32_t session_id)1206 static struct zserv *find_client_internal(uint8_t proto,
1207 unsigned short instance,
1208 uint32_t session_id)
1209 {
1210 struct listnode *node, *nnode;
1211 struct zserv *client = NULL;
1212
1213 for (ALL_LIST_ELEMENTS(zrouter.client_list, node, nnode, client)) {
1214 if (client->proto == proto && client->instance == instance &&
1215 client->session_id == session_id)
1216 break;
1217 }
1218
1219 return client;
1220 }
1221
1222 /*
1223 * Public api that searches for a client session; this version is
1224 * used from the zebra main pthread.
1225 */
zserv_find_client(uint8_t proto,unsigned short instance)1226 struct zserv *zserv_find_client(uint8_t proto, unsigned short instance)
1227 {
1228 struct zserv *client;
1229
1230 frr_with_mutex(&client_mutex) {
1231 client = find_client_internal(proto, instance, 0);
1232 }
1233
1234 return client;
1235 }
1236
1237 /*
1238 * Retrieve a client by its protocol, instance number, and session id.
1239 */
zserv_find_client_session(uint8_t proto,unsigned short instance,uint32_t session_id)1240 struct zserv *zserv_find_client_session(uint8_t proto, unsigned short instance,
1241 uint32_t session_id)
1242 {
1243 struct zserv *client;
1244
1245 frr_with_mutex(&client_mutex) {
1246 client = find_client_internal(proto, instance, session_id);
1247 }
1248
1249 return client;
1250
1251 }
1252
1253 /* This command is for debugging purpose. */
1254 DEFUN (show_zebra_client,
1255 show_zebra_client_cmd,
1256 "show zebra client",
1257 SHOW_STR
1258 ZEBRA_STR
1259 "Client information\n")
1260 {
1261 struct listnode *node;
1262 struct zserv *client;
1263
1264 for (ALL_LIST_ELEMENTS_RO(zrouter.client_list, node, client)) {
1265 zebra_show_client_detail(vty, client);
1266 /* Show GR info if present */
1267 zebra_show_stale_client_detail(vty, client);
1268 }
1269
1270 return CMD_SUCCESS;
1271 }
1272
1273 /* This command is for debugging purpose. */
1274 DEFUN (show_zebra_client_summary,
1275 show_zebra_client_summary_cmd,
1276 "show zebra client summary",
1277 SHOW_STR
1278 ZEBRA_STR
1279 "Client information brief\n"
1280 "Brief Summary\n")
1281 {
1282 struct listnode *node;
1283 struct zserv *client;
1284
1285 vty_out(vty,
1286 "Name Connect Time Last Read Last Write IPv4 Routes IPv6 Routes \n");
1287 vty_out(vty,
1288 "--------------------------------------------------------------------------------\n");
1289
1290 for (ALL_LIST_ELEMENTS_RO(zrouter.client_list, node, client))
1291 zebra_show_client_brief(vty, client);
1292
1293 vty_out(vty, "Routes column shows (added+updated)/deleted\n");
1294 return CMD_SUCCESS;
1295 }
1296
zserv_init(void)1297 void zserv_init(void)
1298 {
1299 /* Client list init. */
1300 zrouter.client_list = list_new();
1301 zrouter.stale_client_list = list_new();
1302
1303 /* Misc init. */
1304 zsock = -1;
1305 pthread_mutex_init(&client_mutex, NULL);
1306
1307 install_element(ENABLE_NODE, &show_zebra_client_cmd);
1308 install_element(ENABLE_NODE, &show_zebra_client_summary_cmd);
1309 }
1310