13b3a8eb9SGleb Smirnoff /*- 2fe267a55SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3fe267a55SPedro F. Giffuni * 43b3a8eb9SGleb Smirnoff * Copyright (c) 2010 Luigi Rizzo, Riccardo Panicucci, Universita` di Pisa 53b3a8eb9SGleb Smirnoff * All rights reserved 63b3a8eb9SGleb Smirnoff * 73b3a8eb9SGleb Smirnoff * Redistribution and use in source and binary forms, with or without 83b3a8eb9SGleb Smirnoff * modification, are permitted provided that the following conditions 93b3a8eb9SGleb Smirnoff * are met: 103b3a8eb9SGleb Smirnoff * 1. Redistributions of source code must retain the above copyright 113b3a8eb9SGleb Smirnoff * notice, this list of conditions and the following disclaimer. 123b3a8eb9SGleb Smirnoff * 2. Redistributions in binary form must reproduce the above copyright 133b3a8eb9SGleb Smirnoff * notice, this list of conditions and the following disclaimer in the 143b3a8eb9SGleb Smirnoff * documentation and/or other materials provided with the distribution. 153b3a8eb9SGleb Smirnoff * 163b3a8eb9SGleb Smirnoff * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 173b3a8eb9SGleb Smirnoff * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 183b3a8eb9SGleb Smirnoff * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 193b3a8eb9SGleb Smirnoff * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 203b3a8eb9SGleb Smirnoff * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 213b3a8eb9SGleb Smirnoff * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 223b3a8eb9SGleb Smirnoff * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 233b3a8eb9SGleb Smirnoff * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 243b3a8eb9SGleb Smirnoff * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 253b3a8eb9SGleb Smirnoff * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 263b3a8eb9SGleb Smirnoff * SUCH DAMAGE. 273b3a8eb9SGleb Smirnoff */ 283b3a8eb9SGleb Smirnoff 293b3a8eb9SGleb Smirnoff /* 303b3a8eb9SGleb Smirnoff * internal dummynet APIs. 313b3a8eb9SGleb Smirnoff * 323b3a8eb9SGleb Smirnoff * $FreeBSD$ 333b3a8eb9SGleb Smirnoff */ 343b3a8eb9SGleb Smirnoff 353b3a8eb9SGleb Smirnoff #ifndef _IP_DN_PRIVATE_H 363b3a8eb9SGleb Smirnoff #define _IP_DN_PRIVATE_H 373b3a8eb9SGleb Smirnoff 383b3a8eb9SGleb Smirnoff /* debugging support 393b3a8eb9SGleb Smirnoff * use ND() to remove debugging, D() to print a line, 403b3a8eb9SGleb Smirnoff * DX(level, ...) to print above a certain level 413b3a8eb9SGleb Smirnoff * If you redefine D() you are expected to redefine all. 423b3a8eb9SGleb Smirnoff */ 433b3a8eb9SGleb Smirnoff #ifndef D 443b3a8eb9SGleb Smirnoff #define ND(fmt, ...) do {} while (0) 453b3a8eb9SGleb Smirnoff #define D1(fmt, ...) do {} while (0) 463b3a8eb9SGleb Smirnoff #define D(fmt, ...) printf("%-10s " fmt "\n", \ 473b3a8eb9SGleb Smirnoff __FUNCTION__, ## __VA_ARGS__) 483b3a8eb9SGleb Smirnoff #define DX(lev, fmt, ...) do { \ 493b3a8eb9SGleb Smirnoff if (dn_cfg.debug > lev) D(fmt, ## __VA_ARGS__); } while (0) 503b3a8eb9SGleb Smirnoff #endif 513b3a8eb9SGleb Smirnoff 523b3a8eb9SGleb Smirnoff MALLOC_DECLARE(M_DUMMYNET); 533b3a8eb9SGleb Smirnoff 543b3a8eb9SGleb Smirnoff #ifndef __linux__ 553b3a8eb9SGleb Smirnoff #define div64(a, b) ((int64_t)(a) / (int64_t)(b)) 563b3a8eb9SGleb Smirnoff #endif 573b3a8eb9SGleb Smirnoff 583b3a8eb9SGleb Smirnoff #define DN_LOCK_INIT() do { \ 593b3a8eb9SGleb Smirnoff mtx_init(&dn_cfg.uh_mtx, "dn_uh", NULL, MTX_DEF); \ 603b3a8eb9SGleb Smirnoff mtx_init(&dn_cfg.bh_mtx, "dn_bh", NULL, MTX_DEF); \ 613b3a8eb9SGleb Smirnoff } while (0) 623b3a8eb9SGleb Smirnoff #define DN_LOCK_DESTROY() do { \ 633b3a8eb9SGleb Smirnoff mtx_destroy(&dn_cfg.uh_mtx); \ 643b3a8eb9SGleb Smirnoff mtx_destroy(&dn_cfg.bh_mtx); \ 653b3a8eb9SGleb Smirnoff } while (0) 663b3a8eb9SGleb Smirnoff #if 0 /* not used yet */ 673b3a8eb9SGleb Smirnoff #define DN_UH_RLOCK() mtx_lock(&dn_cfg.uh_mtx) 683b3a8eb9SGleb Smirnoff #define DN_UH_RUNLOCK() mtx_unlock(&dn_cfg.uh_mtx) 693b3a8eb9SGleb Smirnoff #define DN_UH_WLOCK() mtx_lock(&dn_cfg.uh_mtx) 703b3a8eb9SGleb Smirnoff #define DN_UH_WUNLOCK() mtx_unlock(&dn_cfg.uh_mtx) 713b3a8eb9SGleb Smirnoff #define DN_UH_LOCK_ASSERT() mtx_assert(&dn_cfg.uh_mtx, MA_OWNED) 723b3a8eb9SGleb Smirnoff #endif 733b3a8eb9SGleb Smirnoff 743b3a8eb9SGleb Smirnoff #define DN_BH_RLOCK() mtx_lock(&dn_cfg.uh_mtx) 753b3a8eb9SGleb Smirnoff #define DN_BH_RUNLOCK() mtx_unlock(&dn_cfg.uh_mtx) 763b3a8eb9SGleb Smirnoff #define DN_BH_WLOCK() mtx_lock(&dn_cfg.uh_mtx) 773b3a8eb9SGleb Smirnoff #define DN_BH_WUNLOCK() mtx_unlock(&dn_cfg.uh_mtx) 783b3a8eb9SGleb Smirnoff #define DN_BH_LOCK_ASSERT() mtx_assert(&dn_cfg.uh_mtx, MA_OWNED) 793b3a8eb9SGleb Smirnoff 803b3a8eb9SGleb Smirnoff SLIST_HEAD(dn_schk_head, dn_schk); 813b3a8eb9SGleb Smirnoff SLIST_HEAD(dn_sch_inst_head, dn_sch_inst); 823b3a8eb9SGleb Smirnoff SLIST_HEAD(dn_fsk_head, dn_fsk); 833b3a8eb9SGleb Smirnoff SLIST_HEAD(dn_queue_head, dn_queue); 843b3a8eb9SGleb Smirnoff SLIST_HEAD(dn_alg_head, dn_alg); 853b3a8eb9SGleb Smirnoff 8691336b40SDon Lewis #ifdef NEW_AQM 8791336b40SDon Lewis SLIST_HEAD(dn_aqm_head, dn_aqm); /* for new AQMs */ 8891336b40SDon Lewis #endif 8991336b40SDon Lewis 903b3a8eb9SGleb Smirnoff struct mq { /* a basic queue of packets*/ 913b3a8eb9SGleb Smirnoff struct mbuf *head, *tail; 9241d10f90SLuigi Rizzo int count; 933b3a8eb9SGleb Smirnoff }; 943b3a8eb9SGleb Smirnoff 953b3a8eb9SGleb Smirnoff static inline void 963b3a8eb9SGleb Smirnoff set_oid(struct dn_id *o, int type, int len) 973b3a8eb9SGleb Smirnoff { 983b3a8eb9SGleb Smirnoff o->type = type; 993b3a8eb9SGleb Smirnoff o->len = len; 1003b3a8eb9SGleb Smirnoff o->subtype = 0; 101352bc63dSLuigi Rizzo } 1023b3a8eb9SGleb Smirnoff 1033b3a8eb9SGleb Smirnoff /* 1043b3a8eb9SGleb Smirnoff * configuration and global data for a dummynet instance 1053b3a8eb9SGleb Smirnoff * 1063b3a8eb9SGleb Smirnoff * When a configuration is modified from userland, 'id' is incremented 1073b3a8eb9SGleb Smirnoff * so we can use the value to check for stale pointers. 1083b3a8eb9SGleb Smirnoff */ 1093b3a8eb9SGleb Smirnoff struct dn_parms { 1103b3a8eb9SGleb Smirnoff uint32_t id; /* configuration version */ 1113b3a8eb9SGleb Smirnoff 1123b3a8eb9SGleb Smirnoff /* defaults (sysctl-accessible) */ 1133b3a8eb9SGleb Smirnoff int red_lookup_depth; 1143b3a8eb9SGleb Smirnoff int red_avg_pkt_size; 1153b3a8eb9SGleb Smirnoff int red_max_pkt_size; 1163b3a8eb9SGleb Smirnoff int hash_size; 1173b3a8eb9SGleb Smirnoff int max_hash_size; 1183b3a8eb9SGleb Smirnoff long byte_limit; /* max queue sizes */ 1193b3a8eb9SGleb Smirnoff long slot_limit; 1203b3a8eb9SGleb Smirnoff 1213b3a8eb9SGleb Smirnoff int io_fast; 1223b3a8eb9SGleb Smirnoff int debug; 1233b3a8eb9SGleb Smirnoff 1243b3a8eb9SGleb Smirnoff /* timekeeping */ 1253b3a8eb9SGleb Smirnoff struct timeval prev_t; /* last time dummynet_tick ran */ 1263b3a8eb9SGleb Smirnoff struct dn_heap evheap; /* scheduled events */ 1273b3a8eb9SGleb Smirnoff 1283b3a8eb9SGleb Smirnoff /* counters of objects -- used for reporting space */ 1293b3a8eb9SGleb Smirnoff int schk_count; 1303b3a8eb9SGleb Smirnoff int si_count; 1313b3a8eb9SGleb Smirnoff int fsk_count; 1323b3a8eb9SGleb Smirnoff int queue_count; 1333b3a8eb9SGleb Smirnoff 1343b3a8eb9SGleb Smirnoff /* ticks and other stuff */ 1353b3a8eb9SGleb Smirnoff uint64_t curr_time; 1363b3a8eb9SGleb Smirnoff /* flowsets and schedulers are in hash tables, with 'hash_size' 1373b3a8eb9SGleb Smirnoff * buckets. fshash is looked up at every packet arrival 1383b3a8eb9SGleb Smirnoff * so better be generous if we expect many entries. 1393b3a8eb9SGleb Smirnoff */ 1403b3a8eb9SGleb Smirnoff struct dn_ht *fshash; 1413b3a8eb9SGleb Smirnoff struct dn_ht *schedhash; 1423b3a8eb9SGleb Smirnoff /* list of flowsets without a scheduler -- use sch_chain */ 1433b3a8eb9SGleb Smirnoff struct dn_fsk_head fsu; /* list of unlinked flowsets */ 1443b3a8eb9SGleb Smirnoff struct dn_alg_head schedlist; /* list of algorithms */ 14591336b40SDon Lewis #ifdef NEW_AQM 14691336b40SDon Lewis struct dn_aqm_head aqmlist; /* list of AQMs */ 14791336b40SDon Lewis #endif 1483b3a8eb9SGleb Smirnoff 1493b3a8eb9SGleb Smirnoff /* Store the fs/sch to scan when draining. The value is the 1503b3a8eb9SGleb Smirnoff * bucket number of the hash table. Expire can be disabled 1513b3a8eb9SGleb Smirnoff * with net.inet.ip.dummynet.expire=0, or it happens every 1523b3a8eb9SGleb Smirnoff * expire ticks. 1533b3a8eb9SGleb Smirnoff **/ 1543b3a8eb9SGleb Smirnoff int drain_fs; 1553b3a8eb9SGleb Smirnoff int drain_sch; 1563b3a8eb9SGleb Smirnoff uint32_t expire; 1573b3a8eb9SGleb Smirnoff uint32_t expire_cycle; /* tick count */ 1583b3a8eb9SGleb Smirnoff 1593b3a8eb9SGleb Smirnoff int init_done; 1603b3a8eb9SGleb Smirnoff 1613b3a8eb9SGleb Smirnoff /* if the upper half is busy doing something long, 1623b3a8eb9SGleb Smirnoff * can set the busy flag and we will enqueue packets in 1633b3a8eb9SGleb Smirnoff * a queue for later processing. 1643b3a8eb9SGleb Smirnoff */ 1653b3a8eb9SGleb Smirnoff int busy; 1663b3a8eb9SGleb Smirnoff struct mq pending; 1673b3a8eb9SGleb Smirnoff 1683b3a8eb9SGleb Smirnoff #ifdef _KERNEL 1693b3a8eb9SGleb Smirnoff /* 1703b3a8eb9SGleb Smirnoff * This file is normally used in the kernel, unless we do 1713b3a8eb9SGleb Smirnoff * some userland tests, in which case we do not need a mtx. 1723b3a8eb9SGleb Smirnoff * uh_mtx arbitrates between system calls and also 1733b3a8eb9SGleb Smirnoff * protects fshash, schedhash and fsunlinked. 1743b3a8eb9SGleb Smirnoff * These structures are readonly for the lower half. 1753b3a8eb9SGleb Smirnoff * bh_mtx protects all other structures which may be 1763b3a8eb9SGleb Smirnoff * modified upon packet arrivals 1773b3a8eb9SGleb Smirnoff */ 1783b3a8eb9SGleb Smirnoff #if defined( __linux__ ) || defined( _WIN32 ) 1793b3a8eb9SGleb Smirnoff spinlock_t uh_mtx; 1803b3a8eb9SGleb Smirnoff spinlock_t bh_mtx; 1813b3a8eb9SGleb Smirnoff #else 1823b3a8eb9SGleb Smirnoff struct mtx uh_mtx; 1833b3a8eb9SGleb Smirnoff struct mtx bh_mtx; 1843b3a8eb9SGleb Smirnoff #endif 1853b3a8eb9SGleb Smirnoff 1863b3a8eb9SGleb Smirnoff #endif /* _KERNEL */ 1873b3a8eb9SGleb Smirnoff }; 1883b3a8eb9SGleb Smirnoff 1893b3a8eb9SGleb Smirnoff /* 1903b3a8eb9SGleb Smirnoff * Delay line, contains all packets on output from a link. 1913b3a8eb9SGleb Smirnoff * Every scheduler instance has one. 1923b3a8eb9SGleb Smirnoff */ 1933b3a8eb9SGleb Smirnoff struct delay_line { 1943b3a8eb9SGleb Smirnoff struct dn_id oid; 1953b3a8eb9SGleb Smirnoff struct dn_sch_inst *si; 1963b3a8eb9SGleb Smirnoff struct mq mq; 1973b3a8eb9SGleb Smirnoff }; 1983b3a8eb9SGleb Smirnoff 1993b3a8eb9SGleb Smirnoff /* 2003b3a8eb9SGleb Smirnoff * The kernel side of a flowset. It is linked in a hash table 2013b3a8eb9SGleb Smirnoff * of flowsets, and in a list of children of their parent scheduler. 2023b3a8eb9SGleb Smirnoff * qht is either the queue or (if HAVE_MASK) a hash table queues. 2033b3a8eb9SGleb Smirnoff * Note that the mask to use is the (flow_mask|sched_mask), which 2043b3a8eb9SGleb Smirnoff * changes as we attach/detach schedulers. So we store it here. 2053b3a8eb9SGleb Smirnoff * 2063b3a8eb9SGleb Smirnoff * XXX If we want to add scheduler-specific parameters, we need to 2073b3a8eb9SGleb Smirnoff * put them in external storage because the scheduler may not be 2083b3a8eb9SGleb Smirnoff * available when the fsk is created. 2093b3a8eb9SGleb Smirnoff */ 2103b3a8eb9SGleb Smirnoff struct dn_fsk { /* kernel side of a flowset */ 2113b3a8eb9SGleb Smirnoff struct dn_fs fs; 2123b3a8eb9SGleb Smirnoff SLIST_ENTRY(dn_fsk) fsk_next; /* hash chain for fshash */ 2133b3a8eb9SGleb Smirnoff 2143b3a8eb9SGleb Smirnoff struct ipfw_flow_id fsk_mask; 2153b3a8eb9SGleb Smirnoff 2163b3a8eb9SGleb Smirnoff /* qht is a hash table of queues, or just a single queue 2173b3a8eb9SGleb Smirnoff * a bit in fs.flags tells us which one 2183b3a8eb9SGleb Smirnoff */ 2193b3a8eb9SGleb Smirnoff struct dn_ht *qht; 2203b3a8eb9SGleb Smirnoff struct dn_schk *sched; /* Sched we are linked to */ 2213b3a8eb9SGleb Smirnoff SLIST_ENTRY(dn_fsk) sch_chain; /* list of fsk attached to sched */ 2223b3a8eb9SGleb Smirnoff 2233b3a8eb9SGleb Smirnoff /* bucket index used by drain routine to drain queues for this 2243b3a8eb9SGleb Smirnoff * flowset 2253b3a8eb9SGleb Smirnoff */ 2263b3a8eb9SGleb Smirnoff int drain_bucket; 2273b3a8eb9SGleb Smirnoff /* Parameter realted to RED / GRED */ 2283b3a8eb9SGleb Smirnoff /* original values are in dn_fs*/ 2293b3a8eb9SGleb Smirnoff int w_q ; /* queue weight (scaled) */ 2303b3a8eb9SGleb Smirnoff int max_th ; /* maximum threshold for queue (scaled) */ 2313b3a8eb9SGleb Smirnoff int min_th ; /* minimum threshold for queue (scaled) */ 2323b3a8eb9SGleb Smirnoff int max_p ; /* maximum value for p_b (scaled) */ 2333b3a8eb9SGleb Smirnoff 2343b3a8eb9SGleb Smirnoff u_int c_1 ; /* max_p/(max_th-min_th) (scaled) */ 2353b3a8eb9SGleb Smirnoff u_int c_2 ; /* max_p*min_th/(max_th-min_th) (scaled) */ 2363b3a8eb9SGleb Smirnoff u_int c_3 ; /* for GRED, (1-max_p)/max_th (scaled) */ 2373b3a8eb9SGleb Smirnoff u_int c_4 ; /* for GRED, 1 - 2*max_p (scaled) */ 2383b3a8eb9SGleb Smirnoff u_int * w_q_lookup ; /* lookup table for computing (1-w_q)^t */ 2393b3a8eb9SGleb Smirnoff u_int lookup_depth ; /* depth of lookup table */ 2403b3a8eb9SGleb Smirnoff int lookup_step ; /* granularity inside the lookup table */ 2413b3a8eb9SGleb Smirnoff int lookup_weight ; /* equal to (1-w_q)^t / (1-w_q)^(t+1) */ 2423b3a8eb9SGleb Smirnoff int avg_pkt_size ; /* medium packet size */ 2433b3a8eb9SGleb Smirnoff int max_pkt_size ; /* max packet size */ 24491336b40SDon Lewis #ifdef NEW_AQM 24591336b40SDon Lewis struct dn_aqm *aqmfp; /* Pointer to AQM functions */ 24691336b40SDon Lewis void *aqmcfg; /* configuration parameters for AQM */ 24791336b40SDon Lewis #endif 2483b3a8eb9SGleb Smirnoff }; 2493b3a8eb9SGleb Smirnoff 2503b3a8eb9SGleb Smirnoff /* 2513b3a8eb9SGleb Smirnoff * A queue is created as a child of a flowset unless it belongs to 2523b3a8eb9SGleb Smirnoff * a !MULTIQUEUE scheduler. It is normally in a hash table in the 2533b3a8eb9SGleb Smirnoff * flowset. fs always points to the parent flowset. 2543b3a8eb9SGleb Smirnoff * si normally points to the sch_inst, unless the flowset has been 2553b3a8eb9SGleb Smirnoff * detached from the scheduler -- in this case si == NULL and we 2563b3a8eb9SGleb Smirnoff * should not enqueue. 2573b3a8eb9SGleb Smirnoff */ 2583b3a8eb9SGleb Smirnoff struct dn_queue { 2593b3a8eb9SGleb Smirnoff struct dn_flow ni; /* oid, flow_id, stats */ 2603b3a8eb9SGleb Smirnoff struct mq mq; /* packets queue */ 2613b3a8eb9SGleb Smirnoff struct dn_sch_inst *_si; /* owner scheduler instance */ 2623b3a8eb9SGleb Smirnoff SLIST_ENTRY(dn_queue) q_next; /* hash chain list for qht */ 2633b3a8eb9SGleb Smirnoff struct dn_fsk *fs; /* parent flowset. */ 2643b3a8eb9SGleb Smirnoff 2653b3a8eb9SGleb Smirnoff /* RED parameters */ 2663b3a8eb9SGleb Smirnoff int avg; /* average queue length est. (scaled) */ 2673b3a8eb9SGleb Smirnoff int count; /* arrivals since last RED drop */ 2683b3a8eb9SGleb Smirnoff int random; /* random value (scaled) */ 2693b3a8eb9SGleb Smirnoff uint64_t q_time; /* start of queue idle time */ 27091336b40SDon Lewis #ifdef NEW_AQM 27191336b40SDon Lewis void *aqm_status; /* per-queue status variables*/ 27291336b40SDon Lewis #endif 2733b3a8eb9SGleb Smirnoff 2743b3a8eb9SGleb Smirnoff }; 2753b3a8eb9SGleb Smirnoff 2763b3a8eb9SGleb Smirnoff /* 2773b3a8eb9SGleb Smirnoff * The kernel side of a scheduler. Contains the userland config, 2783b3a8eb9SGleb Smirnoff * a link, pointer to extra config arguments from command line, 2793b3a8eb9SGleb Smirnoff * kernel flags, and a pointer to the scheduler methods. 2803b3a8eb9SGleb Smirnoff * It is stored in a hash table, and holds a list of all 2813b3a8eb9SGleb Smirnoff * flowsets and scheduler instances. 2823b3a8eb9SGleb Smirnoff * XXX sch must be at the beginning, see schk_hash(). 2833b3a8eb9SGleb Smirnoff */ 2843b3a8eb9SGleb Smirnoff struct dn_schk { 2853b3a8eb9SGleb Smirnoff struct dn_sch sch; 2863b3a8eb9SGleb Smirnoff struct dn_alg *fp; /* Pointer to scheduler functions */ 2873b3a8eb9SGleb Smirnoff struct dn_link link; /* The link, embedded */ 2883b3a8eb9SGleb Smirnoff struct dn_profile *profile; /* delay profile, if any */ 2893b3a8eb9SGleb Smirnoff struct dn_id *cfg; /* extra config arguments */ 2903b3a8eb9SGleb Smirnoff 2913b3a8eb9SGleb Smirnoff SLIST_ENTRY(dn_schk) schk_next; /* hash chain for schedhash */ 2923b3a8eb9SGleb Smirnoff 2933b3a8eb9SGleb Smirnoff struct dn_fsk_head fsk_list; /* all fsk linked to me */ 2943b3a8eb9SGleb Smirnoff struct dn_fsk *fs; /* Flowset for !MULTIQUEUE */ 2953b3a8eb9SGleb Smirnoff 2963b3a8eb9SGleb Smirnoff /* bucket index used by the drain routine to drain the scheduler 2973b3a8eb9SGleb Smirnoff * instance for this flowset. 2983b3a8eb9SGleb Smirnoff */ 2993b3a8eb9SGleb Smirnoff int drain_bucket; 3003b3a8eb9SGleb Smirnoff 3013b3a8eb9SGleb Smirnoff /* Hash table of all instances (through sch.sched_mask) 3023b3a8eb9SGleb Smirnoff * or single instance if no mask. Always valid. 3033b3a8eb9SGleb Smirnoff */ 3043b3a8eb9SGleb Smirnoff struct dn_ht *siht; 3053b3a8eb9SGleb Smirnoff }; 3063b3a8eb9SGleb Smirnoff 3073b3a8eb9SGleb Smirnoff 3083b3a8eb9SGleb Smirnoff /* 3093b3a8eb9SGleb Smirnoff * Scheduler instance. 3103b3a8eb9SGleb Smirnoff * Contains variables and all queues relative to a this instance. 3113b3a8eb9SGleb Smirnoff * This struct is created a runtime. 3123b3a8eb9SGleb Smirnoff */ 3133b3a8eb9SGleb Smirnoff struct dn_sch_inst { 3143b3a8eb9SGleb Smirnoff struct dn_flow ni; /* oid, flowid and stats */ 3153b3a8eb9SGleb Smirnoff SLIST_ENTRY(dn_sch_inst) si_next; /* hash chain for siht */ 3163b3a8eb9SGleb Smirnoff struct delay_line dline; 3173b3a8eb9SGleb Smirnoff struct dn_schk *sched; /* the template */ 3183b3a8eb9SGleb Smirnoff int kflags; /* DN_ACTIVE */ 3193b3a8eb9SGleb Smirnoff 3203b3a8eb9SGleb Smirnoff int64_t credit; /* bits I can transmit (more or less). */ 3213b3a8eb9SGleb Smirnoff uint64_t sched_time; /* time link was scheduled in ready_heap */ 3223b3a8eb9SGleb Smirnoff uint64_t idle_time; /* start of scheduler instance idle time */ 3233b3a8eb9SGleb Smirnoff 3243b3a8eb9SGleb Smirnoff /* q_count is the number of queues that this instance is using. 3253b3a8eb9SGleb Smirnoff * The counter is incremented or decremented when 3263b3a8eb9SGleb Smirnoff * a reference from the queue is created or deleted. 3273b3a8eb9SGleb Smirnoff * It is used to make sure that a scheduler instance can be safely 3283b3a8eb9SGleb Smirnoff * deleted by the drain routine. See notes below. 3293b3a8eb9SGleb Smirnoff */ 3303b3a8eb9SGleb Smirnoff int q_count; 3313b3a8eb9SGleb Smirnoff 3323b3a8eb9SGleb Smirnoff }; 3333b3a8eb9SGleb Smirnoff 3343b3a8eb9SGleb Smirnoff /* 3353b3a8eb9SGleb Smirnoff * NOTE about object drain. 3363b3a8eb9SGleb Smirnoff * The system will automatically (XXX check when) drain queues and 3373b3a8eb9SGleb Smirnoff * scheduler instances when they are idle. 3383b3a8eb9SGleb Smirnoff * A queue is idle when it has no packets; an instance is idle when 3393b3a8eb9SGleb Smirnoff * it is not in the evheap heap, and the corresponding delay line is empty. 3403b3a8eb9SGleb Smirnoff * A queue can be safely deleted when it is idle because of the scheduler 3413b3a8eb9SGleb Smirnoff * function xxx_free_queue() will remove any references to it. 3423b3a8eb9SGleb Smirnoff * An instance can be only deleted when no queues reference it. To be sure 3433b3a8eb9SGleb Smirnoff * of that, a counter (q_count) stores the number of queues that are pointing 3443b3a8eb9SGleb Smirnoff * to the instance. 3453b3a8eb9SGleb Smirnoff * 3463b3a8eb9SGleb Smirnoff * XXX 3473b3a8eb9SGleb Smirnoff * Order of scan: 3483b3a8eb9SGleb Smirnoff * - take all flowset in a bucket for the flowset hash table 3493b3a8eb9SGleb Smirnoff * - take all queues in a bucket for the flowset 3503b3a8eb9SGleb Smirnoff * - increment the queue bucket 3513b3a8eb9SGleb Smirnoff * - scan next flowset bucket 3523b3a8eb9SGleb Smirnoff * Nothing is done if a bucket contains no entries. 3533b3a8eb9SGleb Smirnoff * 3543b3a8eb9SGleb Smirnoff * The same schema is used for sceduler instances 3553b3a8eb9SGleb Smirnoff */ 3563b3a8eb9SGleb Smirnoff 3573b3a8eb9SGleb Smirnoff 3583b3a8eb9SGleb Smirnoff /* kernel-side flags. Linux has DN_DELETE in fcntl.h 3593b3a8eb9SGleb Smirnoff */ 3603b3a8eb9SGleb Smirnoff enum { 3613b3a8eb9SGleb Smirnoff /* 1 and 2 are reserved for the SCAN flags */ 3623b3a8eb9SGleb Smirnoff DN_DESTROY = 0x0004, /* destroy */ 3633b3a8eb9SGleb Smirnoff DN_DELETE_FS = 0x0008, /* destroy flowset */ 3643b3a8eb9SGleb Smirnoff DN_DETACH = 0x0010, 3653b3a8eb9SGleb Smirnoff DN_ACTIVE = 0x0020, /* object is in evheap */ 3663b3a8eb9SGleb Smirnoff DN_F_DLINE = 0x0040, /* object is a delay line */ 3673b3a8eb9SGleb Smirnoff DN_DEL_SAFE = 0x0080, /* delete a queue only if no longer needed 3683b3a8eb9SGleb Smirnoff * by scheduler */ 3693b3a8eb9SGleb Smirnoff DN_QHT_IS_Q = 0x0100, /* in flowset, qht is a single queue */ 3703b3a8eb9SGleb Smirnoff }; 3713b3a8eb9SGleb Smirnoff 3724001fcbeSDon Lewis /* 3734001fcbeSDon Lewis * Packets processed by dummynet have an mbuf tag associated with 3744001fcbeSDon Lewis * them that carries their dummynet state. 3754001fcbeSDon Lewis * Outside dummynet, only the 'rule' field is relevant, and it must 3764001fcbeSDon Lewis * be at the beginning of the structure. 3774001fcbeSDon Lewis */ 3784001fcbeSDon Lewis struct dn_pkt_tag { 3794001fcbeSDon Lewis struct ipfw_rule_ref rule; /* matching rule */ 3804001fcbeSDon Lewis 3814001fcbeSDon Lewis /* second part, dummynet specific */ 3824001fcbeSDon Lewis int dn_dir; /* action when packet comes out.*/ 3834001fcbeSDon Lewis /* see ip_fw_private.h */ 3844001fcbeSDon Lewis uint64_t output_time; /* when the pkt is due for delivery*/ 3854001fcbeSDon Lewis struct ifnet *ifp; /* interface, for ip_output */ 3864001fcbeSDon Lewis struct _ip6dn_args ip6opt; /* XXX ipv6 options */ 3874001fcbeSDon Lewis uint16_t iphdr_off; /* IP header offset for mtodo() */ 3884001fcbeSDon Lewis }; 3894001fcbeSDon Lewis 3903b3a8eb9SGleb Smirnoff extern struct dn_parms dn_cfg; 3913b3a8eb9SGleb Smirnoff //VNET_DECLARE(struct dn_parms, _base_dn_cfg); 3923b3a8eb9SGleb Smirnoff //#define dn_cfg VNET(_base_dn_cfg) 3933b3a8eb9SGleb Smirnoff 3943b3a8eb9SGleb Smirnoff int dummynet_io(struct mbuf **, int , struct ip_fw_args *); 3953b3a8eb9SGleb Smirnoff void dummynet_task(void *context, int pending); 3963b3a8eb9SGleb Smirnoff void dn_reschedule(void); 3974001fcbeSDon Lewis struct dn_pkt_tag * dn_tag_get(struct mbuf *m); 3983b3a8eb9SGleb Smirnoff 3993b3a8eb9SGleb Smirnoff struct dn_queue *ipdn_q_find(struct dn_fsk *, struct dn_sch_inst *, 4003b3a8eb9SGleb Smirnoff struct ipfw_flow_id *); 4013b3a8eb9SGleb Smirnoff struct dn_sch_inst *ipdn_si_find(struct dn_schk *, struct ipfw_flow_id *); 4023b3a8eb9SGleb Smirnoff 4033b3a8eb9SGleb Smirnoff /* 4043b3a8eb9SGleb Smirnoff * copy_range is a template for requests for ranges of pipes/queues/scheds. 4053b3a8eb9SGleb Smirnoff * The number of ranges is variable and can be derived by o.len. 4063b3a8eb9SGleb Smirnoff * As a default, we use a small number of entries so that the struct 4073b3a8eb9SGleb Smirnoff * fits easily on the stack and is sufficient for most common requests. 4083b3a8eb9SGleb Smirnoff */ 4093b3a8eb9SGleb Smirnoff #define DEFAULT_RANGES 5 4103b3a8eb9SGleb Smirnoff struct copy_range { 4113b3a8eb9SGleb Smirnoff struct dn_id o; 4123b3a8eb9SGleb Smirnoff uint32_t r[ 2 * DEFAULT_RANGES ]; 4133b3a8eb9SGleb Smirnoff }; 4143b3a8eb9SGleb Smirnoff 4153b3a8eb9SGleb Smirnoff struct copy_args { 4163b3a8eb9SGleb Smirnoff char **start; 4173b3a8eb9SGleb Smirnoff char *end; 4183b3a8eb9SGleb Smirnoff int flags; 4193b3a8eb9SGleb Smirnoff int type; 4203b3a8eb9SGleb Smirnoff struct copy_range *extra; /* extra filtering */ 4213b3a8eb9SGleb Smirnoff }; 4223b3a8eb9SGleb Smirnoff 4233b3a8eb9SGleb Smirnoff struct sockopt; 4243b3a8eb9SGleb Smirnoff int ip_dummynet_compat(struct sockopt *sopt); 4253b3a8eb9SGleb Smirnoff int dummynet_get(struct sockopt *sopt, void **compat); 4263b3a8eb9SGleb Smirnoff int dn_c_copy_q (void *_ni, void *arg); 4273b3a8eb9SGleb Smirnoff int dn_c_copy_pipe(struct dn_schk *s, struct copy_args *a, int nq); 4283b3a8eb9SGleb Smirnoff int dn_c_copy_fs(struct dn_fsk *f, struct copy_args *a, int nq); 4293b3a8eb9SGleb Smirnoff int dn_compat_copy_queue(struct copy_args *a, void *_o); 4303b3a8eb9SGleb Smirnoff int dn_compat_copy_pipe(struct copy_args *a, void *_o); 4313b3a8eb9SGleb Smirnoff int copy_data_helper_compat(void *_o, void *_arg); 4323b3a8eb9SGleb Smirnoff int dn_compat_calc_size(void); 4333b3a8eb9SGleb Smirnoff int do_config(void *p, int l); 4343b3a8eb9SGleb Smirnoff 4353b3a8eb9SGleb Smirnoff /* function to drain idle object */ 4363b3a8eb9SGleb Smirnoff void dn_drain_scheduler(void); 4373b3a8eb9SGleb Smirnoff void dn_drain_queue(void); 4383b3a8eb9SGleb Smirnoff 43991336b40SDon Lewis #ifdef NEW_AQM 44091336b40SDon Lewis int ecn_mark(struct mbuf* m); 44191336b40SDon Lewis 44291336b40SDon Lewis /* moved from ip_dn_io.c to here to be available for AQMs modules*/ 44391336b40SDon Lewis static inline void 44491336b40SDon Lewis mq_append(struct mq *q, struct mbuf *m) 44591336b40SDon Lewis { 44691336b40SDon Lewis #ifdef USERSPACE 44791336b40SDon Lewis // buffers from netmap need to be copied 44891336b40SDon Lewis // XXX note that the routine is not expected to fail 44991336b40SDon Lewis ND("append %p to %p", m, q); 45091336b40SDon Lewis if (m->m_flags & M_STACK) { 45191336b40SDon Lewis struct mbuf *m_new; 45291336b40SDon Lewis void *p; 45391336b40SDon Lewis int l, ofs; 45491336b40SDon Lewis 45591336b40SDon Lewis ofs = m->m_data - m->__m_extbuf; 45691336b40SDon Lewis // XXX allocate 45791336b40SDon Lewis MGETHDR(m_new, M_NOWAIT, MT_DATA); 45891336b40SDon Lewis ND("*** WARNING, volatile buf %p ext %p %d dofs %d m_new %p", 45991336b40SDon Lewis m, m->__m_extbuf, m->__m_extlen, ofs, m_new); 46091336b40SDon Lewis p = m_new->__m_extbuf; /* new pointer */ 46191336b40SDon Lewis l = m_new->__m_extlen; /* new len */ 46291336b40SDon Lewis if (l <= m->__m_extlen) { 46391336b40SDon Lewis panic("extlen too large"); 46491336b40SDon Lewis } 46591336b40SDon Lewis 46691336b40SDon Lewis *m_new = *m; // copy 46791336b40SDon Lewis m_new->m_flags &= ~M_STACK; 46891336b40SDon Lewis m_new->__m_extbuf = p; // point to new buffer 46991336b40SDon Lewis _pkt_copy(m->__m_extbuf, p, m->__m_extlen); 47091336b40SDon Lewis m_new->m_data = p + ofs; 47191336b40SDon Lewis m = m_new; 47291336b40SDon Lewis } 47391336b40SDon Lewis #endif /* USERSPACE */ 47491336b40SDon Lewis if (q->head == NULL) 47591336b40SDon Lewis q->head = m; 47691336b40SDon Lewis else 47791336b40SDon Lewis q->tail->m_nextpkt = m; 47891336b40SDon Lewis q->count++; 47991336b40SDon Lewis q->tail = m; 48091336b40SDon Lewis m->m_nextpkt = NULL; 48191336b40SDon Lewis } 48291336b40SDon Lewis #endif /* NEW_AQM */ 48391336b40SDon Lewis 4843b3a8eb9SGleb Smirnoff #endif /* _IP_DN_PRIVATE_H */ 485