1#!/bin/bash
2#
3# Multiqueue: Using pktgen threads for sending on multiple CPUs
4#  * adding devices to kernel threads which are in the same NUMA node
5#  * bound devices queue's irq affinity to the threads, 1:1 mapping
6#  * notice the naming scheme for keeping device names unique
7#  * nameing scheme: dev@thread_number
8#  * flow variation via random UDP source port
9#
10basedir=`dirname $0`
11source ${basedir}/functions.sh
12root_check_run_with_sudo "$@"
13#
14# Required param: -i dev in $DEV
15source ${basedir}/parameters.sh
16
17# Base Config
18DELAY="0"        # Zero means max speed
19[ -z "$COUNT" ]     && COUNT="20000000"   # Zero means indefinitely
20[ -z "$CLONE_SKB" ] && CLONE_SKB="0"
21
22# Flow variation random source port between min and max
23UDP_MIN=9
24UDP_MAX=109
25
26node=`get_iface_node $DEV`
27irq_array=(`get_iface_irqs $DEV`)
28cpu_array=(`get_node_cpus $node`)
29
30[ $THREADS -gt ${#irq_array[*]} -o $THREADS -gt ${#cpu_array[*]}  ] && \
31	err 1 "Thread number $THREADS exceeds: min (${#irq_array[*]},${#cpu_array[*]})"
32
33# (example of setting default params in your script)
34if [ -z "$DEST_IP" ]; then
35    [ -z "$IP6" ] && DEST_IP="198.18.0.42" || DEST_IP="FD00::1"
36fi
37[ -z "$DST_MAC" ] && DST_MAC="90:e2:ba:ff:ff:ff"
38
39# General cleanup everything since last run
40pg_ctrl "reset"
41
42# Threads are specified with parameter -t value in $THREADS
43for ((i = 0; i < $THREADS; i++)); do
44    # The device name is extended with @name, using thread number to
45    # make then unique, but any name will do.
46    # Set the queue's irq affinity to this $thread (processor)
47    # if '-f' is designated, offset cpu id
48    thread=${cpu_array[$((i+F_THREAD))]}
49    dev=${DEV}@${thread}
50    echo $thread > /proc/irq/${irq_array[$i]}/smp_affinity_list
51    info "irq ${irq_array[$i]} is set affinity to `cat /proc/irq/${irq_array[$i]}/smp_affinity_list`"
52
53    # Add remove all other devices and add_device $dev to thread
54    pg_thread $thread "rem_device_all"
55    pg_thread $thread "add_device" $dev
56
57    # select queue and bind the queue and $dev in 1:1 relationship
58    queue_num=$i
59    info "queue number is $queue_num"
60    pg_set $dev "queue_map_min $queue_num"
61    pg_set $dev "queue_map_max $queue_num"
62
63    # Notice config queue to map to cpu (mirrors smp_processor_id())
64    # It is beneficial to map IRQ /proc/irq/*/smp_affinity 1:1 to CPU number
65    pg_set $dev "flag QUEUE_MAP_CPU"
66
67    # Base config of dev
68    pg_set $dev "count $COUNT"
69    pg_set $dev "clone_skb $CLONE_SKB"
70    pg_set $dev "pkt_size $PKT_SIZE"
71    pg_set $dev "delay $DELAY"
72
73    # Flag example disabling timestamping
74    pg_set $dev "flag NO_TIMESTAMP"
75
76    # Destination
77    pg_set $dev "dst_mac $DST_MAC"
78    pg_set $dev "dst$IP6 $DEST_IP"
79
80    # Setup random UDP port src range
81    pg_set $dev "flag UDPSRC_RND"
82    pg_set $dev "udp_src_min $UDP_MIN"
83    pg_set $dev "udp_src_max $UDP_MAX"
84done
85
86# start_run
87echo "Running... ctrl^C to stop" >&2
88pg_ctrl "start"
89echo "Done" >&2
90
91# Print results
92for ((i = 0; i < $THREADS; i++)); do
93    thread=${cpu_array[$((i+F_THREAD))]}
94    dev=${DEV}@${thread}
95    echo "Device: $dev"
96    cat /proc/net/pktgen/$dev | grep -A2 "Result:"
97done
98