18aa0cb00SJakub Kicinski /* 28aa0cb00SJakub Kicinski * Copyright (C) 2017 Netronome Systems, Inc. 38aa0cb00SJakub Kicinski * 48aa0cb00SJakub Kicinski * This software is dual licensed under the GNU General License Version 2, 58aa0cb00SJakub Kicinski * June 1991 as shown in the file COPYING in the top-level directory of this 68aa0cb00SJakub Kicinski * source tree or the BSD 2-Clause License provided below. You have the 78aa0cb00SJakub Kicinski * option to license this software under the complete terms of either license. 88aa0cb00SJakub Kicinski * 98aa0cb00SJakub Kicinski * The BSD 2-Clause License: 108aa0cb00SJakub Kicinski * 118aa0cb00SJakub Kicinski * Redistribution and use in source and binary forms, with or 128aa0cb00SJakub Kicinski * without modification, are permitted provided that the following 138aa0cb00SJakub Kicinski * conditions are met: 148aa0cb00SJakub Kicinski * 158aa0cb00SJakub Kicinski * 1. Redistributions of source code must retain the above 168aa0cb00SJakub Kicinski * copyright notice, this list of conditions and the following 178aa0cb00SJakub Kicinski * disclaimer. 188aa0cb00SJakub Kicinski * 198aa0cb00SJakub Kicinski * 2. Redistributions in binary form must reproduce the above 208aa0cb00SJakub Kicinski * copyright notice, this list of conditions and the following 218aa0cb00SJakub Kicinski * disclaimer in the documentation and/or other materials 228aa0cb00SJakub Kicinski * provided with the distribution. 238aa0cb00SJakub Kicinski * 248aa0cb00SJakub Kicinski * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 258aa0cb00SJakub Kicinski * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 268aa0cb00SJakub Kicinski * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 278aa0cb00SJakub Kicinski * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 288aa0cb00SJakub Kicinski * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 298aa0cb00SJakub Kicinski * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 308aa0cb00SJakub Kicinski * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 318aa0cb00SJakub Kicinski * SOFTWARE. 328aa0cb00SJakub Kicinski */ 338aa0cb00SJakub Kicinski 34bb45e51cSJakub Kicinski #include <net/pkt_cls.h> 35bb45e51cSJakub Kicinski 368aa0cb00SJakub Kicinski #include "../nfpcore/nfp_cpp.h" 3777a844eeSJakub Kicinski #include "../nfpcore/nfp_nffw.h" 388aa0cb00SJakub Kicinski #include "../nfp_app.h" 398aa0cb00SJakub Kicinski #include "../nfp_main.h" 408aa0cb00SJakub Kicinski #include "../nfp_net.h" 418aa0cb00SJakub Kicinski #include "../nfp_port.h" 420d49eaf4SJakub Kicinski #include "fw.h" 43bb45e51cSJakub Kicinski #include "main.h" 44bb45e51cSJakub Kicinski 45bb45e51cSJakub Kicinski static bool nfp_net_ebpf_capable(struct nfp_net *nn) 46bb45e51cSJakub Kicinski { 470f6cf4ddSJakub Kicinski #ifdef __LITTLE_ENDIAN 48bb45e51cSJakub Kicinski if (nn->cap & NFP_NET_CFG_CTRL_BPF && 49bb45e51cSJakub Kicinski nn_readb(nn, NFP_NET_CFG_BPF_ABI) == NFP_NET_BPF_ABI) 50bb45e51cSJakub Kicinski return true; 510f6cf4ddSJakub Kicinski #endif 52bb45e51cSJakub Kicinski return false; 53bb45e51cSJakub Kicinski } 54bb45e51cSJakub Kicinski 55bb45e51cSJakub Kicinski static int 56bb45e51cSJakub Kicinski nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn, 57bb45e51cSJakub Kicinski struct bpf_prog *prog) 58bb45e51cSJakub Kicinski { 599ce7a956SJakub Kicinski bool running, xdp_running; 60bb45e51cSJakub Kicinski int ret; 61bb45e51cSJakub Kicinski 62bb45e51cSJakub Kicinski if (!nfp_net_ebpf_capable(nn)) 63bb45e51cSJakub Kicinski return -EINVAL; 64bb45e51cSJakub Kicinski 659ce7a956SJakub Kicinski running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF; 669ce7a956SJakub Kicinski xdp_running = running && nn->dp.bpf_offload_xdp; 67bb45e51cSJakub Kicinski 689ce7a956SJakub Kicinski if (!prog && !xdp_running) 699ce7a956SJakub Kicinski return 0; 709ce7a956SJakub Kicinski if (prog && running && !xdp_running) 719ce7a956SJakub Kicinski return -EBUSY; 729ce7a956SJakub Kicinski 73e4a91cd5SJakub Kicinski ret = nfp_net_bpf_offload(nn, prog, running); 74bb45e51cSJakub Kicinski /* Stop offload if replace not possible */ 759ce7a956SJakub Kicinski if (ret && prog) 76bb45e51cSJakub Kicinski nfp_bpf_xdp_offload(app, nn, NULL); 779ce7a956SJakub Kicinski 78bb45e51cSJakub Kicinski nn->dp.bpf_offload_xdp = prog && !ret; 79bb45e51cSJakub Kicinski return ret; 80bb45e51cSJakub Kicinski } 81bb45e51cSJakub Kicinski 82bb45e51cSJakub Kicinski static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn) 83bb45e51cSJakub Kicinski { 84bb45e51cSJakub Kicinski return nfp_net_ebpf_capable(nn) ? "BPF" : ""; 85bb45e51cSJakub Kicinski } 868aa0cb00SJakub Kicinski 8790d97315SJiri Pirko static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type, 8890d97315SJiri Pirko void *type_data, void *cb_priv) 89bb45e51cSJakub Kicinski { 90de4784caSJiri Pirko struct tc_cls_bpf_offload *cls_bpf = type_data; 9190d97315SJiri Pirko struct nfp_net *nn = cb_priv; 92bb45e51cSJakub Kicinski 939ce7a956SJakub Kicinski if (type != TC_SETUP_CLSBPF || 949ce7a956SJakub Kicinski !tc_can_offload(nn->dp.netdev) || 959ce7a956SJakub Kicinski !nfp_net_ebpf_capable(nn) || 965fd9fc4eSJiri Pirko cls_bpf->common.protocol != htons(ETH_P_ALL) || 975fd9fc4eSJiri Pirko cls_bpf->common.chain_index) 98bb45e51cSJakub Kicinski return -EOPNOTSUPP; 99f449657fSJakub Kicinski if (nn->dp.bpf_offload_xdp) 100f449657fSJakub Kicinski return -EBUSY; 101f449657fSJakub Kicinski 102012bb8a8SJakub Kicinski /* Only support TC direct action */ 103012bb8a8SJakub Kicinski if (!cls_bpf->exts_integrated || 104012bb8a8SJakub Kicinski tcf_exts_has_actions(cls_bpf->exts)) { 105012bb8a8SJakub Kicinski nn_err(nn, "only direct action with no legacy actions supported\n"); 106012bb8a8SJakub Kicinski return -EOPNOTSUPP; 107012bb8a8SJakub Kicinski } 108012bb8a8SJakub Kicinski 1099ce7a956SJakub Kicinski switch (cls_bpf->command) { 1109ce7a956SJakub Kicinski case TC_CLSBPF_REPLACE: 111e4a91cd5SJakub Kicinski return nfp_net_bpf_offload(nn, cls_bpf->prog, true); 1129ce7a956SJakub Kicinski case TC_CLSBPF_ADD: 113e4a91cd5SJakub Kicinski return nfp_net_bpf_offload(nn, cls_bpf->prog, false); 1149ce7a956SJakub Kicinski case TC_CLSBPF_DESTROY: 115e4a91cd5SJakub Kicinski return nfp_net_bpf_offload(nn, NULL, true); 11690d97315SJiri Pirko default: 11790d97315SJiri Pirko return -EOPNOTSUPP; 11890d97315SJiri Pirko } 11990d97315SJiri Pirko } 12090d97315SJiri Pirko 12190d97315SJiri Pirko static int nfp_bpf_setup_tc_block(struct net_device *netdev, 12290d97315SJiri Pirko struct tc_block_offload *f) 12390d97315SJiri Pirko { 12490d97315SJiri Pirko struct nfp_net *nn = netdev_priv(netdev); 12590d97315SJiri Pirko 12690d97315SJiri Pirko if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 12790d97315SJiri Pirko return -EOPNOTSUPP; 12890d97315SJiri Pirko 12990d97315SJiri Pirko switch (f->command) { 13090d97315SJiri Pirko case TC_BLOCK_BIND: 13190d97315SJiri Pirko return tcf_block_cb_register(f->block, 13290d97315SJiri Pirko nfp_bpf_setup_tc_block_cb, 13390d97315SJiri Pirko nn, nn); 13490d97315SJiri Pirko case TC_BLOCK_UNBIND: 13590d97315SJiri Pirko tcf_block_cb_unregister(f->block, 13690d97315SJiri Pirko nfp_bpf_setup_tc_block_cb, 13790d97315SJiri Pirko nn); 13890d97315SJiri Pirko return 0; 13990d97315SJiri Pirko default: 14090d97315SJiri Pirko return -EOPNOTSUPP; 14190d97315SJiri Pirko } 14290d97315SJiri Pirko } 14390d97315SJiri Pirko 14490d97315SJiri Pirko static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev, 14590d97315SJiri Pirko enum tc_setup_type type, void *type_data) 14690d97315SJiri Pirko { 14790d97315SJiri Pirko switch (type) { 14890d97315SJiri Pirko case TC_SETUP_BLOCK: 14990d97315SJiri Pirko return nfp_bpf_setup_tc_block(netdev, type_data); 15090d97315SJiri Pirko default: 15190d97315SJiri Pirko return -EOPNOTSUPP; 15290d97315SJiri Pirko } 153bb45e51cSJakub Kicinski } 154bb45e51cSJakub Kicinski 155bb45e51cSJakub Kicinski static bool nfp_bpf_tc_busy(struct nfp_app *app, struct nfp_net *nn) 156bb45e51cSJakub Kicinski { 157bb45e51cSJakub Kicinski return nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF; 158bb45e51cSJakub Kicinski } 159bb45e51cSJakub Kicinski 1600d49eaf4SJakub Kicinski static int 1610d49eaf4SJakub Kicinski nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value, 1620d49eaf4SJakub Kicinski u32 length) 1630d49eaf4SJakub Kicinski { 1640d49eaf4SJakub Kicinski struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value; 1650d49eaf4SJakub Kicinski struct nfp_cpp *cpp = bpf->app->pf->cpp; 1660d49eaf4SJakub Kicinski 1670d49eaf4SJakub Kicinski if (length < sizeof(*cap)) { 1680d49eaf4SJakub Kicinski nfp_err(cpp, "truncated adjust_head TLV: %d\n", length); 1690d49eaf4SJakub Kicinski return -EINVAL; 1700d49eaf4SJakub Kicinski } 1710d49eaf4SJakub Kicinski 1720d49eaf4SJakub Kicinski bpf->adjust_head.flags = readl(&cap->flags); 1730d49eaf4SJakub Kicinski bpf->adjust_head.off_min = readl(&cap->off_min); 1740d49eaf4SJakub Kicinski bpf->adjust_head.off_max = readl(&cap->off_max); 1758231f844SJakub Kicinski bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub); 1768231f844SJakub Kicinski bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add); 1770d49eaf4SJakub Kicinski 1780d49eaf4SJakub Kicinski if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) { 1790d49eaf4SJakub Kicinski nfp_err(cpp, "invalid adjust_head TLV: min > max\n"); 1800d49eaf4SJakub Kicinski return -EINVAL; 1810d49eaf4SJakub Kicinski } 1820d49eaf4SJakub Kicinski if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) || 1830d49eaf4SJakub Kicinski !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) { 1840d49eaf4SJakub Kicinski nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n"); 1850d49eaf4SJakub Kicinski memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head)); 1860d49eaf4SJakub Kicinski return 0; 1870d49eaf4SJakub Kicinski } 1880d49eaf4SJakub Kicinski 1890d49eaf4SJakub Kicinski return 0; 1900d49eaf4SJakub Kicinski } 1910d49eaf4SJakub Kicinski 19277a844eeSJakub Kicinski static int nfp_bpf_parse_capabilities(struct nfp_app *app) 19377a844eeSJakub Kicinski { 19477a844eeSJakub Kicinski struct nfp_cpp *cpp = app->pf->cpp; 19577a844eeSJakub Kicinski struct nfp_cpp_area *area; 19677a844eeSJakub Kicinski u8 __iomem *mem, *start; 19777a844eeSJakub Kicinski 19877a844eeSJakub Kicinski mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap", 19977a844eeSJakub Kicinski 8, &area); 20077a844eeSJakub Kicinski if (IS_ERR(mem)) 20177a844eeSJakub Kicinski return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem); 20277a844eeSJakub Kicinski 20377a844eeSJakub Kicinski start = mem; 20477a844eeSJakub Kicinski while (mem - start + 8 < nfp_cpp_area_size(area)) { 2050d49eaf4SJakub Kicinski u8 __iomem *value; 20677a844eeSJakub Kicinski u32 type, length; 20777a844eeSJakub Kicinski 20877a844eeSJakub Kicinski type = readl(mem); 20977a844eeSJakub Kicinski length = readl(mem + 4); 2100d49eaf4SJakub Kicinski value = mem + 8; 21177a844eeSJakub Kicinski 21277a844eeSJakub Kicinski mem += 8 + length; 21377a844eeSJakub Kicinski if (mem - start > nfp_cpp_area_size(area)) 21477a844eeSJakub Kicinski goto err_release_free; 21577a844eeSJakub Kicinski 21677a844eeSJakub Kicinski switch (type) { 2170d49eaf4SJakub Kicinski case NFP_BPF_CAP_TYPE_ADJUST_HEAD: 2180d49eaf4SJakub Kicinski if (nfp_bpf_parse_cap_adjust_head(app->priv, value, 2190d49eaf4SJakub Kicinski length)) 2200d49eaf4SJakub Kicinski goto err_release_free; 2210d49eaf4SJakub Kicinski break; 22277a844eeSJakub Kicinski default: 22377a844eeSJakub Kicinski nfp_dbg(cpp, "unknown BPF capability: %d\n", type); 22477a844eeSJakub Kicinski break; 22577a844eeSJakub Kicinski } 22677a844eeSJakub Kicinski } 22777a844eeSJakub Kicinski if (mem - start != nfp_cpp_area_size(area)) { 228*0bce7c9aSJakub Kicinski nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n", 22977a844eeSJakub Kicinski mem - start, nfp_cpp_area_size(area)); 23077a844eeSJakub Kicinski goto err_release_free; 23177a844eeSJakub Kicinski } 23277a844eeSJakub Kicinski 23377a844eeSJakub Kicinski nfp_cpp_area_release_free(area); 23477a844eeSJakub Kicinski 23577a844eeSJakub Kicinski return 0; 23677a844eeSJakub Kicinski 23777a844eeSJakub Kicinski err_release_free: 238*0bce7c9aSJakub Kicinski nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start); 23977a844eeSJakub Kicinski nfp_cpp_area_release_free(area); 24077a844eeSJakub Kicinski return -EINVAL; 24177a844eeSJakub Kicinski } 24277a844eeSJakub Kicinski 24377a844eeSJakub Kicinski static int nfp_bpf_init(struct nfp_app *app) 24477a844eeSJakub Kicinski { 24577a844eeSJakub Kicinski struct nfp_app_bpf *bpf; 24677a844eeSJakub Kicinski int err; 24777a844eeSJakub Kicinski 24877a844eeSJakub Kicinski bpf = kzalloc(sizeof(*bpf), GFP_KERNEL); 24977a844eeSJakub Kicinski if (!bpf) 25077a844eeSJakub Kicinski return -ENOMEM; 25177a844eeSJakub Kicinski bpf->app = app; 25277a844eeSJakub Kicinski app->priv = bpf; 25377a844eeSJakub Kicinski 25477a844eeSJakub Kicinski err = nfp_bpf_parse_capabilities(app); 25577a844eeSJakub Kicinski if (err) 25677a844eeSJakub Kicinski goto err_free_bpf; 25777a844eeSJakub Kicinski 25877a844eeSJakub Kicinski return 0; 25977a844eeSJakub Kicinski 26077a844eeSJakub Kicinski err_free_bpf: 26177a844eeSJakub Kicinski kfree(bpf); 26277a844eeSJakub Kicinski return err; 26377a844eeSJakub Kicinski } 26477a844eeSJakub Kicinski 26577a844eeSJakub Kicinski static void nfp_bpf_clean(struct nfp_app *app) 26677a844eeSJakub Kicinski { 26777a844eeSJakub Kicinski kfree(app->priv); 26877a844eeSJakub Kicinski } 26977a844eeSJakub Kicinski 2708aa0cb00SJakub Kicinski const struct nfp_app_type app_bpf = { 2718aa0cb00SJakub Kicinski .id = NFP_APP_BPF_NIC, 2722707d6f1SJakub Kicinski .name = "ebpf", 2738aa0cb00SJakub Kicinski 27477a844eeSJakub Kicinski .init = nfp_bpf_init, 27577a844eeSJakub Kicinski .clean = nfp_bpf_clean, 27677a844eeSJakub Kicinski 277bb45e51cSJakub Kicinski .extra_cap = nfp_bpf_extra_cap, 278bb45e51cSJakub Kicinski 279012bb8a8SJakub Kicinski .vnic_alloc = nfp_app_nic_vnic_alloc, 280bb45e51cSJakub Kicinski 281bb45e51cSJakub Kicinski .setup_tc = nfp_bpf_setup_tc, 282bb45e51cSJakub Kicinski .tc_busy = nfp_bpf_tc_busy, 283bb45e51cSJakub Kicinski .xdp_offload = nfp_bpf_xdp_offload, 284c6c580d7SJakub Kicinski 285c6c580d7SJakub Kicinski .bpf_verifier_prep = nfp_bpf_verifier_prep, 286c6c580d7SJakub Kicinski .bpf_translate = nfp_bpf_translate, 287c6c580d7SJakub Kicinski .bpf_destroy = nfp_bpf_destroy, 2888aa0cb00SJakub Kicinski }; 289