154b96380SRuslan Bukin /*-
254b96380SRuslan Bukin * SPDX-License-Identifier: BSD-2-Clause
354b96380SRuslan Bukin *
454b96380SRuslan Bukin * Copyright (c) 2022 Ruslan Bukin <br@bsdpad.com>
5d220b1cfSCristian Marussi * Copyright (c) 2023 Arm Ltd
654b96380SRuslan Bukin *
754b96380SRuslan Bukin * This work was supported by Innovate UK project 105694, "Digital Security
854b96380SRuslan Bukin * by Design (DSbD) Technology Platform Prototype".
954b96380SRuslan Bukin *
1054b96380SRuslan Bukin * Redistribution and use in source and binary forms, with or without
1154b96380SRuslan Bukin * modification, are permitted provided that the following conditions
1254b96380SRuslan Bukin * are met:
1354b96380SRuslan Bukin * 1. Redistributions of source code must retain the above copyright
1454b96380SRuslan Bukin * notice, this list of conditions and the following disclaimer.
1554b96380SRuslan Bukin * 2. Redistributions in binary form must reproduce the above copyright
1654b96380SRuslan Bukin * notice, this list of conditions and the following disclaimer in the
1754b96380SRuslan Bukin * documentation and/or other materials provided with the distribution.
1854b96380SRuslan Bukin *
1954b96380SRuslan Bukin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
2054b96380SRuslan Bukin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
2154b96380SRuslan Bukin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2254b96380SRuslan Bukin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
2354b96380SRuslan Bukin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
2454b96380SRuslan Bukin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
2554b96380SRuslan Bukin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
2654b96380SRuslan Bukin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2754b96380SRuslan Bukin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2854b96380SRuslan Bukin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
2954b96380SRuslan Bukin * SUCH DAMAGE.
3054b96380SRuslan Bukin */
3154b96380SRuslan Bukin
3254b96380SRuslan Bukin #include <sys/param.h>
3354b96380SRuslan Bukin #include <sys/systm.h>
343595f18fSCristian Marussi #include <sys/_bitset.h>
353595f18fSCristian Marussi #include <sys/bitset.h>
3654b96380SRuslan Bukin #include <sys/bus.h>
3754b96380SRuslan Bukin #include <sys/cpu.h>
383595f18fSCristian Marussi #include <sys/endian.h>
3954b96380SRuslan Bukin #include <sys/kernel.h>
4054b96380SRuslan Bukin #include <sys/lock.h>
413595f18fSCristian Marussi #include <sys/malloc.h>
4254b96380SRuslan Bukin #include <sys/module.h>
4354b96380SRuslan Bukin #include <sys/mutex.h>
443595f18fSCristian Marussi #include <sys/queue.h>
4535f93203SCristian Marussi #include <sys/refcount.h>
4654b96380SRuslan Bukin
47be82b3a0SEmmanuel Vadot #include <dev/clk/clk.h>
4854b96380SRuslan Bukin #include <dev/fdt/simplebus.h>
4954b96380SRuslan Bukin #include <dev/fdt/fdt_common.h>
5054b96380SRuslan Bukin #include <dev/ofw/ofw_bus_subr.h>
5154b96380SRuslan Bukin
5254b96380SRuslan Bukin #include "scmi.h"
5354b96380SRuslan Bukin #include "scmi_protocols.h"
5454b96380SRuslan Bukin
553595f18fSCristian Marussi #define SCMI_MAX_TOKEN 1024
563595f18fSCristian Marussi
57d220b1cfSCristian Marussi #define SCMI_HDR_TOKEN_S 18
58d220b1cfSCristian Marussi #define SCMI_HDR_TOKEN_BF (0x3fff)
59d220b1cfSCristian Marussi #define SCMI_HDR_TOKEN_M (SCMI_HDR_TOKEN_BF << SCMI_HDR_TOKEN_S)
6054b96380SRuslan Bukin
61d220b1cfSCristian Marussi #define SCMI_HDR_PROTOCOL_ID_S 10
62d220b1cfSCristian Marussi #define SCMI_HDR_PROTOCOL_ID_BF (0xff)
63d220b1cfSCristian Marussi #define SCMI_HDR_PROTOCOL_ID_M \
64d220b1cfSCristian Marussi (SCMI_HDR_PROTOCOL_ID_BF << SCMI_HDR_PROTOCOL_ID_S)
6554b96380SRuslan Bukin
66d220b1cfSCristian Marussi #define SCMI_HDR_MESSAGE_TYPE_S 8
67d220b1cfSCristian Marussi #define SCMI_HDR_MESSAGE_TYPE_BF (0x3)
68d220b1cfSCristian Marussi #define SCMI_HDR_MESSAGE_TYPE_M \
69d220b1cfSCristian Marussi (SCMI_HDR_MESSAGE_TYPE_BF << SCMI_HDR_MESSAGE_TYPE_S)
7054b96380SRuslan Bukin
71d220b1cfSCristian Marussi #define SCMI_HDR_MESSAGE_ID_S 0
72d220b1cfSCristian Marussi #define SCMI_HDR_MESSAGE_ID_BF (0xff)
73d220b1cfSCristian Marussi #define SCMI_HDR_MESSAGE_ID_M \
74d220b1cfSCristian Marussi (SCMI_HDR_MESSAGE_ID_BF << SCMI_HDR_MESSAGE_ID_S)
7554b96380SRuslan Bukin
76d220b1cfSCristian Marussi #define SCMI_MSG_TYPE_CMD 0
77d220b1cfSCristian Marussi #define SCMI_MSG_TYPE_DRESP 2
78d220b1cfSCristian Marussi #define SCMI_MSG_TYPE_NOTIF 3
7954b96380SRuslan Bukin
803595f18fSCristian Marussi #define SCMI_MSG_TYPE_CHECK(_h, _t) \
813595f18fSCristian Marussi ((((_h) & SCMI_HDR_MESSAGE_TYPE_M) >> SCMI_HDR_MESSAGE_TYPE_S) == (_t))
8254b96380SRuslan Bukin
833595f18fSCristian Marussi #define SCMI_IS_MSG_TYPE_NOTIF(h) \
843595f18fSCristian Marussi SCMI_MSG_TYPE_CHECK((h), SCMI_MSG_TYPE_NOTIF)
853595f18fSCristian Marussi #define SCMI_IS_MSG_TYPE_DRESP(h) \
863595f18fSCristian Marussi SCMI_MSG_TYPE_CHECK((h), SCMI_MSG_TYPE_DRESP)
8754b96380SRuslan Bukin
883595f18fSCristian Marussi #define SCMI_MSG_TOKEN(_hdr) \
893595f18fSCristian Marussi (((_hdr) & SCMI_HDR_TOKEN_M) >> SCMI_HDR_TOKEN_S)
9054b96380SRuslan Bukin
9135f93203SCristian Marussi struct scmi_req {
9235f93203SCristian Marussi int cnt;
9335f93203SCristian Marussi bool timed_out;
9435f93203SCristian Marussi bool use_polling;
9535f93203SCristian Marussi bool done;
9635f93203SCristian Marussi struct mtx mtx;
9735f93203SCristian Marussi LIST_ENTRY(scmi_req) next;
9835f93203SCristian Marussi int protocol_id;
9935f93203SCristian Marussi int message_id;
10035f93203SCristian Marussi int token;
10135f93203SCristian Marussi uint32_t header;
10235f93203SCristian Marussi struct scmi_msg msg;
10335f93203SCristian Marussi };
10435f93203SCristian Marussi
10535f93203SCristian Marussi #define buf_to_msg(b) __containerof((b), struct scmi_msg, payld)
10635f93203SCristian Marussi #define msg_to_req(m) __containerof((m), struct scmi_req, msg)
10735f93203SCristian Marussi #define buf_to_req(b) msg_to_req(buf_to_msg(b))
10835f93203SCristian Marussi
10935f93203SCristian Marussi LIST_HEAD(reqs_head, scmi_req);
11035f93203SCristian Marussi
11135f93203SCristian Marussi struct scmi_reqs_pool {
11235f93203SCristian Marussi struct mtx mtx;
11335f93203SCristian Marussi struct reqs_head head;
11435f93203SCristian Marussi };
11535f93203SCristian Marussi
1163595f18fSCristian Marussi BITSET_DEFINE(_scmi_tokens, SCMI_MAX_TOKEN);
1173595f18fSCristian Marussi LIST_HEAD(inflight_head, scmi_req);
1183595f18fSCristian Marussi #define REQHASH(_sc, _tk) \
1193595f18fSCristian Marussi (&((_sc)->trs->inflight_ht[(_tk) & (_sc)->trs->inflight_mask]))
12054b96380SRuslan Bukin
1213595f18fSCristian Marussi struct scmi_transport {
1223595f18fSCristian Marussi unsigned long next_id;
1233595f18fSCristian Marussi struct _scmi_tokens avail_tokens;
1243595f18fSCristian Marussi struct inflight_head *inflight_ht;
1253595f18fSCristian Marussi unsigned long inflight_mask;
12635f93203SCristian Marussi struct scmi_reqs_pool *chans[SCMI_CHAN_MAX];
1273595f18fSCristian Marussi struct mtx mtx;
1283595f18fSCristian Marussi };
129a0ba2a97SCristian Marussi
1303595f18fSCristian Marussi static int scmi_transport_init(struct scmi_softc *);
1313595f18fSCristian Marussi static void scmi_transport_cleanup(struct scmi_softc *);
13235f93203SCristian Marussi static struct scmi_reqs_pool *scmi_reqs_pool_allocate(const int, const int);
13335f93203SCristian Marussi static void scmi_reqs_pool_free(struct scmi_reqs_pool *);
13435f93203SCristian Marussi static struct scmi_req *scmi_req_alloc(struct scmi_softc *, enum scmi_chan);
13535f93203SCristian Marussi static void scmi_req_free_unlocked(struct scmi_softc *,
13635f93203SCristian Marussi enum scmi_chan, struct scmi_req *);
13735f93203SCristian Marussi static void scmi_req_get(struct scmi_softc *, struct scmi_req *);
13835f93203SCristian Marussi static void scmi_req_put(struct scmi_softc *, struct scmi_req *);
1393595f18fSCristian Marussi static int scmi_token_pick(struct scmi_softc *);
1403595f18fSCristian Marussi static void scmi_token_release_unlocked(struct scmi_softc *, int);
1413595f18fSCristian Marussi static int scmi_req_track_inflight(struct scmi_softc *,
1423595f18fSCristian Marussi struct scmi_req *);
1433595f18fSCristian Marussi static int scmi_req_drop_inflight(struct scmi_softc *,
1443595f18fSCristian Marussi struct scmi_req *);
1453595f18fSCristian Marussi static struct scmi_req *scmi_req_lookup_inflight(struct scmi_softc *, uint32_t);
14654b96380SRuslan Bukin
1473595f18fSCristian Marussi static int scmi_wait_for_response(struct scmi_softc *,
14835f93203SCristian Marussi struct scmi_req *, void **);
1493595f18fSCristian Marussi static void scmi_process_response(struct scmi_softc *, uint32_t);
15054b96380SRuslan Bukin
151d46f01fdSAndrew Turner int
scmi_attach(device_t dev)15254b96380SRuslan Bukin scmi_attach(device_t dev)
15354b96380SRuslan Bukin {
15454b96380SRuslan Bukin struct scmi_softc *sc;
15554b96380SRuslan Bukin phandle_t node;
15654b96380SRuslan Bukin int error;
15754b96380SRuslan Bukin
15854b96380SRuslan Bukin sc = device_get_softc(dev);
15954b96380SRuslan Bukin sc->dev = dev;
16054b96380SRuslan Bukin
16154b96380SRuslan Bukin node = ofw_bus_get_node(dev);
16254b96380SRuslan Bukin if (node == -1)
16354b96380SRuslan Bukin return (ENXIO);
16454b96380SRuslan Bukin
16554b96380SRuslan Bukin simplebus_init(dev, node);
16654b96380SRuslan Bukin
1673595f18fSCristian Marussi error = scmi_transport_init(sc);
168403ca28cSCristian Marussi if (error != 0)
169403ca28cSCristian Marussi return (error);
170403ca28cSCristian Marussi
1713595f18fSCristian Marussi device_printf(dev, "Transport reply timeout initialized to %dms\n",
1723595f18fSCristian Marussi sc->trs_desc.reply_timo_ms);
1733595f18fSCristian Marussi
17454b96380SRuslan Bukin /*
17554b96380SRuslan Bukin * Allow devices to identify.
17654b96380SRuslan Bukin */
17754b96380SRuslan Bukin bus_generic_probe(dev);
17854b96380SRuslan Bukin
17954b96380SRuslan Bukin /*
18054b96380SRuslan Bukin * Now walk the OFW tree and attach top-level devices.
18154b96380SRuslan Bukin */
18254b96380SRuslan Bukin for (node = OF_child(node); node > 0; node = OF_peer(node))
18354b96380SRuslan Bukin simplebus_add_device(dev, node, 0, NULL, -1, NULL);
18454b96380SRuslan Bukin
18554b96380SRuslan Bukin error = bus_generic_attach(dev);
18654b96380SRuslan Bukin
18754b96380SRuslan Bukin return (error);
18854b96380SRuslan Bukin }
18954b96380SRuslan Bukin
19054b96380SRuslan Bukin static int
scmi_detach(device_t dev)19154b96380SRuslan Bukin scmi_detach(device_t dev)
19254b96380SRuslan Bukin {
1933595f18fSCristian Marussi struct scmi_softc *sc;
19454b96380SRuslan Bukin
1953595f18fSCristian Marussi sc = device_get_softc(dev);
1963595f18fSCristian Marussi scmi_transport_cleanup(sc);
1973595f18fSCristian Marussi
19854b96380SRuslan Bukin return (0);
19954b96380SRuslan Bukin }
20054b96380SRuslan Bukin
20154b96380SRuslan Bukin static device_method_t scmi_methods[] = {
20254b96380SRuslan Bukin DEVMETHOD(device_attach, scmi_attach),
20354b96380SRuslan Bukin DEVMETHOD(device_detach, scmi_detach),
204d46f01fdSAndrew Turner
20554b96380SRuslan Bukin DEVMETHOD_END
20654b96380SRuslan Bukin };
20754b96380SRuslan Bukin
20854b96380SRuslan Bukin DEFINE_CLASS_1(scmi, scmi_driver, scmi_methods, sizeof(struct scmi_softc),
20954b96380SRuslan Bukin simplebus_driver);
21054b96380SRuslan Bukin
21154b96380SRuslan Bukin DRIVER_MODULE(scmi, simplebus, scmi_driver, 0, 0);
21254b96380SRuslan Bukin MODULE_VERSION(scmi, 1);
2133595f18fSCristian Marussi
21435f93203SCristian Marussi static struct scmi_reqs_pool *
scmi_reqs_pool_allocate(const int max_msg,const int max_payld_sz)21535f93203SCristian Marussi scmi_reqs_pool_allocate(const int max_msg, const int max_payld_sz)
21635f93203SCristian Marussi {
21735f93203SCristian Marussi struct scmi_reqs_pool *rp;
21835f93203SCristian Marussi struct scmi_req *req;
21935f93203SCristian Marussi
22035f93203SCristian Marussi rp = malloc(sizeof(*rp), M_DEVBUF, M_ZERO | M_WAITOK);
22135f93203SCristian Marussi
22235f93203SCristian Marussi LIST_INIT(&rp->head);
22335f93203SCristian Marussi for (int i = 0; i < max_msg; i++) {
22435f93203SCristian Marussi req = malloc(sizeof(*req) + max_payld_sz,
22535f93203SCristian Marussi M_DEVBUF, M_ZERO | M_WAITOK);
22635f93203SCristian Marussi
22735f93203SCristian Marussi mtx_init(&req->mtx, "req", "SCMI", MTX_SPIN);
22835f93203SCristian Marussi LIST_INSERT_HEAD(&rp->head, req, next);
22935f93203SCristian Marussi }
23035f93203SCristian Marussi
23135f93203SCristian Marussi mtx_init(&rp->mtx, "reqs_pool", "SCMI", MTX_SPIN);
23235f93203SCristian Marussi
23335f93203SCristian Marussi return (rp);
23435f93203SCristian Marussi }
23535f93203SCristian Marussi
23635f93203SCristian Marussi static void
scmi_reqs_pool_free(struct scmi_reqs_pool * rp)23735f93203SCristian Marussi scmi_reqs_pool_free(struct scmi_reqs_pool *rp)
23835f93203SCristian Marussi {
23935f93203SCristian Marussi struct scmi_req *req;
24035f93203SCristian Marussi
24135f93203SCristian Marussi LIST_FOREACH(req, &rp->head, next) {
24235f93203SCristian Marussi mtx_destroy(&req->mtx);
24335f93203SCristian Marussi free(req, M_DEVBUF);
24435f93203SCristian Marussi }
24535f93203SCristian Marussi
24635f93203SCristian Marussi mtx_destroy(&rp->mtx);
24735f93203SCristian Marussi free(rp, M_DEVBUF);
24835f93203SCristian Marussi }
24935f93203SCristian Marussi
2503595f18fSCristian Marussi static int
scmi_transport_init(struct scmi_softc * sc)2513595f18fSCristian Marussi scmi_transport_init(struct scmi_softc *sc)
2523595f18fSCristian Marussi {
2533595f18fSCristian Marussi struct scmi_transport *trs;
2543595f18fSCristian Marussi int ret;
2553595f18fSCristian Marussi
2563595f18fSCristian Marussi trs = malloc(sizeof(*trs), M_DEVBUF, M_ZERO | M_WAITOK);
2573595f18fSCristian Marussi
2583595f18fSCristian Marussi BIT_FILL(SCMI_MAX_TOKEN, &trs->avail_tokens);
2593595f18fSCristian Marussi mtx_init(&trs->mtx, "tokens", "SCMI", MTX_SPIN);
2603595f18fSCristian Marussi
2613595f18fSCristian Marussi trs->inflight_ht = hashinit(SCMI_MAX_MSG, M_DEVBUF,
2623595f18fSCristian Marussi &trs->inflight_mask);
2633595f18fSCristian Marussi
26435f93203SCristian Marussi trs->chans[SCMI_CHAN_A2P] =
26535f93203SCristian Marussi scmi_reqs_pool_allocate(SCMI_MAX_MSG, SCMI_MAX_MSG_PAYLD_SIZE);
26635f93203SCristian Marussi if (trs->chans[SCMI_CHAN_A2P] == NULL) {
26735f93203SCristian Marussi free(trs, M_DEVBUF);
26835f93203SCristian Marussi return (ENOMEM);
26935f93203SCristian Marussi }
27035f93203SCristian Marussi
27135f93203SCristian Marussi trs->chans[SCMI_CHAN_P2A] =
27235f93203SCristian Marussi scmi_reqs_pool_allocate(SCMI_MAX_MSG, SCMI_MAX_MSG_PAYLD_SIZE);
27335f93203SCristian Marussi if (trs->chans[SCMI_CHAN_P2A] == NULL) {
27435f93203SCristian Marussi scmi_reqs_pool_free(trs->chans[SCMI_CHAN_A2P]);
27535f93203SCristian Marussi free(trs, M_DEVBUF);
27635f93203SCristian Marussi return (ENOMEM);
27735f93203SCristian Marussi }
27835f93203SCristian Marussi
2793595f18fSCristian Marussi sc->trs = trs;
2803595f18fSCristian Marussi ret = SCMI_TRANSPORT_INIT(sc->dev);
2813595f18fSCristian Marussi if (ret != 0) {
28235f93203SCristian Marussi scmi_reqs_pool_free(trs->chans[SCMI_CHAN_A2P]);
28335f93203SCristian Marussi scmi_reqs_pool_free(trs->chans[SCMI_CHAN_P2A]);
2843595f18fSCristian Marussi free(trs, M_DEVBUF);
2853595f18fSCristian Marussi return (ret);
2863595f18fSCristian Marussi }
2873595f18fSCristian Marussi
2883595f18fSCristian Marussi return (0);
2893595f18fSCristian Marussi }
2903595f18fSCristian Marussi static void
scmi_transport_cleanup(struct scmi_softc * sc)2913595f18fSCristian Marussi scmi_transport_cleanup(struct scmi_softc *sc)
2923595f18fSCristian Marussi {
2933595f18fSCristian Marussi
2943595f18fSCristian Marussi SCMI_TRANSPORT_CLEANUP(sc->dev);
2953595f18fSCristian Marussi mtx_destroy(&sc->trs->mtx);
2963595f18fSCristian Marussi hashdestroy(sc->trs->inflight_ht, M_DEVBUF, sc->trs->inflight_mask);
29735f93203SCristian Marussi scmi_reqs_pool_free(sc->trs->chans[SCMI_CHAN_A2P]);
29835f93203SCristian Marussi scmi_reqs_pool_free(sc->trs->chans[SCMI_CHAN_P2A]);
2993595f18fSCristian Marussi free(sc->trs, M_DEVBUF);
3003595f18fSCristian Marussi }
3013595f18fSCristian Marussi
30235f93203SCristian Marussi static struct scmi_req *
scmi_req_alloc(struct scmi_softc * sc,enum scmi_chan ch_idx)30335f93203SCristian Marussi scmi_req_alloc(struct scmi_softc *sc, enum scmi_chan ch_idx)
30435f93203SCristian Marussi {
30535f93203SCristian Marussi struct scmi_reqs_pool *rp;
30635f93203SCristian Marussi struct scmi_req *req = NULL;
30735f93203SCristian Marussi
30835f93203SCristian Marussi rp = sc->trs->chans[ch_idx];
30935f93203SCristian Marussi mtx_lock_spin(&rp->mtx);
31035f93203SCristian Marussi if (!LIST_EMPTY(&rp->head)) {
31135f93203SCristian Marussi req = LIST_FIRST(&rp->head);
31235f93203SCristian Marussi LIST_REMOVE_HEAD(&rp->head, next);
31335f93203SCristian Marussi }
31435f93203SCristian Marussi mtx_unlock_spin(&rp->mtx);
31535f93203SCristian Marussi
31635f93203SCristian Marussi if (req != NULL)
31735f93203SCristian Marussi refcount_init(&req->cnt, 1);
31835f93203SCristian Marussi
31935f93203SCristian Marussi return (req);
32035f93203SCristian Marussi }
32135f93203SCristian Marussi
32235f93203SCristian Marussi static void
scmi_req_free_unlocked(struct scmi_softc * sc,enum scmi_chan ch_idx,struct scmi_req * req)32335f93203SCristian Marussi scmi_req_free_unlocked(struct scmi_softc *sc, enum scmi_chan ch_idx,
32435f93203SCristian Marussi struct scmi_req *req)
32535f93203SCristian Marussi {
32635f93203SCristian Marussi struct scmi_reqs_pool *rp;
32735f93203SCristian Marussi
32835f93203SCristian Marussi rp = sc->trs->chans[ch_idx];
32935f93203SCristian Marussi mtx_lock_spin(&rp->mtx);
33035f93203SCristian Marussi req->timed_out = false;
33135f93203SCristian Marussi req->done = false;
33235f93203SCristian Marussi refcount_init(&req->cnt, 0);
33335f93203SCristian Marussi LIST_INSERT_HEAD(&rp->head, req, next);
33435f93203SCristian Marussi mtx_unlock_spin(&rp->mtx);
33535f93203SCristian Marussi }
33635f93203SCristian Marussi
33735f93203SCristian Marussi static void
scmi_req_get(struct scmi_softc * sc,struct scmi_req * req)33835f93203SCristian Marussi scmi_req_get(struct scmi_softc *sc, struct scmi_req *req)
33935f93203SCristian Marussi {
34035f93203SCristian Marussi bool ok;
34135f93203SCristian Marussi
34235f93203SCristian Marussi mtx_lock_spin(&req->mtx);
34335f93203SCristian Marussi ok = refcount_acquire_if_not_zero(&req->cnt);
34435f93203SCristian Marussi mtx_unlock_spin(&req->mtx);
34535f93203SCristian Marussi
34635f93203SCristian Marussi if (!ok)
34735f93203SCristian Marussi device_printf(sc->dev, "%s() -- BAD REFCOUNT\n", __func__);
34835f93203SCristian Marussi
34935f93203SCristian Marussi return;
35035f93203SCristian Marussi }
35135f93203SCristian Marussi
35235f93203SCristian Marussi static void
scmi_req_put(struct scmi_softc * sc,struct scmi_req * req)35335f93203SCristian Marussi scmi_req_put(struct scmi_softc *sc, struct scmi_req *req)
35435f93203SCristian Marussi {
35535f93203SCristian Marussi mtx_lock_spin(&req->mtx);
35635f93203SCristian Marussi if (!refcount_release_if_not_last(&req->cnt)) {
35735f93203SCristian Marussi bzero(&req->msg, sizeof(req->msg) + SCMI_MAX_MSG_PAYLD_SIZE);
35835f93203SCristian Marussi scmi_req_free_unlocked(sc, SCMI_CHAN_A2P, req);
35935f93203SCristian Marussi }
36035f93203SCristian Marussi mtx_unlock_spin(&req->mtx);
36135f93203SCristian Marussi }
36235f93203SCristian Marussi
3633595f18fSCristian Marussi static int
scmi_token_pick(struct scmi_softc * sc)3643595f18fSCristian Marussi scmi_token_pick(struct scmi_softc *sc)
3653595f18fSCristian Marussi {
3663595f18fSCristian Marussi unsigned long next_msg_id, token;
3673595f18fSCristian Marussi
3683595f18fSCristian Marussi mtx_lock_spin(&sc->trs->mtx);
3693595f18fSCristian Marussi /*
3703595f18fSCristian Marussi * next_id is a monotonically increasing unsigned long that can be used
3713595f18fSCristian Marussi * for tracing purposes; next_msg_id is a 10-bit sequence number derived
3723595f18fSCristian Marussi * from it.
3733595f18fSCristian Marussi */
3743595f18fSCristian Marussi next_msg_id = sc->trs->next_id++ & SCMI_HDR_TOKEN_BF;
3753595f18fSCristian Marussi token = BIT_FFS_AT(SCMI_MAX_TOKEN, &sc->trs->avail_tokens, next_msg_id);
3763595f18fSCristian Marussi /* TODO Account for wrap-arounds and holes */
3773595f18fSCristian Marussi if (token != 0)
3783595f18fSCristian Marussi BIT_CLR(SCMI_MAX_TOKEN, token - 1, &sc->trs->avail_tokens);
3793595f18fSCristian Marussi mtx_unlock_spin(&sc->trs->mtx);
3803595f18fSCristian Marussi
3813595f18fSCristian Marussi /*
3823595f18fSCristian Marussi * BIT_FFS_AT returns 1-indexed values, so 0 means failure to find a
3833595f18fSCristian Marussi * free slot: all possible SCMI messages are in-flight using all of the
3843595f18fSCristian Marussi * SCMI_MAX_TOKEN sequence numbers.
3853595f18fSCristian Marussi */
3863595f18fSCristian Marussi if (!token)
3873595f18fSCristian Marussi return (-EBUSY);
3883595f18fSCristian Marussi
3893595f18fSCristian Marussi return ((int)(token - 1));
3903595f18fSCristian Marussi }
3913595f18fSCristian Marussi
3923595f18fSCristian Marussi static void
scmi_token_release_unlocked(struct scmi_softc * sc,int token)3933595f18fSCristian Marussi scmi_token_release_unlocked(struct scmi_softc *sc, int token)
3943595f18fSCristian Marussi {
3953595f18fSCristian Marussi
3963595f18fSCristian Marussi BIT_SET(SCMI_MAX_TOKEN, token, &sc->trs->avail_tokens);
3973595f18fSCristian Marussi }
3983595f18fSCristian Marussi
3993595f18fSCristian Marussi static int
scmi_finalize_req(struct scmi_softc * sc,struct scmi_req * req)4003595f18fSCristian Marussi scmi_finalize_req(struct scmi_softc *sc, struct scmi_req *req)
4013595f18fSCristian Marussi {
4023595f18fSCristian Marussi uint32_t header = 0;
4033595f18fSCristian Marussi
4043595f18fSCristian Marussi req->token = scmi_token_pick(sc);
4053595f18fSCristian Marussi if (req->token < 0)
4063595f18fSCristian Marussi return (EBUSY);
4073595f18fSCristian Marussi
4083595f18fSCristian Marussi header = req->message_id;
4093595f18fSCristian Marussi header |= SCMI_MSG_TYPE_CMD << SCMI_HDR_MESSAGE_TYPE_S;
4103595f18fSCristian Marussi header |= req->protocol_id << SCMI_HDR_PROTOCOL_ID_S;
4113595f18fSCristian Marussi header |= req->token << SCMI_HDR_TOKEN_S;
4123595f18fSCristian Marussi
41335f93203SCristian Marussi req->header = htole32(header);
41435f93203SCristian Marussi req->msg.hdr = htole32(header);
4153595f18fSCristian Marussi
4163595f18fSCristian Marussi return (0);
4173595f18fSCristian Marussi }
4183595f18fSCristian Marussi
4193595f18fSCristian Marussi static int
scmi_req_track_inflight(struct scmi_softc * sc,struct scmi_req * req)4203595f18fSCristian Marussi scmi_req_track_inflight(struct scmi_softc *sc, struct scmi_req *req)
4213595f18fSCristian Marussi {
4223595f18fSCristian Marussi int error;
4233595f18fSCristian Marussi
4243595f18fSCristian Marussi /* build hdr, pick token */
4253595f18fSCristian Marussi error = scmi_finalize_req(sc, req);
4263595f18fSCristian Marussi if (error != 0)
4273595f18fSCristian Marussi return (error);
4283595f18fSCristian Marussi
42935f93203SCristian Marussi /* Bump refcount to get hold of this in-flight transaction */
43035f93203SCristian Marussi scmi_req_get(sc, req);
43135f93203SCristian Marussi /* Register in the inflight hashtable */
4323595f18fSCristian Marussi mtx_lock_spin(&sc->trs->mtx);
4333595f18fSCristian Marussi LIST_INSERT_HEAD(REQHASH(sc, req->token), req, next);
4343595f18fSCristian Marussi mtx_unlock_spin(&sc->trs->mtx);
4353595f18fSCristian Marussi
4363595f18fSCristian Marussi return (0);
4373595f18fSCristian Marussi }
4383595f18fSCristian Marussi
4393595f18fSCristian Marussi static int
scmi_req_drop_inflight(struct scmi_softc * sc,struct scmi_req * req)4403595f18fSCristian Marussi scmi_req_drop_inflight(struct scmi_softc *sc, struct scmi_req *req)
4413595f18fSCristian Marussi {
4423595f18fSCristian Marussi
44335f93203SCristian Marussi /* Remove from inflight hashtable at first ... */
4443595f18fSCristian Marussi mtx_lock_spin(&sc->trs->mtx);
4453595f18fSCristian Marussi LIST_REMOVE(req, next);
4463595f18fSCristian Marussi scmi_token_release_unlocked(sc, req->token);
4473595f18fSCristian Marussi mtx_unlock_spin(&sc->trs->mtx);
44835f93203SCristian Marussi /* ...and drop refcount..potentially releasing *req */
44935f93203SCristian Marussi scmi_req_put(sc, req);
4503595f18fSCristian Marussi
4513595f18fSCristian Marussi return (0);
4523595f18fSCristian Marussi }
4533595f18fSCristian Marussi
4543595f18fSCristian Marussi static struct scmi_req *
scmi_req_lookup_inflight(struct scmi_softc * sc,uint32_t hdr)4553595f18fSCristian Marussi scmi_req_lookup_inflight(struct scmi_softc *sc, uint32_t hdr)
4563595f18fSCristian Marussi {
4573595f18fSCristian Marussi struct scmi_req *req = NULL;
4583595f18fSCristian Marussi unsigned int token;
4593595f18fSCristian Marussi
4603595f18fSCristian Marussi token = SCMI_MSG_TOKEN(hdr);
4613595f18fSCristian Marussi mtx_lock_spin(&sc->trs->mtx);
4623595f18fSCristian Marussi LIST_FOREACH(req, REQHASH(sc, token), next) {
4633595f18fSCristian Marussi if (req->token == token)
4643595f18fSCristian Marussi break;
4653595f18fSCristian Marussi }
4663595f18fSCristian Marussi mtx_unlock_spin(&sc->trs->mtx);
4673595f18fSCristian Marussi
4683595f18fSCristian Marussi return (req);
4693595f18fSCristian Marussi }
4703595f18fSCristian Marussi
4713595f18fSCristian Marussi static void
scmi_process_response(struct scmi_softc * sc,uint32_t hdr)4723595f18fSCristian Marussi scmi_process_response(struct scmi_softc *sc, uint32_t hdr)
4733595f18fSCristian Marussi {
47435f93203SCristian Marussi bool timed_out = false;
4753595f18fSCristian Marussi struct scmi_req *req;
4763595f18fSCristian Marussi
4773595f18fSCristian Marussi req = scmi_req_lookup_inflight(sc, hdr);
4783595f18fSCristian Marussi if (req == NULL) {
4793595f18fSCristian Marussi device_printf(sc->dev,
4803595f18fSCristian Marussi "Unexpected reply with header |%X| - token: 0x%X Drop.\n",
4813595f18fSCristian Marussi hdr, SCMI_MSG_TOKEN(hdr));
4823595f18fSCristian Marussi return;
4833595f18fSCristian Marussi }
4843595f18fSCristian Marussi
48535f93203SCristian Marussi mtx_lock_spin(&req->mtx);
4863595f18fSCristian Marussi req->done = true;
487a87dd741SCristian Marussi if (!req->timed_out) {
488a87dd741SCristian Marussi /*
489a87dd741SCristian Marussi * Consider the case in which a polled message is picked
490a87dd741SCristian Marussi * by chance on the IRQ path on another CPU: setting poll_done
491a87dd741SCristian Marussi * will terminate the other poll loop.
492a87dd741SCristian Marussi */
493a87dd741SCristian Marussi if (!req->msg.polling)
4943595f18fSCristian Marussi wakeup(req);
49535f93203SCristian Marussi else
496a87dd741SCristian Marussi atomic_store_rel_int(&req->msg.poll_done, 1);
497a87dd741SCristian Marussi } else {
49835f93203SCristian Marussi timed_out = true;
499a87dd741SCristian Marussi }
50035f93203SCristian Marussi mtx_unlock_spin(&req->mtx);
50135f93203SCristian Marussi
50235f93203SCristian Marussi if (timed_out)
50335f93203SCristian Marussi device_printf(sc->dev,
50435f93203SCristian Marussi "Late reply for timed-out request - token: 0x%X. Ignore.\n",
50535f93203SCristian Marussi req->token);
50635f93203SCristian Marussi
50735f93203SCristian Marussi /*
50835f93203SCristian Marussi * In case of a late reply to a timed-out transaction this will
50935f93203SCristian Marussi * finally free the pending scmi_req
51035f93203SCristian Marussi */
51135f93203SCristian Marussi scmi_req_drop_inflight(sc, req);
5123595f18fSCristian Marussi }
5133595f18fSCristian Marussi
5143595f18fSCristian Marussi void
scmi_rx_irq_callback(device_t dev,void * chan,uint32_t hdr)5153595f18fSCristian Marussi scmi_rx_irq_callback(device_t dev, void *chan, uint32_t hdr)
5163595f18fSCristian Marussi {
5173595f18fSCristian Marussi struct scmi_softc *sc;
5183595f18fSCristian Marussi
5193595f18fSCristian Marussi sc = device_get_softc(dev);
5203595f18fSCristian Marussi
5213595f18fSCristian Marussi if (SCMI_IS_MSG_TYPE_NOTIF(hdr) || SCMI_IS_MSG_TYPE_DRESP(hdr)) {
5223595f18fSCristian Marussi device_printf(dev, "DRESP/NOTIF unsupported. Drop.\n");
5233595f18fSCristian Marussi SCMI_CLEAR_CHANNEL(dev, chan);
5243595f18fSCristian Marussi return;
5253595f18fSCristian Marussi }
5263595f18fSCristian Marussi
5273595f18fSCristian Marussi scmi_process_response(sc, hdr);
5283595f18fSCristian Marussi }
5293595f18fSCristian Marussi
5303595f18fSCristian Marussi static int
scmi_wait_for_response(struct scmi_softc * sc,struct scmi_req * req,void ** out)53135f93203SCristian Marussi scmi_wait_for_response(struct scmi_softc *sc, struct scmi_req *req, void **out)
5323595f18fSCristian Marussi {
5333595f18fSCristian Marussi int ret;
5343595f18fSCristian Marussi
53535f93203SCristian Marussi if (req->msg.polling) {
53635f93203SCristian Marussi bool needs_drop;
53735f93203SCristian Marussi
53835f93203SCristian Marussi ret = SCMI_POLL_MSG(sc->dev, &req->msg,
53935f93203SCristian Marussi sc->trs_desc.reply_timo_ms);
54035f93203SCristian Marussi /*
54135f93203SCristian Marussi * Drop reference to successfully polled req unless it had
54235f93203SCristian Marussi * already also been processed on the IRQ path.
54335f93203SCristian Marussi * Addresses a possible race-condition between polling and
54435f93203SCristian Marussi * interrupt reception paths.
54535f93203SCristian Marussi */
54635f93203SCristian Marussi mtx_lock_spin(&req->mtx);
54735f93203SCristian Marussi needs_drop = (ret == 0) && !req->done;
54835f93203SCristian Marussi mtx_unlock_spin(&req->mtx);
54935f93203SCristian Marussi if (needs_drop)
55035f93203SCristian Marussi scmi_req_drop_inflight(sc, req);
55135f93203SCristian Marussi if (ret == 0 && req->msg.hdr != req->header) {
55235f93203SCristian Marussi device_printf(sc->dev,
55335f93203SCristian Marussi "Malformed reply with header |%08X|. Expected: |%08X|Drop.\n",
55435f93203SCristian Marussi le32toh(req->msg.hdr), le32toh(req->header));
55535f93203SCristian Marussi }
5563595f18fSCristian Marussi } else {
5573595f18fSCristian Marussi ret = tsleep(req, 0, "scmi_wait4",
5583595f18fSCristian Marussi (sc->trs_desc.reply_timo_ms * hz) / 1000);
5593595f18fSCristian Marussi /* Check for lost wakeups since there is no associated lock */
56035f93203SCristian Marussi mtx_lock_spin(&req->mtx);
5613595f18fSCristian Marussi if (ret != 0 && req->done)
5623595f18fSCristian Marussi ret = 0;
56335f93203SCristian Marussi mtx_unlock_spin(&req->mtx);
5643595f18fSCristian Marussi }
5653595f18fSCristian Marussi
56635f93203SCristian Marussi if (ret == 0) {
56735f93203SCristian Marussi SCMI_COLLECT_REPLY(sc->dev, &req->msg);
56835f93203SCristian Marussi if (req->msg.payld[0] != 0)
56935f93203SCristian Marussi ret = req->msg.payld[0];
57035f93203SCristian Marussi *out = &req->msg.payld[SCMI_MSG_HDR_SIZE];
57135f93203SCristian Marussi } else {
57235f93203SCristian Marussi mtx_lock_spin(&req->mtx);
57335f93203SCristian Marussi req->timed_out = true;
57435f93203SCristian Marussi mtx_unlock_spin(&req->mtx);
5753595f18fSCristian Marussi device_printf(sc->dev,
5763595f18fSCristian Marussi "Request for token 0x%X timed-out.\n", req->token);
57735f93203SCristian Marussi }
5783595f18fSCristian Marussi
5793595f18fSCristian Marussi SCMI_TX_COMPLETE(sc->dev, NULL);
5803595f18fSCristian Marussi
5813595f18fSCristian Marussi return (ret);
5823595f18fSCristian Marussi }
5833595f18fSCristian Marussi
58435f93203SCristian Marussi void *
scmi_buf_get(device_t dev,uint8_t protocol_id,uint8_t message_id,int tx_payld_sz,int rx_payld_sz)58535f93203SCristian Marussi scmi_buf_get(device_t dev, uint8_t protocol_id, uint8_t message_id,
58635f93203SCristian Marussi int tx_payld_sz, int rx_payld_sz)
5873595f18fSCristian Marussi {
5883595f18fSCristian Marussi struct scmi_softc *sc;
58935f93203SCristian Marussi struct scmi_req *req;
59035f93203SCristian Marussi
59135f93203SCristian Marussi sc = device_get_softc(dev);
59235f93203SCristian Marussi
59335f93203SCristian Marussi if (tx_payld_sz > SCMI_MAX_MSG_PAYLD_SIZE ||
59435f93203SCristian Marussi rx_payld_sz > SCMI_MAX_MSG_REPLY_SIZE) {
59535f93203SCristian Marussi device_printf(dev, "Unsupported payload size. Drop.\n");
59635f93203SCristian Marussi return (NULL);
59735f93203SCristian Marussi }
59835f93203SCristian Marussi
59935f93203SCristian Marussi /* Pick one from free list */
60035f93203SCristian Marussi req = scmi_req_alloc(sc, SCMI_CHAN_A2P);
60135f93203SCristian Marussi if (req == NULL)
60235f93203SCristian Marussi return (NULL);
60335f93203SCristian Marussi
60435f93203SCristian Marussi req->protocol_id = protocol_id & SCMI_HDR_PROTOCOL_ID_BF;
60535f93203SCristian Marussi req->message_id = message_id & SCMI_HDR_MESSAGE_ID_BF;
60635f93203SCristian Marussi req->msg.tx_len = sizeof(req->msg.hdr) + tx_payld_sz;
60735f93203SCristian Marussi req->msg.rx_len = rx_payld_sz ?
60835f93203SCristian Marussi rx_payld_sz + 2 * sizeof(uint32_t) : SCMI_MAX_MSG_SIZE;
60935f93203SCristian Marussi
61035f93203SCristian Marussi return (&req->msg.payld[0]);
61135f93203SCristian Marussi }
61235f93203SCristian Marussi
61335f93203SCristian Marussi void
scmi_buf_put(device_t dev,void * buf)61435f93203SCristian Marussi scmi_buf_put(device_t dev, void *buf)
61535f93203SCristian Marussi {
61635f93203SCristian Marussi struct scmi_softc *sc;
61735f93203SCristian Marussi struct scmi_req *req;
61835f93203SCristian Marussi
61935f93203SCristian Marussi sc = device_get_softc(dev);
62035f93203SCristian Marussi
62135f93203SCristian Marussi req = buf_to_req(buf);
62235f93203SCristian Marussi scmi_req_put(sc, req);
62335f93203SCristian Marussi }
62435f93203SCristian Marussi
62535f93203SCristian Marussi int
scmi_request(device_t dev,void * in,void ** out)62635f93203SCristian Marussi scmi_request(device_t dev, void *in, void **out)
62735f93203SCristian Marussi {
62835f93203SCristian Marussi struct scmi_softc *sc;
62935f93203SCristian Marussi struct scmi_req *req;
6303595f18fSCristian Marussi int error;
6313595f18fSCristian Marussi
6323595f18fSCristian Marussi sc = device_get_softc(dev);
6333595f18fSCristian Marussi
63435f93203SCristian Marussi req = buf_to_req(in);
63535f93203SCristian Marussi
63635f93203SCristian Marussi req->msg.polling =
63735f93203SCristian Marussi (cold || sc->trs_desc.no_completion_irq || req->use_polling);
6383595f18fSCristian Marussi
6393595f18fSCristian Marussi /* Set inflight and send using transport specific method - refc-2 */
6403595f18fSCristian Marussi error = scmi_req_track_inflight(sc, req);
6413595f18fSCristian Marussi if (error != 0)
6423595f18fSCristian Marussi return (error);
6433595f18fSCristian Marussi
64435f93203SCristian Marussi error = SCMI_XFER_MSG(sc->dev, &req->msg);
64535f93203SCristian Marussi if (error != 0) {
6463595f18fSCristian Marussi scmi_req_drop_inflight(sc, req);
6473595f18fSCristian Marussi return (error);
6483595f18fSCristian Marussi }
64935f93203SCristian Marussi
65035f93203SCristian Marussi return (scmi_wait_for_response(sc, req, out));
65135f93203SCristian Marussi }
652