1//
2// Copyright 2013-2015 Ettus Research LLC
3// Copyright 2018 Ettus Research, a National Instruments Company
4//
5// SPDX-License-Identifier: GPL-3.0-or-later
6//
7
8// "push" and "pop" introduced in GCC 4.6; works with all clang
9#if defined(__clang__) || defined(__GNUC__) && (__GNUC__ > 3) && (__GNUC_MINOR__ > 5)
10    #pragma GCC diagnostic push
11#endif
12#if defined(__clang__) || defined(__GNUC__)
13    #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
14#endif
15
16template <typename data_t>
17nirio_fifo<data_t>::nirio_fifo(
18    niriok_proxy::sptr riok_proxy,
19    const fifo_direction_t direction,
20    const std::string& name,
21    const uint32_t fifo_instance) :
22    _name(name),
23    _fifo_direction(direction),
24    _fifo_channel(fifo_instance),
25    _datatype_info(_get_datatype_info()),
26    _state(UNMAPPED),
27    _remaining_in_claimed_block(0),
28    _remaining_acquirable_elements(0),
29    _mem_map(),
30    _riok_proxy_ptr(riok_proxy),
31    _expected_xfer_count(0),
32    _dma_base_addr(0),
33    _elements_buffer(NULL),
34    _actual_depth_in_elements(0),
35    _total_elements_acquired(0),
36    _frame_size_in_elements(0),
37    _fifo_optimization_option(MINIMIZE_LATENCY)
38{
39    nirio_status status = 0;
40    nirio_status_chain(_riok_proxy_ptr->set_attribute(RIO_ADDRESS_SPACE, BUS_INTERFACE), status);
41    uint32_t base_addr = 0;
42    uint32_t addr_space_word = 0;
43    nirio_status_chain(_riok_proxy_ptr->peek(0x1C, base_addr), status);
44    nirio_status_chain(_riok_proxy_ptr->peek(0xC, addr_space_word), status);
45    _dma_base_addr = base_addr + (_fifo_channel * (1<<((addr_space_word>>16)&0xF)));
46}
47
48template <typename data_t>
49nirio_fifo<data_t>::~nirio_fifo()
50{
51    finalize();
52}
53
54template <typename data_t>
55nirio_status nirio_fifo<data_t>::initialize(
56    const size_t requested_depth,
57    const size_t frame_size_in_elements,
58    size_t& actual_depth,
59    size_t& actual_size,
60    const fifo_optimization_option_t fifo_optimization_option)
61{
62    nirio_status status = NiRio_Status_Success;
63    if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized;
64    boost::unique_lock<boost::recursive_mutex> lock(_mutex);
65
66    if (_state == UNMAPPED) {
67
68        _frame_size_in_elements = frame_size_in_elements;
69        _fifo_optimization_option = fifo_optimization_option;
70
71        uint32_t actual_depth_u32 = 0;
72        uint32_t actual_size_u32 = 0;
73
74        //Forcefully stop the fifo if it is running
75        _riok_proxy_ptr->stop_fifo(_fifo_channel);    //Cleanup operation. Ignore status.
76
77        //Configure the FIFO now that we know it is stopped
78        status = _riok_proxy_ptr->configure_fifo(
79            _fifo_channel,
80            static_cast<uint32_t>(requested_depth),
81            1,
82            actual_depth_u32,
83            actual_size_u32);
84        if (nirio_status_fatal(status)) return status;
85
86        actual_depth = static_cast<size_t>(actual_depth_u32);
87        _actual_depth_in_elements = actual_depth;
88        actual_size = static_cast<size_t>(actual_size_u32);
89
90        status = _riok_proxy_ptr->map_fifo_memory(_fifo_channel, actual_size, _mem_map);
91
92        if (nirio_status_not_fatal(status)) {
93            _state = MAPPED;
94        }
95    } else {
96        status = NiRio_Status_SoftwareFault;
97    }
98    return status;
99}
100
101template <typename data_t>
102void nirio_fifo<data_t>::finalize()
103{
104    boost::unique_lock<boost::recursive_mutex> lock(_mutex);
105
106    //If the FIFO is started, the stop will change the state to MAPPED.
107    stop();
108
109    if (_state == MAPPED) {
110        _riok_proxy_ptr->unmap_fifo_memory(_mem_map);
111        _state = UNMAPPED;    //Assume teardown succeeded
112    }
113}
114
115
116template <typename data_t>
117bool nirio_fifo<data_t>::_acquire_block_from_rio_buffer(
118    size_t elements_requested,
119    uint64_t timeout_in_ms,
120    const fifo_optimization_option_t fifo_optimization_option,
121    nirio_status& status)
122{
123    uint32_t elements_acquired_u32 = 0;
124    uint32_t elements_remaining_u32 = 0;
125    size_t elements_to_request = 0;
126    void* elements_buffer = NULL;
127
128    if (fifo_optimization_option == MAXIMIZE_THROUGHPUT)
129    {
130        // We'll maximize throughput by acquiring all the data that is available
131        // But this comes at the cost of an extra wait_on_fifo in which we query
132        // the total available by requesting 0.
133
134        // first, see how many are available to acquire
135        // by trying to acquire 0
136        nirio_status_chain(_riok_proxy_ptr->wait_on_fifo(
137            _fifo_channel,
138            0, // elements requested
139            static_cast<uint32_t>(_datatype_info.scalar_type),
140            _datatype_info.width * 8,
141            0, // timeout
142            _fifo_direction == OUTPUT_FIFO,
143            elements_buffer,
144            elements_acquired_u32,
145            elements_remaining_u32),
146            status);
147
148        // acquire the maximum possible elements- all remaining
149        // yet limit to a multiple of the frame size
150        // (don't want to acquire partial frames)
151        elements_to_request = elements_remaining_u32 - (elements_remaining_u32 % _frame_size_in_elements);
152
153        // the next call to wait_on_fifo can have a 0 timeout since we
154        // know there is at least as much data as we will request available
155        timeout_in_ms = 0;
156    }
157    else
158    {
159        // fifo_optimization_option == MINIMIZE_LATENCY
160        // acquire either the minimum amount (frame size) or the amount remaining from the last call
161        // (coerced to a multiple of frames)
162        elements_to_request = std::max(
163            elements_requested,
164            (_remaining_acquirable_elements - (_remaining_acquirable_elements % _frame_size_in_elements)));
165    }
166
167    nirio_status_chain(_riok_proxy_ptr->wait_on_fifo(
168        _fifo_channel,
169        elements_to_request,
170        static_cast<uint32_t>(_datatype_info.scalar_type),
171        _datatype_info.width * 8,
172        timeout_in_ms,
173        _fifo_direction == OUTPUT_FIFO,
174        elements_buffer,
175        elements_acquired_u32,
176        elements_remaining_u32),
177        status);
178
179    if (nirio_status_not_fatal(status))
180    {
181        _remaining_acquirable_elements = static_cast<size_t>(elements_remaining_u32);
182
183        if (elements_acquired_u32 > 0)
184        {
185            _total_elements_acquired += static_cast<size_t>(elements_acquired_u32);
186            _remaining_in_claimed_block = static_cast<size_t>(elements_acquired_u32);
187            _elements_buffer = static_cast<data_t*>(elements_buffer);
188        }
189
190        return true;
191    }
192
193    return false;
194}
195
196
197template <typename data_t>
198nirio_status nirio_fifo<data_t>::start()
199{
200    nirio_status status = NiRio_Status_Success;
201    if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized;
202
203    boost::unique_lock<boost::recursive_mutex> lock(_mutex);
204
205    if (_state == STARTED) {
206        //Do nothing. Already started.
207    } else if (_state == MAPPED) {
208
209        _total_elements_acquired = 0;
210        _remaining_in_claimed_block = 0;
211        _remaining_acquirable_elements = 0;
212
213        status = _riok_proxy_ptr->start_fifo(_fifo_channel);
214
215        if (nirio_status_not_fatal(status)) {
216            _state = STARTED;
217            _expected_xfer_count = 0;
218
219            if (_fifo_direction == OUTPUT_FIFO)
220            {
221                // pre-acquire a block of data
222                // (should be entire DMA buffer at this point)
223
224                // requesting 0 elements, but it will acquire all since MAXIMUM_THROUGHPUT
225                _acquire_block_from_rio_buffer(0, 1000, MAXIMIZE_THROUGHPUT, status);
226            }
227        }
228    } else {
229        status = NiRio_Status_ResourceNotInitialized;
230    }
231    return status;
232}
233
234template <typename data_t>
235nirio_status nirio_fifo<data_t>::stop()
236{
237    nirio_status status = NiRio_Status_Success;
238    if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized;
239
240    boost::unique_lock<boost::recursive_mutex> lock(_mutex);
241
242    if (_state == STARTED) {
243
244        // release any remaining acquired elements
245        if (_total_elements_acquired > 0) release(_total_elements_acquired);
246        _total_elements_acquired = 0;
247        _remaining_in_claimed_block = 0;
248        _remaining_acquirable_elements = 0;
249
250        status = _riok_proxy_ptr->stop_fifo(_fifo_channel);
251
252        _state = MAPPED;    //Assume teardown succeeded
253    }
254
255    return status;
256}
257
258template <typename data_t>
259nirio_status nirio_fifo<data_t>::acquire(
260    data_t*& elements,
261    const size_t elements_requested,
262    const uint32_t timeout,
263    size_t& elements_acquired,
264    size_t& elements_remaining)
265{
266    nirio_status status = NiRio_Status_Success;
267    if (!_riok_proxy_ptr || _mem_map.is_null()) return NiRio_Status_ResourceNotInitialized;
268
269    boost::unique_lock<boost::recursive_mutex> lock(_mutex);
270
271    if (_state == STARTED) {
272
273        if (_remaining_in_claimed_block == 0)
274        {
275
276            // so acquire some now
277            if (!_acquire_block_from_rio_buffer(
278                     elements_requested,
279                     timeout,
280                     _fifo_optimization_option,
281                     status))
282            {
283                elements_acquired = 0;
284                elements_remaining = _remaining_acquirable_elements;
285                return status;
286            }
287
288
289            if (get_direction() == INPUT_FIFO &&
290                UHD_NIRIO_RX_FIFO_XFER_CHECK_EN &&
291                _riok_proxy_ptr->get_rio_quirks().rx_fifo_xfer_check_en())
292            {
293                _expected_xfer_count += static_cast<uint64_t>(elements_requested * sizeof(data_t));
294                status = _ensure_transfer_completed(timeout);
295            }
296        }
297
298        if (nirio_status_not_fatal(status))
299        {
300            // Assign the request the proper area of the DMA FIFO buffer
301            elements = _elements_buffer;
302            elements_acquired = std::min(_remaining_in_claimed_block, elements_requested);
303            _remaining_in_claimed_block -= elements_acquired;
304            elements_remaining = _remaining_in_claimed_block + _remaining_acquirable_elements;
305
306            // Advance the write pointer forward in the acquired elements buffer
307            _elements_buffer += elements_acquired;
308        }
309
310    } else {
311        status = NiRio_Status_ResourceNotInitialized;
312    }
313
314    return status;
315}
316
317template <typename data_t>
318nirio_status nirio_fifo<data_t>::release(const size_t elements)
319{
320    nirio_status status = NiRio_Status_Success;
321    if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized;
322
323    boost::unique_lock<boost::recursive_mutex> lock(_mutex);
324
325    if (_state == STARTED) {
326        status = _riok_proxy_ptr->grant_fifo(
327            _fifo_channel,
328            static_cast<uint32_t>(elements));
329        _total_elements_acquired -= elements;
330    } else {
331        status = NiRio_Status_ResourceNotInitialized;
332    }
333
334    return status;
335}
336
337template <typename data_t>
338nirio_status nirio_fifo<data_t>::read(
339    data_t* buf,
340    const uint32_t num_elements,
341    const uint32_t timeout,
342    uint32_t& num_read,
343    uint32_t& num_remaining)
344{
345    nirio_status status = NiRio_Status_Success;
346    if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized;
347
348    boost::unique_lock<boost::recursive_mutex> lock(_mutex);
349
350    if (_state == STARTED) {
351        status = _riok_proxy_ptr->read_fifo(
352            _fifo_channel,
353            num_elements,
354            static_cast<void*>(buf),
355            _datatype_info.width,
356            static_cast<uint32_t>(_datatype_info.scalar_type),
357            _datatype_info.width * 8,
358            timeout,
359            num_read,
360            num_remaining);
361    } else {
362        status = NiRio_Status_ResourceNotInitialized;
363    }
364
365    return status;
366}
367
368template <typename data_t>
369nirio_status nirio_fifo<data_t>::write(
370    const data_t* buf,
371    const uint32_t num_elements,
372    const uint32_t timeout,
373    uint32_t& num_remaining)
374{
375    nirio_status status = NiRio_Status_Success;
376    if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized;
377
378    boost::unique_lock<boost::recursive_mutex> lock(_mutex);
379
380    if (_state == STARTED) {
381        status = _riok_proxy_ptr->write_fifo(
382            _fifo_channel,
383            num_elements,
384            buf,
385            _datatype_info.width,
386            static_cast<uint32_t>(_datatype_info.scalar_type),
387            _datatype_info.width * 8,
388            timeout,
389            num_remaining);
390    } else {
391        status = NiRio_Status_ResourceNotInitialized;
392    }
393
394    return status;
395}
396
397template <typename data_t>
398nirio_status nirio_fifo<data_t>::_get_transfer_count(uint64_t& transfer_count)
399{
400    //_riok_proxy_ptr must be valid and _mutex must be locked
401
402    nirio_status status = NiRio_Status_Success;
403    uint32_t lower_half = 0, upper_half = 0;
404    nirio_status_chain(_riok_proxy_ptr->peek(_dma_base_addr + 0xA8, lower_half), status);  //Latches both halves
405    nirio_status_chain(_riok_proxy_ptr->peek(_dma_base_addr + 0xAC, upper_half), status);
406
407    if (nirio_status_not_fatal(status)) {
408        transfer_count = lower_half | (((uint64_t)upper_half) << 32);
409    }
410    return status;
411}
412
413template <typename data_t>
414nirio_status nirio_fifo<data_t>::_ensure_transfer_completed(uint32_t timeout_ms)
415{
416    //_riok_proxy_ptr must be valid and _mutex must be locked
417
418    static const size_t MIN_TIMEOUT_IN_US = 2000;
419
420    nirio_status status = NiRio_Status_Success;
421    uint64_t actual_xfer_count = 0;
422    nirio_status_chain(_get_transfer_count(actual_xfer_count), status);
423
424    //We count the elapsed time using a simple counter instead of the high
425    //resolution timebase for efficiency reasons. The call to fetch the time
426    //requires a user-kernel transition which has a large overhead compared
427    //to a simple mem read. As a tradeoff, we deal with a less precise timeout.
428    size_t approx_us_elapsed = 0;
429    while (
430        nirio_status_not_fatal(status) &&
431        (_expected_xfer_count > actual_xfer_count) &&
432        approx_us_elapsed++ < std::max<size_t>(MIN_TIMEOUT_IN_US, timeout_ms * 1000)
433    ) {
434        std::this_thread::sleep_for(std::chrono::microseconds(1));
435        nirio_status_chain(_get_transfer_count(actual_xfer_count), status);
436    }
437
438    if (_expected_xfer_count > actual_xfer_count) {
439        nirio_status_chain(NiRio_Status_CommunicationTimeout, status);
440    }
441
442    return status;
443}
444
445template <>
446inline datatype_info_t nirio_fifo<int8_t>::_get_datatype_info()
447{
448    return datatype_info_t(RIO_SCALAR_TYPE_IB, 1);
449}
450
451template <>
452inline datatype_info_t nirio_fifo<int16_t>::_get_datatype_info()
453{
454    return datatype_info_t(RIO_SCALAR_TYPE_IW, 2);
455}
456
457template <>
458inline datatype_info_t nirio_fifo<int32_t>::_get_datatype_info()
459{
460    return datatype_info_t(RIO_SCALAR_TYPE_IL, 4);
461}
462
463template <>
464inline datatype_info_t nirio_fifo<int64_t>::_get_datatype_info()
465{
466    return datatype_info_t(RIO_SCALAR_TYPE_IQ, 8);
467}
468
469template <>
470inline datatype_info_t nirio_fifo<uint8_t>::_get_datatype_info()
471{
472    return datatype_info_t(RIO_SCALAR_TYPE_UB, 1);
473}
474
475template <>
476inline datatype_info_t nirio_fifo<uint16_t>::_get_datatype_info()
477{
478    return datatype_info_t(RIO_SCALAR_TYPE_UW, 2);
479}
480
481template <>
482inline datatype_info_t nirio_fifo<uint32_t>::_get_datatype_info()
483{
484    return datatype_info_t(RIO_SCALAR_TYPE_UL, 4);
485}
486
487template <>
488inline datatype_info_t nirio_fifo<uint64_t>::_get_datatype_info()
489{
490    return datatype_info_t(RIO_SCALAR_TYPE_UQ, 8);
491}
492
493#if defined(__clang__) || defined(__GNUC__) && (__GNUC__ > 3) && (__GNUC_MINOR__ > 5)
494    #pragma GCC diagnostic pop
495#endif
496