1// 2// Copyright 2013-2015 Ettus Research LLC 3// Copyright 2018 Ettus Research, a National Instruments Company 4// 5// SPDX-License-Identifier: GPL-3.0-or-later 6// 7 8// "push" and "pop" introduced in GCC 4.6; works with all clang 9#if defined(__clang__) || defined(__GNUC__) && (__GNUC__ > 3) && (__GNUC_MINOR__ > 5) 10 #pragma GCC diagnostic push 11#endif 12#if defined(__clang__) || defined(__GNUC__) 13 #pragma GCC diagnostic ignored "-Wmissing-field-initializers" 14#endif 15 16template <typename data_t> 17nirio_fifo<data_t>::nirio_fifo( 18 niriok_proxy::sptr riok_proxy, 19 const fifo_direction_t direction, 20 const std::string& name, 21 const uint32_t fifo_instance) : 22 _name(name), 23 _fifo_direction(direction), 24 _fifo_channel(fifo_instance), 25 _datatype_info(_get_datatype_info()), 26 _state(UNMAPPED), 27 _remaining_in_claimed_block(0), 28 _remaining_acquirable_elements(0), 29 _mem_map(), 30 _riok_proxy_ptr(riok_proxy), 31 _expected_xfer_count(0), 32 _dma_base_addr(0), 33 _elements_buffer(NULL), 34 _actual_depth_in_elements(0), 35 _total_elements_acquired(0), 36 _frame_size_in_elements(0), 37 _fifo_optimization_option(MINIMIZE_LATENCY) 38{ 39 nirio_status status = 0; 40 nirio_status_chain(_riok_proxy_ptr->set_attribute(RIO_ADDRESS_SPACE, BUS_INTERFACE), status); 41 uint32_t base_addr = 0; 42 uint32_t addr_space_word = 0; 43 nirio_status_chain(_riok_proxy_ptr->peek(0x1C, base_addr), status); 44 nirio_status_chain(_riok_proxy_ptr->peek(0xC, addr_space_word), status); 45 _dma_base_addr = base_addr + (_fifo_channel * (1<<((addr_space_word>>16)&0xF))); 46} 47 48template <typename data_t> 49nirio_fifo<data_t>::~nirio_fifo() 50{ 51 finalize(); 52} 53 54template <typename data_t> 55nirio_status nirio_fifo<data_t>::initialize( 56 const size_t requested_depth, 57 const size_t frame_size_in_elements, 58 size_t& actual_depth, 59 size_t& actual_size, 60 const fifo_optimization_option_t fifo_optimization_option) 61{ 62 nirio_status status = NiRio_Status_Success; 63 if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized; 64 boost::unique_lock<boost::recursive_mutex> lock(_mutex); 65 66 if (_state == UNMAPPED) { 67 68 _frame_size_in_elements = frame_size_in_elements; 69 _fifo_optimization_option = fifo_optimization_option; 70 71 uint32_t actual_depth_u32 = 0; 72 uint32_t actual_size_u32 = 0; 73 74 //Forcefully stop the fifo if it is running 75 _riok_proxy_ptr->stop_fifo(_fifo_channel); //Cleanup operation. Ignore status. 76 77 //Configure the FIFO now that we know it is stopped 78 status = _riok_proxy_ptr->configure_fifo( 79 _fifo_channel, 80 static_cast<uint32_t>(requested_depth), 81 1, 82 actual_depth_u32, 83 actual_size_u32); 84 if (nirio_status_fatal(status)) return status; 85 86 actual_depth = static_cast<size_t>(actual_depth_u32); 87 _actual_depth_in_elements = actual_depth; 88 actual_size = static_cast<size_t>(actual_size_u32); 89 90 status = _riok_proxy_ptr->map_fifo_memory(_fifo_channel, actual_size, _mem_map); 91 92 if (nirio_status_not_fatal(status)) { 93 _state = MAPPED; 94 } 95 } else { 96 status = NiRio_Status_SoftwareFault; 97 } 98 return status; 99} 100 101template <typename data_t> 102void nirio_fifo<data_t>::finalize() 103{ 104 boost::unique_lock<boost::recursive_mutex> lock(_mutex); 105 106 //If the FIFO is started, the stop will change the state to MAPPED. 107 stop(); 108 109 if (_state == MAPPED) { 110 _riok_proxy_ptr->unmap_fifo_memory(_mem_map); 111 _state = UNMAPPED; //Assume teardown succeeded 112 } 113} 114 115 116template <typename data_t> 117bool nirio_fifo<data_t>::_acquire_block_from_rio_buffer( 118 size_t elements_requested, 119 uint64_t timeout_in_ms, 120 const fifo_optimization_option_t fifo_optimization_option, 121 nirio_status& status) 122{ 123 uint32_t elements_acquired_u32 = 0; 124 uint32_t elements_remaining_u32 = 0; 125 size_t elements_to_request = 0; 126 void* elements_buffer = NULL; 127 128 if (fifo_optimization_option == MAXIMIZE_THROUGHPUT) 129 { 130 // We'll maximize throughput by acquiring all the data that is available 131 // But this comes at the cost of an extra wait_on_fifo in which we query 132 // the total available by requesting 0. 133 134 // first, see how many are available to acquire 135 // by trying to acquire 0 136 nirio_status_chain(_riok_proxy_ptr->wait_on_fifo( 137 _fifo_channel, 138 0, // elements requested 139 static_cast<uint32_t>(_datatype_info.scalar_type), 140 _datatype_info.width * 8, 141 0, // timeout 142 _fifo_direction == OUTPUT_FIFO, 143 elements_buffer, 144 elements_acquired_u32, 145 elements_remaining_u32), 146 status); 147 148 // acquire the maximum possible elements- all remaining 149 // yet limit to a multiple of the frame size 150 // (don't want to acquire partial frames) 151 elements_to_request = elements_remaining_u32 - (elements_remaining_u32 % _frame_size_in_elements); 152 153 // the next call to wait_on_fifo can have a 0 timeout since we 154 // know there is at least as much data as we will request available 155 timeout_in_ms = 0; 156 } 157 else 158 { 159 // fifo_optimization_option == MINIMIZE_LATENCY 160 // acquire either the minimum amount (frame size) or the amount remaining from the last call 161 // (coerced to a multiple of frames) 162 elements_to_request = std::max( 163 elements_requested, 164 (_remaining_acquirable_elements - (_remaining_acquirable_elements % _frame_size_in_elements))); 165 } 166 167 nirio_status_chain(_riok_proxy_ptr->wait_on_fifo( 168 _fifo_channel, 169 elements_to_request, 170 static_cast<uint32_t>(_datatype_info.scalar_type), 171 _datatype_info.width * 8, 172 timeout_in_ms, 173 _fifo_direction == OUTPUT_FIFO, 174 elements_buffer, 175 elements_acquired_u32, 176 elements_remaining_u32), 177 status); 178 179 if (nirio_status_not_fatal(status)) 180 { 181 _remaining_acquirable_elements = static_cast<size_t>(elements_remaining_u32); 182 183 if (elements_acquired_u32 > 0) 184 { 185 _total_elements_acquired += static_cast<size_t>(elements_acquired_u32); 186 _remaining_in_claimed_block = static_cast<size_t>(elements_acquired_u32); 187 _elements_buffer = static_cast<data_t*>(elements_buffer); 188 } 189 190 return true; 191 } 192 193 return false; 194} 195 196 197template <typename data_t> 198nirio_status nirio_fifo<data_t>::start() 199{ 200 nirio_status status = NiRio_Status_Success; 201 if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized; 202 203 boost::unique_lock<boost::recursive_mutex> lock(_mutex); 204 205 if (_state == STARTED) { 206 //Do nothing. Already started. 207 } else if (_state == MAPPED) { 208 209 _total_elements_acquired = 0; 210 _remaining_in_claimed_block = 0; 211 _remaining_acquirable_elements = 0; 212 213 status = _riok_proxy_ptr->start_fifo(_fifo_channel); 214 215 if (nirio_status_not_fatal(status)) { 216 _state = STARTED; 217 _expected_xfer_count = 0; 218 219 if (_fifo_direction == OUTPUT_FIFO) 220 { 221 // pre-acquire a block of data 222 // (should be entire DMA buffer at this point) 223 224 // requesting 0 elements, but it will acquire all since MAXIMUM_THROUGHPUT 225 _acquire_block_from_rio_buffer(0, 1000, MAXIMIZE_THROUGHPUT, status); 226 } 227 } 228 } else { 229 status = NiRio_Status_ResourceNotInitialized; 230 } 231 return status; 232} 233 234template <typename data_t> 235nirio_status nirio_fifo<data_t>::stop() 236{ 237 nirio_status status = NiRio_Status_Success; 238 if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized; 239 240 boost::unique_lock<boost::recursive_mutex> lock(_mutex); 241 242 if (_state == STARTED) { 243 244 // release any remaining acquired elements 245 if (_total_elements_acquired > 0) release(_total_elements_acquired); 246 _total_elements_acquired = 0; 247 _remaining_in_claimed_block = 0; 248 _remaining_acquirable_elements = 0; 249 250 status = _riok_proxy_ptr->stop_fifo(_fifo_channel); 251 252 _state = MAPPED; //Assume teardown succeeded 253 } 254 255 return status; 256} 257 258template <typename data_t> 259nirio_status nirio_fifo<data_t>::acquire( 260 data_t*& elements, 261 const size_t elements_requested, 262 const uint32_t timeout, 263 size_t& elements_acquired, 264 size_t& elements_remaining) 265{ 266 nirio_status status = NiRio_Status_Success; 267 if (!_riok_proxy_ptr || _mem_map.is_null()) return NiRio_Status_ResourceNotInitialized; 268 269 boost::unique_lock<boost::recursive_mutex> lock(_mutex); 270 271 if (_state == STARTED) { 272 273 if (_remaining_in_claimed_block == 0) 274 { 275 276 // so acquire some now 277 if (!_acquire_block_from_rio_buffer( 278 elements_requested, 279 timeout, 280 _fifo_optimization_option, 281 status)) 282 { 283 elements_acquired = 0; 284 elements_remaining = _remaining_acquirable_elements; 285 return status; 286 } 287 288 289 if (get_direction() == INPUT_FIFO && 290 UHD_NIRIO_RX_FIFO_XFER_CHECK_EN && 291 _riok_proxy_ptr->get_rio_quirks().rx_fifo_xfer_check_en()) 292 { 293 _expected_xfer_count += static_cast<uint64_t>(elements_requested * sizeof(data_t)); 294 status = _ensure_transfer_completed(timeout); 295 } 296 } 297 298 if (nirio_status_not_fatal(status)) 299 { 300 // Assign the request the proper area of the DMA FIFO buffer 301 elements = _elements_buffer; 302 elements_acquired = std::min(_remaining_in_claimed_block, elements_requested); 303 _remaining_in_claimed_block -= elements_acquired; 304 elements_remaining = _remaining_in_claimed_block + _remaining_acquirable_elements; 305 306 // Advance the write pointer forward in the acquired elements buffer 307 _elements_buffer += elements_acquired; 308 } 309 310 } else { 311 status = NiRio_Status_ResourceNotInitialized; 312 } 313 314 return status; 315} 316 317template <typename data_t> 318nirio_status nirio_fifo<data_t>::release(const size_t elements) 319{ 320 nirio_status status = NiRio_Status_Success; 321 if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized; 322 323 boost::unique_lock<boost::recursive_mutex> lock(_mutex); 324 325 if (_state == STARTED) { 326 status = _riok_proxy_ptr->grant_fifo( 327 _fifo_channel, 328 static_cast<uint32_t>(elements)); 329 _total_elements_acquired -= elements; 330 } else { 331 status = NiRio_Status_ResourceNotInitialized; 332 } 333 334 return status; 335} 336 337template <typename data_t> 338nirio_status nirio_fifo<data_t>::read( 339 data_t* buf, 340 const uint32_t num_elements, 341 const uint32_t timeout, 342 uint32_t& num_read, 343 uint32_t& num_remaining) 344{ 345 nirio_status status = NiRio_Status_Success; 346 if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized; 347 348 boost::unique_lock<boost::recursive_mutex> lock(_mutex); 349 350 if (_state == STARTED) { 351 status = _riok_proxy_ptr->read_fifo( 352 _fifo_channel, 353 num_elements, 354 static_cast<void*>(buf), 355 _datatype_info.width, 356 static_cast<uint32_t>(_datatype_info.scalar_type), 357 _datatype_info.width * 8, 358 timeout, 359 num_read, 360 num_remaining); 361 } else { 362 status = NiRio_Status_ResourceNotInitialized; 363 } 364 365 return status; 366} 367 368template <typename data_t> 369nirio_status nirio_fifo<data_t>::write( 370 const data_t* buf, 371 const uint32_t num_elements, 372 const uint32_t timeout, 373 uint32_t& num_remaining) 374{ 375 nirio_status status = NiRio_Status_Success; 376 if (!_riok_proxy_ptr) return NiRio_Status_ResourceNotInitialized; 377 378 boost::unique_lock<boost::recursive_mutex> lock(_mutex); 379 380 if (_state == STARTED) { 381 status = _riok_proxy_ptr->write_fifo( 382 _fifo_channel, 383 num_elements, 384 buf, 385 _datatype_info.width, 386 static_cast<uint32_t>(_datatype_info.scalar_type), 387 _datatype_info.width * 8, 388 timeout, 389 num_remaining); 390 } else { 391 status = NiRio_Status_ResourceNotInitialized; 392 } 393 394 return status; 395} 396 397template <typename data_t> 398nirio_status nirio_fifo<data_t>::_get_transfer_count(uint64_t& transfer_count) 399{ 400 //_riok_proxy_ptr must be valid and _mutex must be locked 401 402 nirio_status status = NiRio_Status_Success; 403 uint32_t lower_half = 0, upper_half = 0; 404 nirio_status_chain(_riok_proxy_ptr->peek(_dma_base_addr + 0xA8, lower_half), status); //Latches both halves 405 nirio_status_chain(_riok_proxy_ptr->peek(_dma_base_addr + 0xAC, upper_half), status); 406 407 if (nirio_status_not_fatal(status)) { 408 transfer_count = lower_half | (((uint64_t)upper_half) << 32); 409 } 410 return status; 411} 412 413template <typename data_t> 414nirio_status nirio_fifo<data_t>::_ensure_transfer_completed(uint32_t timeout_ms) 415{ 416 //_riok_proxy_ptr must be valid and _mutex must be locked 417 418 static const size_t MIN_TIMEOUT_IN_US = 2000; 419 420 nirio_status status = NiRio_Status_Success; 421 uint64_t actual_xfer_count = 0; 422 nirio_status_chain(_get_transfer_count(actual_xfer_count), status); 423 424 //We count the elapsed time using a simple counter instead of the high 425 //resolution timebase for efficiency reasons. The call to fetch the time 426 //requires a user-kernel transition which has a large overhead compared 427 //to a simple mem read. As a tradeoff, we deal with a less precise timeout. 428 size_t approx_us_elapsed = 0; 429 while ( 430 nirio_status_not_fatal(status) && 431 (_expected_xfer_count > actual_xfer_count) && 432 approx_us_elapsed++ < std::max<size_t>(MIN_TIMEOUT_IN_US, timeout_ms * 1000) 433 ) { 434 std::this_thread::sleep_for(std::chrono::microseconds(1)); 435 nirio_status_chain(_get_transfer_count(actual_xfer_count), status); 436 } 437 438 if (_expected_xfer_count > actual_xfer_count) { 439 nirio_status_chain(NiRio_Status_CommunicationTimeout, status); 440 } 441 442 return status; 443} 444 445template <> 446inline datatype_info_t nirio_fifo<int8_t>::_get_datatype_info() 447{ 448 return datatype_info_t(RIO_SCALAR_TYPE_IB, 1); 449} 450 451template <> 452inline datatype_info_t nirio_fifo<int16_t>::_get_datatype_info() 453{ 454 return datatype_info_t(RIO_SCALAR_TYPE_IW, 2); 455} 456 457template <> 458inline datatype_info_t nirio_fifo<int32_t>::_get_datatype_info() 459{ 460 return datatype_info_t(RIO_SCALAR_TYPE_IL, 4); 461} 462 463template <> 464inline datatype_info_t nirio_fifo<int64_t>::_get_datatype_info() 465{ 466 return datatype_info_t(RIO_SCALAR_TYPE_IQ, 8); 467} 468 469template <> 470inline datatype_info_t nirio_fifo<uint8_t>::_get_datatype_info() 471{ 472 return datatype_info_t(RIO_SCALAR_TYPE_UB, 1); 473} 474 475template <> 476inline datatype_info_t nirio_fifo<uint16_t>::_get_datatype_info() 477{ 478 return datatype_info_t(RIO_SCALAR_TYPE_UW, 2); 479} 480 481template <> 482inline datatype_info_t nirio_fifo<uint32_t>::_get_datatype_info() 483{ 484 return datatype_info_t(RIO_SCALAR_TYPE_UL, 4); 485} 486 487template <> 488inline datatype_info_t nirio_fifo<uint64_t>::_get_datatype_info() 489{ 490 return datatype_info_t(RIO_SCALAR_TYPE_UQ, 8); 491} 492 493#if defined(__clang__) || defined(__GNUC__) && (__GNUC__ > 3) && (__GNUC_MINOR__ > 5) 494 #pragma GCC diagnostic pop 495#endif 496