1# Licensed to the Apache Software Foundation (ASF) under one 2# or more contributor license agreements. See the NOTICE file 3# distributed with this work for additional information 4# regarding copyright ownership. The ASF licenses this file 5# to you under the Apache License, Version 2.0 (the 6# "License"); you may not use this file except in compliance 7# with the License. You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, 12# software distributed under the License is distributed on an 13# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14# KIND, either express or implied. See the License for the 15# specific language governing permissions and limitations 16# under the License. 17 18module RawRecordsStructArrayTests 19 def build_schema(type) 20 field_description = { 21 name: :field, 22 } 23 if type.is_a?(Hash) 24 field_description = field_description.merge(type) 25 else 26 field_description[:type] = type 27 end 28 { 29 column: { 30 type: :struct, 31 fields: [ 32 field_description, 33 ], 34 }, 35 } 36 end 37 38 def test_null 39 records = [ 40 [{"field" => nil}], 41 [nil], 42 ] 43 target = build(:null, records) 44 assert_equal(records, target.raw_records) 45 end 46 47 def test_boolean 48 records = [ 49 [{"field" => true}], 50 [nil], 51 [{"field" => nil}], 52 ] 53 target = build(:boolean, records) 54 assert_equal(records, target.raw_records) 55 end 56 57 def test_int8 58 records = [ 59 [{"field" => -(2 ** 7)}], 60 [nil], 61 [{"field" => nil}], 62 ] 63 target = build(:int8, records) 64 assert_equal(records, target.raw_records) 65 end 66 67 def test_uint8 68 records = [ 69 [{"field" => (2 ** 8) - 1}], 70 [nil], 71 [{"field" => nil}], 72 ] 73 target = build(:uint8, records) 74 assert_equal(records, target.raw_records) 75 end 76 77 def test_int16 78 records = [ 79 [{"field" => -(2 ** 15)}], 80 [nil], 81 [{"field" => nil}], 82 ] 83 target = build(:int16, records) 84 assert_equal(records, target.raw_records) 85 end 86 87 def test_uint16 88 records = [ 89 [{"field" => (2 ** 16) - 1}], 90 [nil], 91 [{"field" => nil}], 92 ] 93 target = build(:uint16, records) 94 assert_equal(records, target.raw_records) 95 end 96 97 def test_int32 98 records = [ 99 [{"field" => -(2 ** 31)}], 100 [nil], 101 [{"field" => nil}], 102 ] 103 target = build(:int32, records) 104 assert_equal(records, target.raw_records) 105 end 106 107 def test_uint32 108 records = [ 109 [{"field" => (2 ** 32) - 1}], 110 [nil], 111 [{"field" => nil}], 112 ] 113 target = build(:uint32, records) 114 assert_equal(records, target.raw_records) 115 end 116 117 def test_int64 118 records = [ 119 [{"field" => -(2 ** 63)}], 120 [nil], 121 [{"field" => nil}], 122 ] 123 target = build(:int64, records) 124 assert_equal(records, target.raw_records) 125 end 126 127 def test_uint64 128 records = [ 129 [{"field" => (2 ** 64) - 1}], 130 [nil], 131 [{"field" => nil}], 132 ] 133 target = build(:uint64, records) 134 assert_equal(records, target.raw_records) 135 end 136 137 def test_float 138 records = [ 139 [{"field" => -1.0}], 140 [nil], 141 [{"field" => nil}], 142 ] 143 target = build(:float, records) 144 assert_equal(records, target.raw_records) 145 end 146 147 def test_double 148 records = [ 149 [{"field" => -1.0}], 150 [nil], 151 [{"field" => nil}], 152 ] 153 target = build(:double, records) 154 assert_equal(records, target.raw_records) 155 end 156 157 def test_binary 158 records = [ 159 [{"field" => "\xff".b}], 160 [nil], 161 [{"field" => nil}], 162 ] 163 target = build(:binary, records) 164 assert_equal(records, target.raw_records) 165 end 166 167 def test_string 168 records = [ 169 [{"field" => "Ruby"}], 170 [nil], 171 [{"field" => nil}], 172 ] 173 target = build(:string, records) 174 assert_equal(records, target.raw_records) 175 end 176 177 def test_date32 178 records = [ 179 [{"field" => Date.new(1960, 1, 1)}], 180 [nil], 181 [{"field" => nil}], 182 ] 183 target = build(:date32, records) 184 assert_equal(records, target.raw_records) 185 end 186 187 def test_date64 188 records = [ 189 [{"field" => DateTime.new(1960, 1, 1, 2, 9, 30)}], 190 [nil], 191 [{"field" => nil}], 192 ] 193 target = build(:date64, records) 194 assert_equal(records, target.raw_records) 195 end 196 197 def test_timestamp_second 198 records = [ 199 [{"field" => Time.parse("1960-01-01T02:09:30Z")}], 200 [nil], 201 [{"field" => nil}], 202 ] 203 target = build({ 204 type: :timestamp, 205 unit: :second, 206 }, 207 records) 208 assert_equal(records, target.raw_records) 209 end 210 211 def test_timestamp_milli 212 records = [ 213 [{"field" => Time.parse("1960-01-01T02:09:30.123Z")}], 214 [nil], 215 [{"field" => nil}], 216 ] 217 target = build({ 218 type: :timestamp, 219 unit: :milli, 220 }, 221 records) 222 assert_equal(records, target.raw_records) 223 end 224 225 def test_timestamp_micro 226 records = [ 227 [{"field" => Time.parse("1960-01-01T02:09:30.123456Z")}], 228 [nil], 229 [{"field" => nil}], 230 ] 231 target = build({ 232 type: :timestamp, 233 unit: :micro, 234 }, 235 records) 236 assert_equal(records, target.raw_records) 237 end 238 239 def test_timestamp_nano 240 records = [ 241 [{"field" => Time.parse("1960-01-01T02:09:30.123456789Z")}], 242 [nil], 243 [{"field" => nil}], 244 ] 245 target = build({ 246 type: :timestamp, 247 unit: :nano, 248 }, 249 records) 250 assert_equal(records, target.raw_records) 251 end 252 253 def test_time32_second 254 unit = Arrow::TimeUnit::SECOND 255 records = [ 256 # 00:10:00 257 [{"field" => Arrow::Time.new(unit, 60 * 10)}], 258 [nil], 259 [{"field" => nil}], 260 ] 261 target = build({ 262 type: :time32, 263 unit: :second, 264 }, 265 records) 266 assert_equal(records, target.raw_records) 267 end 268 269 def test_time32_milli 270 unit = Arrow::TimeUnit::MILLI 271 records = [ 272 # 00:10:00.123 273 [{"field" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}], 274 [nil], 275 [{"field" => nil}], 276 ] 277 target = build({ 278 type: :time32, 279 unit: :milli, 280 }, 281 records) 282 assert_equal(records, target.raw_records) 283 end 284 285 def test_time64_micro 286 unit = Arrow::TimeUnit::MICRO 287 records = [ 288 # 00:10:00.123456 289 [{"field" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}], 290 [nil], 291 [{"field" => nil}], 292 ] 293 target = build({ 294 type: :time64, 295 unit: :micro, 296 }, 297 records) 298 assert_equal(records, target.raw_records) 299 end 300 301 def test_time64_nano 302 unit = Arrow::TimeUnit::NANO 303 records = [ 304 # 00:10:00.123456789 305 [{"field" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}], 306 [nil], 307 [{"field" => nil}], 308 ] 309 target = build({ 310 type: :time64, 311 unit: :nano, 312 }, 313 records) 314 assert_equal(records, target.raw_records) 315 end 316 317 def test_decimal128 318 records = [ 319 [{"field" => BigDecimal("92.92")}], 320 [nil], 321 [{"field" => nil}], 322 ] 323 target = build({ 324 type: :decimal128, 325 precision: 8, 326 scale: 2, 327 }, 328 records) 329 assert_equal(records, target.raw_records) 330 end 331 332 def test_decimal256 333 records = [ 334 [{"field" => BigDecimal("92.92")}], 335 [nil], 336 [{"field" => nil}], 337 ] 338 target = build({ 339 type: :decimal256, 340 precision: 38, 341 scale: 2, 342 }, 343 records) 344 assert_equal(records, target.raw_records) 345 end 346 347 def test_list 348 records = [ 349 [{"field" => [true, nil, false]}], 350 [nil], 351 [{"field" => nil}], 352 ] 353 target = build({ 354 type: :list, 355 field: { 356 name: :sub_element, 357 type: :boolean, 358 }, 359 }, 360 records) 361 assert_equal(records, target.raw_records) 362 end 363 364 def test_struct 365 records = [ 366 [{"field" => {"sub_field" => true}}], 367 [nil], 368 [{"field" => nil}], 369 [{"field" => {"sub_field" => nil}}], 370 ] 371 target = build({ 372 type: :struct, 373 fields: [ 374 { 375 name: :sub_field, 376 type: :boolean, 377 }, 378 ], 379 }, 380 records) 381 assert_equal(records, target.raw_records) 382 end 383 384 def test_map 385 records = [ 386 [{"field" => {"key1" => true, "key2" => nil}}], 387 [nil], 388 [{"field" => nil}], 389 ] 390 target = build({ 391 type: :map, 392 key: :string, 393 item: :boolean, 394 }, 395 records) 396 assert_equal(records, target.raw_records) 397 end 398 399 def test_sparse_union 400 omit("Need to add support for SparseUnionArrayBuilder") 401 records = [ 402 [{"field" => {"field1" => true}}], 403 [nil], 404 [{"field" => nil}], 405 [{"field" => {"field2" => nil}}], 406 ] 407 target = build({ 408 type: :sparse_union, 409 fields: [ 410 { 411 name: :field1, 412 type: :boolean, 413 }, 414 { 415 name: :field2, 416 type: :uint8, 417 }, 418 ], 419 type_codes: [0, 1], 420 }, 421 records) 422 assert_equal(records, target.raw_records) 423 end 424 425 def test_dense_union 426 omit("Need to add support for DenseUnionArrayBuilder") 427 records = [ 428 [{"field" => {"field1" => true}}], 429 [nil], 430 [{"field" => nil}], 431 [{"field" => {"field2" => nil}}], 432 ] 433 target = build({ 434 type: :dense_union, 435 fields: [ 436 { 437 name: :field1, 438 type: :boolean, 439 }, 440 { 441 name: :field2, 442 type: :uint8, 443 }, 444 ], 445 type_codes: [0, 1], 446 }, 447 records) 448 assert_equal(records, target.raw_records) 449 end 450 451 def test_dictionary 452 omit("Need to add support for DictionaryArrayBuilder") 453 records = [ 454 [{"field" => "Ruby"}], 455 [nil], 456 [{"field" => nil}], 457 [{"field" => "GLib"}], 458 ] 459 dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) 460 target = build({ 461 type: :dictionary, 462 index_data_type: :int8, 463 dictionary: dictionary, 464 ordered: true, 465 }, 466 records) 467 assert_equal(records, target.raw_records) 468 end 469end 470 471class RawRecordsRecordBatchStructArrayTest < Test::Unit::TestCase 472 include RawRecordsStructArrayTests 473 474 def build(type, records) 475 Arrow::RecordBatch.new(build_schema(type), records) 476 end 477end 478 479class RawRecordsTableStructArrayTest < Test::Unit::TestCase 480 include RawRecordsStructArrayTests 481 482 def build(type, records) 483 Arrow::Table.new(build_schema(type), records) 484 end 485end 486