1#!/usr/bin/env ruby 2# -*- encoding: binary -*- 3# Copyright (C) 2012-2020 all contributors <cmogstored-public@yhbt.net> 4# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt> 5require 'test/test_helper' 6require 'digest/md5' 7require 'net/http' 8require 'stringio' 9 10class TestHTTPChunkedPut < Test::Unit::TestCase 11 def setup 12 @tmpdir = Dir.mktmpdir('cmogstored-httpchunkedput-test') 13 Dir.mkdir("#@tmpdir/dev666") 14 Dir.mkdir("#@tmpdir/dev6") 15 @to_close = [] 16 @host = TEST_HOST 17 srv = TCPServer.new(@host, 0) 18 @port = srv.addr[1] 19 srv.close 20 @err = Tempfile.new("stderr") 21 cmd = [ "cmogstored", "--docroot=#@tmpdir", "--httplisten=#@host:#@port", 22 "--maxconns=500" ] 23 vg = ENV["VALGRIND"] and cmd = vg.split(/\s+/).concat(cmd) 24 @pid = fork { 25 # $stderr.reopen(@err) 26 @err.close 27 exec(*cmd) 28 } 29 @client = get_client 30 end 31 32 def teardown 33 Process.kill(:QUIT, @pid) rescue nil 34 _, status = Process.waitpid2(@pid) 35 @to_close.each { |io| io.close unless io.closed? } 36 FileUtils.rm_rf(@tmpdir) 37 @err.rewind 38 $stderr.write(@err.read) 39 assert status.success?, status.inspect 40 end 41 42 def test_put 43 Net::HTTP.start(@host, @port) do |http| 44 4.times do |i| 45 put = Net::HTTP::Put.new("/dev666/foo#{i}") 46 body = StringIO.new("BODY!") 47 put.content_type = "application/octet-stream" 48 put["Transfer-Encoding"] = "chunked" 49 put.body_stream = body 50 resp = http.request(put) 51 assert_equal 201, resp.code.to_i, "i=#{i}" 52 assert_equal "BODY!", IO.read("#@tmpdir/dev666/foo#{i}") 53 end 54 end 55 end 56 57 def test_single_write 58 req = "PUT /dev666/zz HTTP/1.1\r\n" \ 59 "Host: #@host:#@port\r\n" \ 60 "Transfer-Encoding: chunked\r\n" \ 61 "\r\n" \ 62 "5\r\nabcde\r\n0\r\n\r\n" 63 @client.write(req) 64 check_abcde 65 end 66 67 def test_edge_finder_no_trailer 68 edge_finder("\r\n") 69 end 70 71 def test_edge_finder_no_trailer_with_get 72 edge_finder("\r\nGET") 73 end 74 75 def test_edge_finder_full_trailer 76 edge_finder("Foo: Bar\r\n\r\n") 77 end 78 79 def test_edge_finder_full_trailer_with_get 80 edge_finder("Foo: Bar\r\n\r\nGET") 81 end 82 83 def test_edge_finder_pipelined 84 edge_finder(:another) 85 end 86 87 def edge_finder(suf) 88 expect = "HTTP/1.1 200 OK\r\n" \ 89 "Date: #{EPOCH}\r\n" \ 90 "Last-Modified: #{EPOCH}\r\n" \ 91 "Content-Length: 5\r\n" \ 92 "Content-Type: application/octet-stream\r\n" \ 93 "Accept-Ranges: bytes\r\n" \ 94 "Connection: close\r\n\r\nabcde" 95 base = "PUT /dev666/zz HTTP/1.1\r\n" \ 96 "Host: #@host:#@port\r\n" \ 97 "Transfer-Encoding: chunked\r\n" \ 98 "\r\n" \ 99 "5\r\nabcde\r\n0\r\n" 100 101 suf = another = "\r\n#{base.sub(%r{/zz}, "/yy")}\r\n" if suf == :another 102 req = base + suf 103 (1..(req.size-1)).each do |i| 104 @client.write(req[0,i]) 105 t_yield 106 @client.write(req[i, req.size]) 107 check_abcde("i=#{i} size=#{req.size}") 108 case suf 109 when another 110 t_yield 111 check_abcde("i=#{i} size=#{req.size} (again)") 112 assert_equal "abcde", IO.read("#@tmpdir/dev666/yy"), "i=#{i}" 113 when /GET\z/ 114 t_yield 115 @client.write " /dev666/zz HTTP/1.0\r\n\r\n" 116 got = @client.read(expect.size) 117 replace_dates!(got) 118 assert_equal expect, got, "i=#{i}" 119 end 120 @client.close 121 @client = TCPSocket.new(@host, @port) 122 end 123 end 124 125 def check_abcde(msg=nil) 126 expect = "HTTP/1.1 201 Created\r\n" \ 127 "Date: #{EPOCH}\r\n" \ 128 "Content-Length: 0\r\n" \ 129 "Content-Type: text/plain\r\n" \ 130 "Connection: keep-alive\r\n\r\n" 131 resp = @client.readpartial(expect.size) 132 assert_kind_of String, resp 133 replace_dates!(resp) 134 assert_equal expect, resp, msg 135 assert_equal "abcde", IO.read("#@tmpdir/dev666/zz") 136 end 137 138 def test_boundary_chunk_size 139 req = "PUT /dev666/zz HTTP/1.1\r\n" \ 140 "Host: #@host:#@port\r\n" \ 141 "Transfer-Encoding: chunked\r\n" \ 142 "\r\n" \ 143 "5" 144 more = "\r\nabcde\r\n0\r\n\r\n" 145 @client.write(req) 146 t_yield 147 @client.write(more) 148 check_abcde 149 end 150 151 def test_boundary_chunk_data 152 req = "PUT /dev666/zz HTTP/1.1\r\n" \ 153 "Host: #@host:#@port\r\n" \ 154 "Transfer-Encoding: chunked\r\n" \ 155 "\r\n" \ 156 "5\r\na" 157 more = "bcde\r\n0\r\n\r\n" 158 @client.write(req) 159 t_yield 160 @client.write(more) 161 check_abcde 162 end 163 164 def test_boundary_chunk_size_last 165 req = "PUT /dev666/zz HTTP/1.1\r\n" \ 166 "Host: #@host:#@port\r\n" \ 167 "Transfer-Encoding: chunked\r\n" \ 168 "\r\n" \ 169 "5\r\nabcde\r\n0\r" 170 more = "\n\r\n" 171 @client.write(req) 172 t_yield 173 @client.write(more) 174 check_abcde 175 end 176 177 def test_boundary_chunk_trailer 178 req = "PUT /dev666/zz HTTP/1.1\r\n" \ 179 "Host: #@host:#@port\r\n" \ 180 "Transfer-Encoding: chunked\r\n" \ 181 "\r\n" \ 182 "5\r\nabcde\r\n0\r\n" 183 more = "\r\n" 184 @client.write(req) 185 t_yield 186 @client.write(more) 187 check_abcde 188 end 189 190 def test_boundary_chunk_trailer_md5 191 req = "PUT /dev666/zz HTTP/1.1\r\n" \ 192 "Host: #@host:#@port\r\n" \ 193 "Transfer-Encoding: chunked\r\n" \ 194 "\r\n" \ 195 "5\r\nabcde\r\n0\r\nContent-MD5:" 196 more = " q1a02StAcTrMWviZhdS3hg==\r\n\r\n" 197 @client.write(req) 198 t_yield 199 @client.write(more) 200 check_abcde 201 end 202 203 def test_boundary_chunk_trailer_md5_pipelined 204 req = "PUT /dev666/zz HTTP/1.1\r\n" \ 205 "Host: #@host:#@port\r\n" \ 206 "Transfer-Encoding: chunked\r\n" \ 207 "\r\n" \ 208 "5\r\nabcde\r\n0\r\nContent-MD5:" 209 more = " q1a02StAcTrMWviZhdS3hg==\r\n\r\nGET" 210 moar = " /dev666/zz HTTP/1.1\r\n" \ 211 "Host: #@host:#@port\r\n" \ 212 "\r\n" 213 @client.write(req) 214 t_yield 215 @client.write(more) 216 check_abcde 217 t_yield 218 @client.write(moar) 219 expect = "HTTP/1.1 200 OK\r\n" \ 220 "Date: #{EPOCH}\r\n" \ 221 "Last-Modified: #{EPOCH}\r\n" \ 222 "Content-Length: 5\r\n" \ 223 "Content-Type: application/octet-stream\r\n" \ 224 "Accept-Ranges: bytes\r\n" \ 225 "Connection: keep-alive\r\n\r\nabcde" 226 resp = @client.read(expect.size) 227 replace_dates!(resp) 228 assert_equal expect, resp 229 end 230 231 def test_boundary_pipelined 232 req = "PUT /dev666/zz HTTP/1.1\r\n" \ 233 "Host: #@host:#@port\r\n" \ 234 "Transfer-Encoding: chunked\r\n" \ 235 "\r\n" \ 236 "5\r\nabcde\r\n0\r\n\r\nGET" 237 more = " /dev666/zz HTTP/1.1\r\n" \ 238 "Host: #@host:#@port\r\n" \ 239 "\r\n" 240 @client.write(req) 241 t_yield 242 check_abcde 243 @client.write(more) 244 expect = "HTTP/1.1 200 OK\r\n" \ 245 "Date: #{EPOCH}\r\n" \ 246 "Last-Modified: #{EPOCH}\r\n" \ 247 "Content-Length: 5\r\n" \ 248 "Content-Type: application/octet-stream\r\n" \ 249 "Accept-Ranges: bytes\r\n" \ 250 "Connection: keep-alive\r\n\r\nabcde" 251 resp = @client.read(expect.size) 252 replace_dates!(resp) 253 assert_equal expect, resp 254 end 255 256 def test_boundary_pipelined_slow_1 257 req = "PUT /dev666/zz HTTP/1.1\r\n" \ 258 "Host: #@host:#@port\r\n" \ 259 "Transfer-Encoding: chunked\r\n" \ 260 "\r\n" \ 261 "5\r\nabcde\r\n0" 262 more = "\r\n\r\nGET" 263 moar = " /dev666/zz HTTP/1.1\r\n" \ 264 "Host: #@host:#@port\r\n" \ 265 "\r\n" 266 @client.write(req) 267 t_yield 268 @client.write(more) 269 check_abcde 270 t_yield 271 @client.write(moar) 272 expect = "HTTP/1.1 200 OK\r\n" \ 273 "Date: #{EPOCH}\r\n" \ 274 "Last-Modified: #{EPOCH}\r\n" \ 275 "Content-Length: 5\r\n" \ 276 "Content-Type: application/octet-stream\r\n" \ 277 "Accept-Ranges: bytes\r\n" \ 278 "Connection: keep-alive\r\n\r\nabcde" 279 resp = @client.read(expect.size) 280 replace_dates!(resp) 281 assert_equal expect, resp 282 end 283 284 def test_zero_byte 285 req = "PUT /dev666/zero HTTP/1.1\r\n" \ 286 "Host: #@host:#@port\r\n" \ 287 "Transfer-Encoding: chunked\r\n\r\n" \ 288 "0\r\n\r\n" 289 @client.write(req) 290 expect = "HTTP/1.1 201 Created\r\n" \ 291 "Date: #{EPOCH}\r\n" \ 292 "Content-Length: 0\r\n" \ 293 "Content-Type: text/plain\r\n" \ 294 "Connection: keep-alive\r\n" \ 295 "\r\n" 296 resp = @client.readpartial(666) 297 replace_dates!(resp) 298 assert_equal expect, resp 299 assert_equal "", IO.read("#@tmpdir/dev666/zero") 300 assert_nil IO.select([@client], nil, nil, 0.1) 301 end 302 303 def test_pipelined 304 req = 2.times.map do |i| 305 "PUT /dev666/#{i} HTTP/1.1\r\n" \ 306 "Host: #@host:#@port\r\n" \ 307 "Transfer-Encoding: chunked\r\n" \ 308 "\r\n" \ 309 "5\r\nabcd#{i}\r\n0\r\n\r\n" 310 end.join 311 @client.write(req) 312 expect = "HTTP/1.1 201 Created\r\n" \ 313 "Date: #{EPOCH}\r\n" \ 314 "Content-Length: 0\r\n" \ 315 "Content-Type: text/plain\r\n" \ 316 "Connection: keep-alive\r\n\r\n" 317 resp = @client.read(expect.size * 2) 318 replace_dates!(resp) 319 assert_equal(expect * 2, resp) 320 assert_equal "abcd0", IO.read("#@tmpdir/dev666/0") 321 assert_equal "abcd1", IO.read("#@tmpdir/dev666/1") 322 assert_nil IO.select([@client], nil, nil, 0.1) 323 end 324 325 def test_pipelined_slow 326 req = 2.times.map do |i| 327 "PUT /dev666/#{i} HTTP/1.1\r\n" \ 328 "Host: #@host:#@port\r\n" \ 329 "Transfer-Encoding: chunked\r\n" \ 330 "\r\n" \ 331 "5\r\nabcd#{i}\r\n0\r\n\r\n" 332 end.join 333 334 req.each_byte do |x| 335 @client.write(x.chr) 336 t_yield 337 end 338 339 expect = "HTTP/1.1 201 Created\r\n" \ 340 "Date: #{EPOCH}\r\n" \ 341 "Content-Length: 0\r\n" \ 342 "Content-Type: text/plain\r\n" \ 343 "Connection: keep-alive\r\n\r\n" 344 resp = @client.read(expect.size * 2) 345 replace_dates!(resp) 346 assert_equal(expect * 2, resp) 347 assert_equal "abcd0", IO.read("#@tmpdir/dev666/0") 348 assert_equal "abcd1", IO.read("#@tmpdir/dev666/1") 349 assert_nil IO.select([@client], nil, nil, 0.1) 350 end 351 352 def test_put_chunk_len_overflow 353 max = 0xffffffff << 64 354 req = "PUT /dev666/foo HTTP/1.0\r\n" \ 355 "Transfer-Encoding: chunked\r\n" \ 356 "\r\n" \ 357 "#{sprintf("%x", max)}\r\n" 358 @client.write(req) 359 resp = @client.read 360 assert_match(%r{\AHTTP/1\.1 507 Insufficient Storage\r\n}, resp) 361 assert ! File.exist?("#@tmpdir/dev666/foo") 362 end 363 364 def test_content_md5_good 365 req = "PUT /dev666/foo HTTP/1.1\r\n" \ 366 "Host: #@host:#@port\r\n" \ 367 "Transfer-Encoding: chunked\r\n" \ 368 "Trailer: Content-MD5\r\n" \ 369 "\r\n" \ 370 "5\r\nabcde\r\n0\r\n" \ 371 "Content-MD5: q1a02StAcTrMWviZhdS3hg==\r\n" \ 372 "\r\n" 373 @client.write(req) 374 line = @client.gets 375 assert_match(%r{\AHTTP/1\.1 201 Created}, line) 376 assert_equal "abcde", File.read("#@tmpdir/dev666/foo") 377 end 378 379 def test_content_md5_bad 380 req = "PUT /dev666/foo HTTP/1.1\r\n" \ 381 "Host: #@host:#@port\r\n" \ 382 "Transfer-Encoding: chunked\r\n" \ 383 "Trailer: Content-MD5\r\n" \ 384 "\r\n" \ 385 "5\r\nabcd!\r\n0\r\n" \ 386 "Content-MD5: q1a02StAcTrMWviZhdS3hg==\r\n" \ 387 "\r\n" 388 @client.write(req) 389 line = @client.gets 390 assert_match(%r{\AHTTP/1\.1 400 Bad Request}, line) 391 assert ! File.exist?("#@tmpdir/dev666/foo") 392 end 393 394 def test_content_md5_funky_boundary 395 # This bug was originally found when I attempted to upload a 396 # 198689228 byte file to 127.0.0.1:7600 with Ruby 397 # mogilefs-client 3.4.0 using chunked encoding and the 398 # Content-MD5 trailer 399 md5 = Digest::MD5.new 400 extra = "#@host:#@port".size - "127.0.0.1:7600".size 401 fid = "0000026374" 402 fid = fid[0...(fid.size - extra)] 403 req = "PUT /dev6/0/000/026/#{fid}.fid HTTP/1.1\r\n" \ 404 "Host: #@host:#@port\r\n" \ 405 "Trailer: Content-MD5\r\n" \ 406 "Transfer-Encoding: chunked\r\n" \ 407 "\r\n" 408 409 bytes = 16384 * 500 + 460 410 bytes = 198689228 if ENV["TEST_EXPENSIVE"] 411 chunk_len = 16384 412 chunk = '*' * chunk_len 413 chunk_str = "4000\r\n#{chunk}\r\n" 414 415 @client.write(req) 416 while bytes > chunk_len 417 bytes -= chunk_len 418 @client.write(chunk_str) 419 md5.update(chunk) 420 end 421 422 final = '!' * bytes 423 @client.write("#{sprintf("%x\r\n", bytes)}#{final}\r\n") 424 md5.update(final) 425 426 content_md5 = [ md5.digest ].pack('m').rstrip! 427 @client.write("0\r\nContent-MD5: #{content_md5}\r\n\r\n") 428 ensure 429 line = @client.gets 430 assert_equal "HTTP/1.1 201 Created\r\n", line, line.inspect 431 end 432 433 def test_chunk_extension_small 434 req = "PUT /dev666/foo HTTP/1.1\r\n" \ 435 "Host: #@host:#@port\r\n" \ 436 "Transfer-Encoding: chunked\r\n" \ 437 "Trailer: Content-MD5\r\n" \ 438 "\r\n" \ 439 "5; foo=bar\r\nabcde\r\n0\r\n" \ 440 "Content-MD5: q1a02StAcTrMWviZhdS3hg==\r\n" \ 441 "\r\n" 442 @client.write(req) 443 line = @client.gets 444 assert_match(%r{\AHTTP/1\.1 201 Created}, line) 445 assert File.exist?("#@tmpdir/dev666/foo") 446 assert_equal "abcde", File.read("#@tmpdir/dev666/foo") 447 end 448 449 def test_chunk_extension_gigantic 450 req = "PUT /dev666/foo HTTP/1.1\r\n" \ 451 "Host: #@host:#@port\r\n" \ 452 "Transfer-Encoding: chunked\r\n" \ 453 "\r\n" 454 @client.write(req) 455 @client.write("5; foo=bar#{'IMABIRD'* 666666}") 456 @client.write("\r\nabcde\r\n0\r\n\r\n") 457 line = @client.gets 458 assert_match(%r{\AHTTP/1\.1 201 Created}, line) 459 assert_equal "abcde", File.read("#@tmpdir/dev666/foo") 460 end 461 462 def test_chunk_trailer_gigantic 463 req = "PUT /dev666/foo HTTP/1.1\r\n" \ 464 "Host: #@host:#@port\r\n" \ 465 "Transfer-Encoding: chunked\r\n" \ 466 "Trailer: OMG\r\n" \ 467 "\r\n" 468 @client.write(req) 469 @client.write("5\r\nabcde\r\n0\r\nOMG: WTFBBQ") 470 100000.times { @client.write("WTFBBQ") } 471 @client.write("\r\n\r\n") 472 line = @client.gets 473 assert_match(%r{\AHTTP/1\.1 201 Created}, line) 474 assert_equal "abcde", File.read("#@tmpdir/dev666/foo") 475 end 476end 477