1require 'net/protocol' 2require 'uri' 3require 'timeout' 4 5 6# 7# 8# added @raw 9# added @real_response 10# 11# 12 13# Modified HTTP to return the real header block 14# This *works* with Ruby1.8.7 and Ruby1.9.1 15 16# HTTPS loses SPI 17 18# The ExtendedHTTP class is used in place of the HTTP class 19# for example, 20# http=ExtendedHTTP.new(@uri.host,@uri.port) 21 22# The ExtendedHTTP class uses the ExtendedHTTPResponse class 23 24class ExtendedHTTP < Net::HTTP #:nodoc: 25 include Net 26 27# class HTTP < Protocol 28 29 # Creates a new Net::HTTP object for the specified +address+. 30 # This method does not open the TCP connection. 31 # 32 # ExtendedHTTP :: initialize 33 # added @raw=[] 34 35 def initialize(address, port = nil) 36 @address = address 37 @port = (port || HTTP.default_port) 38 @curr_http_version = HTTPVersion 39 @no_keepalive_server = false 40 @close_on_empty_response = false 41 @socket = nil 42 @started = false 43 @open_timeout = nil 44 @read_timeout = 60 45 @continue_timeout = nil 46 @debug_output = nil 47 @use_ssl = false 48 @ssl_context = nil 49 @enable_post_connection_check = true 50 @compression = nil 51 @sspi_enabled = false 52 53 #$DEBUG=true 54 55 if defined?(SSL_ATTRIBUTES) 56 SSL_ATTRIBUTES.each do |name| 57 instance_variable_set "@#{name}", nil 58 end 59 end 60 61 # added for whatweb 62 @raw = [] 63 end 64 65 # ExtendedHTTP :: raw 66 # added def raw 67 def raw 68 @raw 69 end 70 71 # ExtendedHTTP :: raw 72 # added @raw 73 def connect 74 D "opening connection to #{conn_address()}..." 75 s = timeout(@open_timeout) { TCPSocket.open(conn_address(), conn_port()) } 76 D "opened" 77 if use_ssl? 78 ssl_parameters = Hash.new 79 iv_list = instance_variables 80 SSL_ATTRIBUTES.each do |name| 81 ivname = "@#{name}".intern 82 if iv_list.include?(ivname) and 83 value = instance_variable_get(ivname) 84 ssl_parameters[name] = value 85 end 86 end 87 @ssl_context = OpenSSL::SSL::SSLContext.new 88 @ssl_context.set_params(ssl_parameters) 89 90 s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context) 91 s.sync_close = true 92 end 93 @socket = BufferedIO.new(s) 94 @socket.read_timeout = @read_timeout 95 @socket.continue_timeout = @continue_timeout 96 @socket.debug_output = @debug_output 97 98 if use_ssl? 99 begin 100 if proxy? 101 @socket.writeline sprintf('CONNECT %s:%s HTTP/%s', 102 @address, @port, HTTPVersion) 103 @socket.writeline "Host: #{@address}:#{@port}" 104 if proxy_user 105 credential = ["#{proxy_user}:#{proxy_pass}"].pack('m') 106 credential.delete!("\r\n") 107 @socket.writeline "Proxy-Authorization: Basic #{credential}" 108 end 109 @socket.writeline '' 110 111 # whatweb 112 # HTTPResponse.read_new(@socket).value 113 x,raw = ExtendedHTTPResponse.read_new(@socket) 114 @raw = raw 115 res = x.value 116 117 end 118 # Server Name Indication (SNI) RFC 3546 119 s.hostname = @address if s.respond_to? :hostname= 120 timeout(@open_timeout) { s.connect } 121 if @ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE 122 s.post_connection_check(@address) 123 end 124 rescue => exception 125 D "Conn close because of connect error #{exception}" 126 @socket.close if @socket and not @socket.closed? 127 raise exception 128 end 129 end 130 on_connect 131 end 132 private :connect 133 134 135 136 137 def transport_request(req) 138 begin_transport req 139 res = catch(:response) { 140 req.exec @socket, @curr_http_version, edit_path(req.path) 141 begin 142 # added for whatweb 143 #res = HTTPResponse.read_new(@socket) 144 res, y = ExtendedHTTPResponse.read_new(@socket) 145 @raw << y 146 # 147 end while res.kind_of?(HTTPContinue) 148 res.reading_body(@socket, req.response_body_permitted?) { 149 yield res if block_given? 150 } 151 res 152 } 153 end_transport req, res 154 res 155 rescue => exception 156 D "Conn close because of error #{exception}" 157 @socket.close if @socket and not @socket.closed? 158 raise exception 159 end 160 161 162end 163 164 165# added @raw 166 class ExtendedHTTPResponse < Net::HTTPResponse # reopen 167 include Net 168 169 class << self 170 171 # read_new and read_status_line are copied from ruby 2 172 def read_new(sock) #:nodoc: internal use only 173 x,httpv, code, msg = read_status_line(sock) 174 @rawlines= x + "\n" 175 res = response_class(code).new(httpv, code, msg) 176 each_response_header(sock) do |k,v| 177 res.add_field k, v 178 end 179 # added for whatweb 180 real = @rawlines 181 #pp real 182 [res,real] 183 end 184 185 private 186 187 def read_status_line(sock) 188 str = sock.readline 189 m = /\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)\s*(.*)\z/in.match(str) or 190 raise HTTPBadResponse, "wrong status line: #{str.dump}" 191 [str] + m.captures 192 end 193 194 195 def each_response_header(sock) 196 key = value = nil 197 while true 198 line = sock.readuntil("\n", true).sub(/\s+\z/, '') 199 # added for whatweb 200 @rawlines << line + "\n" unless line.nil? 201 # 202 203 break if line.empty? 204 if line[0] == ?\s or line[0] == ?\t and value 205 value << ' ' unless value.empty? 206 value << line.strip 207 else 208 yield key, value if key 209 key, value = line.strip.split(/\s*:\s*/, 2) 210 raise Net::HTTPBadResponse, 'wrong header line format' if value.nil? 211 end 212 end 213 yield key, value if key 214 end 215 end 216 217 public 218 219 def initialize(httpv, code, msg) #:nodoc: internal use only 220 @http_version = httpv 221 @code = code 222 @message = msg 223 initialize_http_header nil 224 @body = nil 225 @read = false 226 227 # whatweb 228 @realresponse=[] 229 @rawlines="" 230 end 231 232end 233 234 235 236