1require 'net/protocol'
2require 'uri'
3require 'timeout'
4
5
6#
7#
8# added @raw
9# added @real_response
10#
11#
12
13# Modified HTTP to return the real header block
14# This *works* with Ruby1.8.7 and Ruby1.9.1
15
16# HTTPS loses SPI
17
18# The ExtendedHTTP class is used in place of the HTTP class
19# for example,
20# http=ExtendedHTTP.new(@uri.host,@uri.port)
21
22# The ExtendedHTTP class uses the ExtendedHTTPResponse class
23
24class ExtendedHTTP < Net::HTTP   #:nodoc:
25   include Net
26
27#  class HTTP < Protocol
28
29    # Creates a new Net::HTTP object for the specified +address+.
30    # This method does not open the TCP connection.
31    #
32    # ExtendedHTTP :: initialize
33    # added @raw=[]
34
35    def initialize(address, port = nil)
36      @address = address
37      @port    = (port || HTTP.default_port)
38      @curr_http_version = HTTPVersion
39      @no_keepalive_server = false
40      @close_on_empty_response = false
41      @socket  = nil
42      @started = false
43      @open_timeout = nil
44      @read_timeout = 60
45      @continue_timeout = nil
46      @debug_output = nil
47      @use_ssl = false
48      @ssl_context = nil
49      @enable_post_connection_check = true
50      @compression = nil
51      @sspi_enabled = false
52
53      #$DEBUG=true
54
55      if defined?(SSL_ATTRIBUTES)
56        SSL_ATTRIBUTES.each do |name|
57          instance_variable_set "@#{name}", nil
58        end
59      end
60
61      # added for whatweb
62	  @raw = []
63    end
64
65	# ExtendedHTTP :: raw
66	# added def raw
67    def raw
68	     @raw
69    end
70
71	# ExtendedHTTP :: raw
72	# added @raw
73    def connect
74      D "opening connection to #{conn_address()}..."
75      s = timeout(@open_timeout) { TCPSocket.open(conn_address(), conn_port()) }
76      D "opened"
77      if use_ssl?
78        ssl_parameters = Hash.new
79        iv_list = instance_variables
80        SSL_ATTRIBUTES.each do |name|
81          ivname = "@#{name}".intern
82          if iv_list.include?(ivname) and
83             value = instance_variable_get(ivname)
84            ssl_parameters[name] = value
85          end
86        end
87        @ssl_context = OpenSSL::SSL::SSLContext.new
88        @ssl_context.set_params(ssl_parameters)
89
90        s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context)
91        s.sync_close = true
92      end
93      @socket = BufferedIO.new(s)
94      @socket.read_timeout = @read_timeout
95      @socket.continue_timeout = @continue_timeout
96      @socket.debug_output = @debug_output
97
98      if use_ssl?
99        begin
100          if proxy?
101            @socket.writeline sprintf('CONNECT %s:%s HTTP/%s',
102                                      @address, @port, HTTPVersion)
103            @socket.writeline "Host: #{@address}:#{@port}"
104            if proxy_user
105              credential = ["#{proxy_user}:#{proxy_pass}"].pack('m')
106              credential.delete!("\r\n")
107              @socket.writeline "Proxy-Authorization: Basic #{credential}"
108            end
109            @socket.writeline ''
110
111            # whatweb
112            # HTTPResponse.read_new(@socket).value
113    		  	x,raw = ExtendedHTTPResponse.read_new(@socket)
114    		  	@raw = raw
115    		  	res = x.value
116
117          end
118          # Server Name Indication (SNI) RFC 3546
119          s.hostname = @address if s.respond_to? :hostname=
120          timeout(@open_timeout) { s.connect }
121          if @ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE
122            s.post_connection_check(@address)
123          end
124        rescue => exception
125          D "Conn close because of connect error #{exception}"
126          @socket.close if @socket and not @socket.closed?
127          raise exception
128        end
129      end
130      on_connect
131    end
132    private :connect
133
134
135
136
137    def transport_request(req)
138      begin_transport req
139      res = catch(:response) {
140        req.exec @socket, @curr_http_version, edit_path(req.path)
141        begin
142            # added for whatweb
143            #res = HTTPResponse.read_new(@socket)
144            res, y = ExtendedHTTPResponse.read_new(@socket)
145	  		@raw << y
146	  		#
147        end while res.kind_of?(HTTPContinue)
148        res.reading_body(@socket, req.response_body_permitted?) {
149          yield res if block_given?
150        }
151        res
152      }
153      end_transport req, res
154      res
155    rescue => exception
156      D "Conn close because of error #{exception}"
157      @socket.close if @socket and not @socket.closed?
158      raise exception
159    end
160
161
162end
163
164
165#	added @raw
166 class ExtendedHTTPResponse < Net::HTTPResponse # reopen
167	include Net
168
169    class << self
170
171	  # read_new and read_status_line are copied from ruby 2
172      def read_new(sock)   #:nodoc: internal use only
173        x,httpv, code, msg = read_status_line(sock)
174		@rawlines= x + "\n"
175        res = response_class(code).new(httpv, code, msg)
176        each_response_header(sock) do |k,v|
177          res.add_field k, v
178        end
179		    # added for whatweb
180		    real = @rawlines
181        #pp real
182        [res,real]
183      end
184
185      private
186
187      def read_status_line(sock)
188        str = sock.readline
189        m = /\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)\s*(.*)\z/in.match(str) or
190          raise HTTPBadResponse, "wrong status line: #{str.dump}"
191        [str] + m.captures
192      end
193
194
195      def each_response_header(sock)
196        key = value = nil
197        while true
198          line = sock.readuntil("\n", true).sub(/\s+\z/, '')
199 	        # added for whatweb
200	        @rawlines << line + "\n" unless line.nil?
201	        #
202
203          break if line.empty?
204          if line[0] == ?\s or line[0] == ?\t and value
205            value << ' ' unless value.empty?
206            value << line.strip
207          else
208            yield key, value if key
209            key, value = line.strip.split(/\s*:\s*/, 2)
210            raise Net::HTTPBadResponse, 'wrong header line format' if value.nil?
211          end
212        end
213        yield key, value if key
214      end
215    end
216
217    public
218
219    def initialize(httpv, code, msg)   #:nodoc: internal use only
220      @http_version = httpv
221      @code         = code
222      @message      = msg
223      initialize_http_header nil
224      @body = nil
225      @read = false
226
227      # whatweb
228      @realresponse=[]
229      @rawlines=""
230    end
231
232end
233
234
235
236