1// Package netx contains code to perform network measurements. 2// 3// This library contains replacements for commonly used standard library 4// interfaces that facilitate seamless network measurements. By using 5// such replacements, as opposed to standard library interfaces, we can: 6// 7// * save the timing of HTTP events (e.g. received response headers) 8// * save the timing and result of every Connect, Read, Write, Close operation 9// * save the timing and result of the TLS handshake (including certificates) 10// 11// By default, this library uses the system resolver. In addition, it 12// is possible to configure alternative DNS transports and remote 13// servers. We support DNS over UDP, DNS over TCP, DNS over TLS (DoT), 14// and DNS over HTTPS (DoH). When using an alternative transport, we 15// are also able to intercept and save DNS messages, as well as any 16// other interaction with the remote server (e.g., the result of the 17// TLS handshake for DoT and DoH). 18// 19// We described the design and implementation of the most recent version of 20// this package at <https://github.com/ooni/probe-engine/issues/359>. Such 21// issue also links to a previous design document. 22package netx 23 24import ( 25 "context" 26 "crypto/tls" 27 "crypto/x509" 28 "errors" 29 "net" 30 "net/http" 31 "net/url" 32 33 "github.com/lucas-clemente/quic-go" 34 "github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter" 35 "github.com/ooni/probe-cli/v3/internal/engine/netx/dialer" 36 "github.com/ooni/probe-cli/v3/internal/engine/netx/gocertifi" 37 "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" 38 "github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer" 39 "github.com/ooni/probe-cli/v3/internal/engine/netx/resolver" 40 "github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor" 41 "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" 42 "github.com/ooni/probe-cli/v3/internal/engine/runtimex" 43) 44 45// Logger is the logger assumed by this package 46type Logger interface { 47 Debugf(format string, v ...interface{}) 48 Debug(message string) 49} 50 51// Dialer is the definition of dialer assumed by this package. 52type Dialer interface { 53 DialContext(ctx context.Context, network, address string) (net.Conn, error) 54} 55 56// QUICDialer is the definition of a dialer for QUIC assumed by this package. 57type QUICDialer interface { 58 Dial(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) 59} 60 61// TLSDialer is the definition of a TLS dialer assumed by this package. 62type TLSDialer interface { 63 DialTLSContext(ctx context.Context, network, address string) (net.Conn, error) 64} 65 66// HTTPRoundTripper is the definition of http.HTTPRoundTripper used by this package. 67type HTTPRoundTripper interface { 68 RoundTrip(req *http.Request) (*http.Response, error) 69 CloseIdleConnections() 70} 71 72// Resolver is the interface we expect from a resolver 73type Resolver interface { 74 LookupHost(ctx context.Context, hostname string) (addrs []string, err error) 75 Network() string 76 Address() string 77} 78 79// Config contains configuration for creating a new transport. When any 80// field of Config is nil/empty, we will use a suitable default. 81// 82// We use different savers for different kind of events such that the 83// user of this library can choose what to save. 84type Config struct { 85 BaseResolver Resolver // default: system resolver 86 BogonIsError bool // default: bogon is not error 87 ByteCounter *bytecounter.Counter // default: no explicit byte counting 88 CacheResolutions bool // default: no caching 89 CertPool *x509.CertPool // default: use vendored gocertifi 90 ContextByteCounting bool // default: no implicit byte counting 91 DNSCache map[string][]string // default: cache is empty 92 DialSaver *trace.Saver // default: not saving dials 93 Dialer Dialer // default: dialer.DNSDialer 94 FullResolver Resolver // default: base resolver + goodies 95 QUICDialer QUICDialer // default: quicdialer.DNSDialer 96 HTTP3Enabled bool // default: disabled 97 HTTPSaver *trace.Saver // default: not saving HTTP 98 Logger Logger // default: no logging 99 NoTLSVerify bool // default: perform TLS verify 100 ProxyURL *url.URL // default: no proxy 101 ReadWriteSaver *trace.Saver // default: not saving read/write 102 ResolveSaver *trace.Saver // default: not saving resolves 103 TLSConfig *tls.Config // default: attempt using h2 104 TLSDialer TLSDialer // default: dialer.TLSDialer 105 TLSSaver *trace.Saver // default: not saving TLS 106} 107 108type tlsHandshaker interface { 109 Handshake(ctx context.Context, conn net.Conn, config *tls.Config) ( 110 net.Conn, tls.ConnectionState, error) 111} 112 113// NewDefaultCertPool returns a copy of the default x509 114// certificate pool. This function panics on failure. 115func NewDefaultCertPool() *x509.CertPool { 116 pool, err := gocertifi.CACerts() 117 runtimex.PanicOnError(err, "gocertifi.CACerts() failed") 118 return pool 119} 120 121var defaultCertPool *x509.CertPool = NewDefaultCertPool() 122 123// NewResolver creates a new resolver from the specified config 124func NewResolver(config Config) Resolver { 125 if config.BaseResolver == nil { 126 config.BaseResolver = resolver.SystemResolver{} 127 } 128 var r Resolver = config.BaseResolver 129 r = resolver.AddressResolver{Resolver: r} 130 if config.CacheResolutions { 131 r = &resolver.CacheResolver{Resolver: r} 132 } 133 if config.DNSCache != nil { 134 cache := &resolver.CacheResolver{Resolver: r, ReadOnly: true} 135 for key, values := range config.DNSCache { 136 cache.Set(key, values) 137 } 138 r = cache 139 } 140 if config.BogonIsError { 141 r = resolver.BogonResolver{Resolver: r} 142 } 143 r = resolver.ErrorWrapperResolver{Resolver: r} 144 if config.Logger != nil { 145 r = resolver.LoggingResolver{Logger: config.Logger, Resolver: r} 146 } 147 if config.ResolveSaver != nil { 148 r = resolver.SaverResolver{Resolver: r, Saver: config.ResolveSaver} 149 } 150 return resolver.IDNAResolver{Resolver: r} 151} 152 153// NewDialer creates a new Dialer from the specified config 154func NewDialer(config Config) Dialer { 155 if config.FullResolver == nil { 156 config.FullResolver = NewResolver(config) 157 } 158 var d Dialer = selfcensor.SystemDialer{} 159 d = dialer.TimeoutDialer{Dialer: d} 160 d = dialer.ErrorWrapperDialer{Dialer: d} 161 if config.Logger != nil { 162 d = dialer.LoggingDialer{Dialer: d, Logger: config.Logger} 163 } 164 if config.DialSaver != nil { 165 d = dialer.SaverDialer{Dialer: d, Saver: config.DialSaver} 166 } 167 if config.ReadWriteSaver != nil { 168 d = dialer.SaverConnDialer{Dialer: d, Saver: config.ReadWriteSaver} 169 } 170 d = dialer.DNSDialer{Resolver: config.FullResolver, Dialer: d} 171 d = dialer.ProxyDialer{ProxyURL: config.ProxyURL, Dialer: d} 172 if config.ContextByteCounting { 173 d = dialer.ByteCounterDialer{Dialer: d} 174 } 175 d = dialer.ShapingDialer{Dialer: d} 176 return d 177} 178 179// NewQUICDialer creates a new DNS Dialer for QUIC, with the resolver from the specified config 180func NewQUICDialer(config Config) QUICDialer { 181 if config.FullResolver == nil { 182 config.FullResolver = NewResolver(config) 183 } 184 var d quicdialer.ContextDialer = &quicdialer.SystemDialer{Saver: config.ReadWriteSaver} 185 d = quicdialer.ErrorWrapperDialer{Dialer: d} 186 if config.TLSSaver != nil { 187 d = quicdialer.HandshakeSaver{Saver: config.TLSSaver, Dialer: d} 188 } 189 d = &quicdialer.DNSDialer{Resolver: config.FullResolver, Dialer: d} 190 var dialer QUICDialer = &httptransport.QUICWrapperDialer{Dialer: d} 191 return dialer 192} 193 194// NewTLSDialer creates a new TLSDialer from the specified config 195func NewTLSDialer(config Config) TLSDialer { 196 if config.Dialer == nil { 197 config.Dialer = NewDialer(config) 198 } 199 var h tlsHandshaker = dialer.SystemTLSHandshaker{} 200 h = dialer.TimeoutTLSHandshaker{TLSHandshaker: h} 201 h = dialer.ErrorWrapperTLSHandshaker{TLSHandshaker: h} 202 if config.Logger != nil { 203 h = dialer.LoggingTLSHandshaker{Logger: config.Logger, TLSHandshaker: h} 204 } 205 if config.TLSSaver != nil { 206 h = dialer.SaverTLSHandshaker{TLSHandshaker: h, Saver: config.TLSSaver} 207 } 208 if config.TLSConfig == nil { 209 config.TLSConfig = &tls.Config{NextProtos: []string{"h2", "http/1.1"}} 210 } 211 if config.CertPool == nil { 212 config.CertPool = defaultCertPool 213 } 214 config.TLSConfig.RootCAs = config.CertPool 215 config.TLSConfig.InsecureSkipVerify = config.NoTLSVerify 216 return dialer.TLSDialer{ 217 Config: config.TLSConfig, 218 Dialer: config.Dialer, 219 TLSHandshaker: h, 220 } 221} 222 223// NewHTTPTransport creates a new HTTPRoundTripper. You can further extend the returned 224// HTTPRoundTripper before wrapping it into an http.Client. 225func NewHTTPTransport(config Config) HTTPRoundTripper { 226 if config.Dialer == nil { 227 config.Dialer = NewDialer(config) 228 } 229 if config.TLSDialer == nil { 230 config.TLSDialer = NewTLSDialer(config) 231 } 232 if config.QUICDialer == nil { 233 config.QUICDialer = NewQUICDialer(config) 234 } 235 236 tInfo := allTransportsInfo[config.HTTP3Enabled] 237 txp := tInfo.Factory(httptransport.Config{ 238 Dialer: config.Dialer, QUICDialer: config.QUICDialer, TLSDialer: config.TLSDialer, 239 TLSConfig: config.TLSConfig}) 240 transport := tInfo.TransportName 241 242 if config.ByteCounter != nil { 243 txp = httptransport.ByteCountingTransport{ 244 Counter: config.ByteCounter, RoundTripper: txp} 245 } 246 if config.Logger != nil { 247 txp = httptransport.LoggingTransport{Logger: config.Logger, RoundTripper: txp} 248 } 249 if config.HTTPSaver != nil { 250 txp = httptransport.SaverMetadataHTTPTransport{ 251 RoundTripper: txp, Saver: config.HTTPSaver, Transport: transport} 252 txp = httptransport.SaverBodyHTTPTransport{ 253 RoundTripper: txp, Saver: config.HTTPSaver} 254 txp = httptransport.SaverPerformanceHTTPTransport{ 255 RoundTripper: txp, Saver: config.HTTPSaver} 256 txp = httptransport.SaverTransactionHTTPTransport{ 257 RoundTripper: txp, Saver: config.HTTPSaver} 258 } 259 txp = httptransport.UserAgentTransport{RoundTripper: txp} 260 return txp 261} 262 263// httpTransportInfo contains the constructing function as well as the transport name 264type httpTransportInfo struct { 265 Factory func(httptransport.Config) httptransport.RoundTripper 266 TransportName string 267} 268 269var allTransportsInfo = map[bool]httpTransportInfo{ 270 false: { 271 Factory: httptransport.NewSystemTransport, 272 TransportName: "tcp", 273 }, 274 true: { 275 Factory: httptransport.NewHTTP3Transport, 276 TransportName: "quic", 277 }, 278} 279 280// DNSClient is a DNS client. It wraps a Resolver and it possibly 281// also wraps an HTTP client, but only when we're using DoH. 282type DNSClient struct { 283 Resolver 284 httpClient *http.Client 285} 286 287// CloseIdleConnections closes idle connections, if any. 288func (c DNSClient) CloseIdleConnections() { 289 if c.httpClient != nil { 290 c.httpClient.CloseIdleConnections() 291 } 292} 293 294// NewDNSClient creates a new DNS client. The config argument is used to 295// create the underlying Dialer and/or HTTP transport, if needed. The URL 296// argument describes the kind of client that we want to make: 297// 298// - if the URL is `doh://powerdns`, `doh://google` or `doh://cloudflare` or the URL 299// starts with `https://`, then we create a DoH client. 300// 301// - if the URL is `` or `system:///`, then we create a system client, 302// i.e. a client using the system resolver. 303// 304// - if the URL starts with `udp://`, then we create a client using 305// a resolver that uses the specified UDP endpoint. 306// 307// We return error if the URL does not parse or the URL scheme does not 308// fall into one of the cases described above. 309// 310// If config.ResolveSaver is not nil and we're creating an underlying 311// resolver where this is possible, we will also save events. 312func NewDNSClient(config Config, URL string) (DNSClient, error) { 313 return NewDNSClientWithOverrides(config, URL, "", "", "") 314} 315 316// ErrInvalidTLSVersion indicates that you passed us a string 317// that does not represent a valid TLS version. 318var ErrInvalidTLSVersion = errors.New("invalid TLS version") 319 320// ConfigureTLSVersion configures the correct TLS version into 321// the specified *tls.Config or returns an error. 322func ConfigureTLSVersion(config *tls.Config, version string) error { 323 switch version { 324 case "TLSv1.3": 325 config.MinVersion = tls.VersionTLS13 326 config.MaxVersion = tls.VersionTLS13 327 case "TLSv1.2": 328 config.MinVersion = tls.VersionTLS12 329 config.MaxVersion = tls.VersionTLS12 330 case "TLSv1.1": 331 config.MinVersion = tls.VersionTLS11 332 config.MaxVersion = tls.VersionTLS11 333 case "TLSv1.0", "TLSv1": 334 config.MinVersion = tls.VersionTLS10 335 config.MaxVersion = tls.VersionTLS10 336 case "": 337 // nothing 338 default: 339 return ErrInvalidTLSVersion 340 } 341 return nil 342} 343 344// NewDNSClientWithOverrides creates a new DNS client, similar to NewDNSClient, 345// with the option to override the default Hostname and SNI. 346func NewDNSClientWithOverrides(config Config, URL, hostOverride, SNIOverride, 347 TLSVersion string) (DNSClient, error) { 348 var c DNSClient 349 switch URL { 350 case "doh://powerdns": 351 URL = "https://doh.powerdns.org/" 352 case "doh://google": 353 URL = "https://dns.google/dns-query" 354 case "doh://cloudflare": 355 URL = "https://cloudflare-dns.com/dns-query" 356 case "": 357 URL = "system:///" 358 } 359 resolverURL, err := url.Parse(URL) 360 if err != nil { 361 return c, err 362 } 363 config.TLSConfig = &tls.Config{ServerName: SNIOverride} 364 if err := ConfigureTLSVersion(config.TLSConfig, TLSVersion); err != nil { 365 return c, err 366 } 367 switch resolverURL.Scheme { 368 case "system": 369 c.Resolver = resolver.SystemResolver{} 370 return c, nil 371 case "https": 372 config.TLSConfig.NextProtos = []string{"h2", "http/1.1"} 373 c.httpClient = &http.Client{Transport: NewHTTPTransport(config)} 374 var txp resolver.RoundTripper = resolver.NewDNSOverHTTPSWithHostOverride( 375 c.httpClient, URL, hostOverride) 376 if config.ResolveSaver != nil { 377 txp = resolver.SaverDNSTransport{ 378 RoundTripper: txp, 379 Saver: config.ResolveSaver, 380 } 381 } 382 c.Resolver = resolver.NewSerialResolver(txp) 383 return c, nil 384 case "udp": 385 dialer := NewDialer(config) 386 endpoint, err := makeValidEndpoint(resolverURL) 387 if err != nil { 388 return c, err 389 } 390 var txp resolver.RoundTripper = resolver.NewDNSOverUDP(dialer, endpoint) 391 if config.ResolveSaver != nil { 392 txp = resolver.SaverDNSTransport{ 393 RoundTripper: txp, 394 Saver: config.ResolveSaver, 395 } 396 } 397 c.Resolver = resolver.NewSerialResolver(txp) 398 return c, nil 399 case "dot": 400 config.TLSConfig.NextProtos = []string{"dot"} 401 tlsDialer := NewTLSDialer(config) 402 endpoint, err := makeValidEndpoint(resolverURL) 403 if err != nil { 404 return c, err 405 } 406 var txp resolver.RoundTripper = resolver.NewDNSOverTLS( 407 tlsDialer.DialTLSContext, endpoint) 408 if config.ResolveSaver != nil { 409 txp = resolver.SaverDNSTransport{ 410 RoundTripper: txp, 411 Saver: config.ResolveSaver, 412 } 413 } 414 c.Resolver = resolver.NewSerialResolver(txp) 415 return c, nil 416 case "tcp": 417 dialer := NewDialer(config) 418 endpoint, err := makeValidEndpoint(resolverURL) 419 if err != nil { 420 return c, err 421 } 422 var txp resolver.RoundTripper = resolver.NewDNSOverTCP( 423 dialer.DialContext, endpoint) 424 if config.ResolveSaver != nil { 425 txp = resolver.SaverDNSTransport{ 426 RoundTripper: txp, 427 Saver: config.ResolveSaver, 428 } 429 } 430 c.Resolver = resolver.NewSerialResolver(txp) 431 return c, nil 432 default: 433 return c, errors.New("unsupported resolver scheme") 434 } 435} 436 437// makeValidEndpoint makes a valid endpoint for DoT and Do53 given the 438// input URL representing such endpoint. Specifically, we are 439// concerned with the case where the port is missing. In such a 440// case, we ensure that we are using the default port 853 for DoT 441// and default port 53 for TCP and UDP. 442func makeValidEndpoint(URL *url.URL) (string, error) { 443 // Implementation note: when we're using a quoted IPv6 444 // address, URL.Host contains the quotes but instead the 445 // return value from URL.Hostname() does not. 446 // 447 // For example: 448 // 449 // - Host: [2620:fe::9] 450 // - Hostname(): 2620:fe::9 451 // 452 // We need to keep this in mind when trying to determine 453 // whether there is also a port or not. 454 // 455 // So the first step is to check whether URL.Host is already 456 // a whatever valid TCP/UDP endpoint and, if so, use it. 457 if _, _, err := net.SplitHostPort(URL.Host); err == nil { 458 return URL.Host, nil 459 } 460 // The second step is to assume that appending the default port 461 // to a host parsed by url.Parse should be giving us a valid 462 // endpoint. The possibilities in fact are: 463 // 464 // 1. domain w/o port 465 // 2. IPv4 w/o port 466 // 3. square bracket quoted IPv6 w/o port 467 // 4. other 468 // 469 // In the first three cases, appending a port leads us to a 470 // good endpoint. The fourth case does not. 471 // 472 // For this reason we check again whether we can split it using 473 // net.SplitHostPort. If we cannot, we were in case four. 474 host := URL.Host 475 if URL.Scheme == "dot" { 476 host += ":853" 477 } else { 478 host += ":53" 479 } 480 if _, _, err := net.SplitHostPort(host); err != nil { 481 return "", err 482 } 483 // Otherwise it's one of the three valid cases above. 484 return host, nil 485} 486