1package engine 2 3import ( 4 "context" 5 "errors" 6 "fmt" 7 "io/ioutil" 8 "net/http" 9 "net/url" 10 "os" 11 "path/filepath" 12 "sync" 13 14 "github.com/ooni/probe-engine/atomicx" 15 "github.com/ooni/probe-engine/geolocate" 16 "github.com/ooni/probe-engine/internal/kvstore" 17 "github.com/ooni/probe-engine/internal/platform" 18 "github.com/ooni/probe-engine/internal/sessionresolver" 19 "github.com/ooni/probe-engine/internal/tunnel" 20 "github.com/ooni/probe-engine/model" 21 "github.com/ooni/probe-engine/netx" 22 "github.com/ooni/probe-engine/netx/bytecounter" 23 "github.com/ooni/probe-engine/probeservices" 24 "github.com/ooni/probe-engine/resources" 25 "github.com/ooni/probe-engine/version" 26) 27 28// SessionConfig contains the Session config 29type SessionConfig struct { 30 AssetsDir string 31 AvailableProbeServices []model.Service 32 KVStore KVStore 33 Logger model.Logger 34 ProxyURL *url.URL 35 SoftwareName string 36 SoftwareVersion string 37 TempDir string 38 TorArgs []string 39 TorBinary string 40} 41 42// Session is a measurement session 43type Session struct { 44 assetsDir string 45 availableProbeServices []model.Service 46 availableTestHelpers map[string][]model.Service 47 byteCounter *bytecounter.Counter 48 httpDefaultTransport netx.HTTPRoundTripper 49 kvStore model.KeyValueStore 50 location *geolocate.Results 51 logger model.Logger 52 proxyURL *url.URL 53 queryProbeServicesCount *atomicx.Int64 54 resolver *sessionresolver.Resolver 55 selectedProbeServiceHook func(*model.Service) 56 selectedProbeService *model.Service 57 softwareName string 58 softwareVersion string 59 tempDir string 60 torArgs []string 61 torBinary string 62 tunnelMu sync.Mutex 63 tunnelName string 64 tunnel tunnel.Tunnel 65} 66 67// NewSession creates a new session or returns an error 68func NewSession(config SessionConfig) (*Session, error) { 69 if config.AssetsDir == "" { 70 return nil, errors.New("AssetsDir is empty") 71 } 72 if config.Logger == nil { 73 return nil, errors.New("Logger is empty") 74 } 75 if config.SoftwareName == "" { 76 return nil, errors.New("SoftwareName is empty") 77 } 78 if config.SoftwareVersion == "" { 79 return nil, errors.New("SoftwareVersion is empty") 80 } 81 if config.KVStore == nil { 82 config.KVStore = kvstore.NewMemoryKeyValueStore() 83 } 84 // Implementation note: if config.TempDir is empty, then Go will 85 // use the temporary directory on the current system. This should 86 // work on Desktop. We tested that it did also work on iOS, but 87 // we have also seen on 2020-06-10 that it does not work on Android. 88 tempDir, err := ioutil.TempDir(config.TempDir, "ooniengine") 89 if err != nil { 90 return nil, err 91 } 92 sess := &Session{ 93 assetsDir: config.AssetsDir, 94 availableProbeServices: config.AvailableProbeServices, 95 byteCounter: bytecounter.New(), 96 kvStore: config.KVStore, 97 logger: config.Logger, 98 proxyURL: config.ProxyURL, 99 queryProbeServicesCount: atomicx.NewInt64(), 100 softwareName: config.SoftwareName, 101 softwareVersion: config.SoftwareVersion, 102 tempDir: tempDir, 103 torArgs: config.TorArgs, 104 torBinary: config.TorBinary, 105 } 106 httpConfig := netx.Config{ 107 ByteCounter: sess.byteCounter, 108 BogonIsError: true, 109 Logger: sess.logger, 110 } 111 sess.resolver = sessionresolver.New(httpConfig) 112 httpConfig.FullResolver = sess.resolver 113 httpConfig.ProxyURL = config.ProxyURL // no need to proxy the resolver 114 sess.httpDefaultTransport = netx.NewHTTPTransport(httpConfig) 115 return sess, nil 116} 117 118// ASNDatabasePath returns the path where the ASN database path should 119// be if you have called s.FetchResourcesIdempotent. 120func (s *Session) ASNDatabasePath() string { 121 return filepath.Join(s.assetsDir, resources.ASNDatabaseName) 122} 123 124// KibiBytesReceived accounts for the KibiBytes received by the HTTP clients 125// managed by this session so far, including experiments. 126func (s *Session) KibiBytesReceived() float64 { 127 return s.byteCounter.KibiBytesReceived() 128} 129 130// KibiBytesSent is like KibiBytesReceived but for the bytes sent. 131func (s *Session) KibiBytesSent() float64 { 132 return s.byteCounter.KibiBytesSent() 133} 134 135// Close ensures that we close all the idle connections that the HTTP clients 136// we are currently using may have created. It will also remove the temp dir 137// that contains data from this session. Not calling this function may likely 138// cause memory leaks in your application because of open idle connections, 139// as well as excessive usage of disk space. 140func (s *Session) Close() error { 141 s.httpDefaultTransport.CloseIdleConnections() 142 s.resolver.CloseIdleConnections() 143 s.logger.Infof("%s", s.resolver.Stats()) 144 if s.tunnel != nil { 145 s.tunnel.Stop() 146 } 147 return os.RemoveAll(s.tempDir) 148} 149 150// CountryDatabasePath is like ASNDatabasePath but for the country DB path. 151func (s *Session) CountryDatabasePath() string { 152 return filepath.Join(s.assetsDir, resources.CountryDatabaseName) 153} 154 155// GetTestHelpersByName returns the available test helpers that 156// use the specified name, or false if there's none. 157func (s *Session) GetTestHelpersByName(name string) ([]model.Service, bool) { 158 services, ok := s.availableTestHelpers[name] 159 return services, ok 160} 161 162// DefaultHTTPClient returns the session's default HTTP client. 163func (s *Session) DefaultHTTPClient() *http.Client { 164 return &http.Client{Transport: s.httpDefaultTransport} 165} 166 167// KeyValueStore returns the configured key-value store. 168func (s *Session) KeyValueStore() model.KeyValueStore { 169 return s.kvStore 170} 171 172// Logger returns the logger used by the session. 173func (s *Session) Logger() model.Logger { 174 return s.logger 175} 176 177// MaybeLookupLocation is a caching location lookup call. 178func (s *Session) MaybeLookupLocation() error { 179 return s.MaybeLookupLocationContext(context.Background()) 180} 181 182// MaybeLookupBackends is a caching OONI backends lookup call. 183func (s *Session) MaybeLookupBackends() error { 184 return s.maybeLookupBackends(context.Background()) 185} 186 187// MaybeLookupBackendsContext is like MaybeLookupBackends but with context. 188func (s *Session) MaybeLookupBackendsContext(ctx context.Context) (err error) { 189 return s.maybeLookupBackends(ctx) 190} 191 192// ErrAlreadyUsingProxy indicates that we cannot create a tunnel with 193// a specific name because we already configured a proxy. 194var ErrAlreadyUsingProxy = errors.New( 195 "session: cannot create a new tunnel of this kind: we are already using a proxy", 196) 197 198// MaybeStartTunnel starts the requested tunnel. 199// 200// This function silently succeeds if we're already using a tunnel with 201// the same name or if the requested tunnel name is the empty string. This 202// function fails, tho, when we already have a proxy or a tunnel with 203// another name and we try to open a tunnel. This function of course also 204// fails if we cannot start the requested tunnel. All in all, if you request 205// for a tunnel name that is not the empty string and you get a nil error, 206// you can be confident that session.ProxyURL() gives you the tunnel URL. 207// 208// The tunnel will be closed by session.Close(). 209func (s *Session) MaybeStartTunnel(ctx context.Context, name string) error { 210 s.tunnelMu.Lock() 211 defer s.tunnelMu.Unlock() 212 if s.tunnel != nil && s.tunnelName == name { 213 // We've been asked more than once to start the same tunnel. 214 return nil 215 } 216 if s.proxyURL != nil && name == "" { 217 // The user configured a proxy and here we're not actually trying 218 // to start any tunnel since `name` is empty. 219 return nil 220 } 221 if s.proxyURL != nil || s.tunnel != nil { 222 // We already have a proxy or we have a different tunnel. Because a tunnel 223 // sets a proxy, the second check for s.tunnel is for robustness. 224 return ErrAlreadyUsingProxy 225 } 226 tunnel, err := tunnel.Start(ctx, tunnel.Config{ 227 Name: name, 228 Session: s, 229 }) 230 if err != nil { 231 s.logger.Warnf("cannot start tunnel: %+v", err) 232 return err 233 } 234 // Implementation note: tunnel _may_ be NIL here if name is "" 235 if tunnel == nil { 236 return nil 237 } 238 s.tunnelName = name 239 s.tunnel = tunnel 240 s.proxyURL = tunnel.SOCKS5ProxyURL() 241 return nil 242} 243 244// NewExperimentBuilder returns a new experiment builder 245// for the experiment with the given name, or an error if 246// there's no such experiment with the given name 247func (s *Session) NewExperimentBuilder(name string) (*ExperimentBuilder, error) { 248 return newExperimentBuilder(s, name) 249} 250 251// NewProbeServicesClient creates a new client for talking with the 252// OONI probe services. This function will benchmark the available 253// probe services, and select the fastest. In case all probe services 254// seem to be down, we try again applying circumvention tactics. 255func (s *Session) NewProbeServicesClient(ctx context.Context) (*probeservices.Client, error) { 256 if err := s.maybeLookupBackends(ctx); err != nil { 257 return nil, err 258 } 259 if err := s.MaybeLookupLocationContext(ctx); err != nil { 260 return nil, err 261 } 262 if s.selectedProbeServiceHook != nil { 263 s.selectedProbeServiceHook(s.selectedProbeService) 264 } 265 return probeservices.NewClient(s, *s.selectedProbeService) 266} 267 268// NewSubmitter creates a new submitter instance. 269func (s *Session) NewSubmitter(ctx context.Context) (Submitter, error) { 270 psc, err := s.NewProbeServicesClient(ctx) 271 if err != nil { 272 return nil, err 273 } 274 return probeservices.NewSubmitter(psc, s.Logger()), nil 275} 276 277// NewOrchestraClient creates a new orchestra client. This client is registered 278// and logged in with the OONI orchestra. An error is returned on failure. 279func (s *Session) NewOrchestraClient(ctx context.Context) (model.ExperimentOrchestraClient, error) { 280 clnt, err := s.NewProbeServicesClient(ctx) 281 if err != nil { 282 return nil, err 283 } 284 return s.initOrchestraClient(ctx, clnt, clnt.MaybeLogin) 285} 286 287// Platform returns the current platform. The platform is one of: 288// 289// - android 290// - ios 291// - linux 292// - macos 293// - windows 294// - unknown 295// 296// When running on the iOS simulator, the returned platform is 297// macos rather than ios if CGO is disabled. This is a known issue, 298// that however should have a very limited impact. 299func (s *Session) Platform() string { 300 return platform.Name() 301} 302 303// ProbeASNString returns the probe ASN as a string. 304func (s *Session) ProbeASNString() string { 305 return fmt.Sprintf("AS%d", s.ProbeASN()) 306} 307 308// ProbeASN returns the probe ASN as an integer. 309func (s *Session) ProbeASN() uint { 310 asn := geolocate.DefaultProbeASN 311 if s.location != nil { 312 asn = s.location.ASN 313 } 314 return asn 315} 316 317// ProbeCC returns the probe CC. 318func (s *Session) ProbeCC() string { 319 cc := geolocate.DefaultProbeCC 320 if s.location != nil { 321 cc = s.location.CountryCode 322 } 323 return cc 324} 325 326// ProbeNetworkName returns the probe network name. 327func (s *Session) ProbeNetworkName() string { 328 nn := geolocate.DefaultProbeNetworkName 329 if s.location != nil { 330 nn = s.location.NetworkName 331 } 332 return nn 333} 334 335// ProbeIP returns the probe IP. 336func (s *Session) ProbeIP() string { 337 ip := geolocate.DefaultProbeIP 338 if s.location != nil { 339 ip = s.location.ProbeIP 340 } 341 return ip 342} 343 344// ProxyURL returns the Proxy URL, or nil if not set 345func (s *Session) ProxyURL() *url.URL { 346 return s.proxyURL 347} 348 349// ResolverASNString returns the resolver ASN as a string 350func (s *Session) ResolverASNString() string { 351 return fmt.Sprintf("AS%d", s.ResolverASN()) 352} 353 354// ResolverASN returns the resolver ASN 355func (s *Session) ResolverASN() uint { 356 asn := geolocate.DefaultResolverASN 357 if s.location != nil { 358 asn = s.location.ResolverASN 359 } 360 return asn 361} 362 363// ResolverIP returns the resolver IP 364func (s *Session) ResolverIP() string { 365 ip := geolocate.DefaultResolverIP 366 if s.location != nil { 367 ip = s.location.ResolverIP 368 } 369 return ip 370} 371 372// ResolverNetworkName returns the resolver network name. 373func (s *Session) ResolverNetworkName() string { 374 nn := geolocate.DefaultResolverNetworkName 375 if s.location != nil { 376 nn = s.location.ResolverNetworkName 377 } 378 return nn 379} 380 381// SoftwareName returns the application name. 382func (s *Session) SoftwareName() string { 383 return s.softwareName 384} 385 386// SoftwareVersion returns the application version. 387func (s *Session) SoftwareVersion() string { 388 return s.softwareVersion 389} 390 391// TempDir returns the temporary directory. 392func (s *Session) TempDir() string { 393 return s.tempDir 394} 395 396// TorArgs returns the configured extra args for the tor binary. If not set 397// we will not pass in any extra arg. Applies to `-OTunnel=tor` mainly. 398func (s *Session) TorArgs() []string { 399 return s.torArgs 400} 401 402// TorBinary returns the configured path to the tor binary. If not set 403// we will attempt to use "tor". Applies to `-OTunnel=tor` mainly. 404func (s *Session) TorBinary() string { 405 return s.torBinary 406} 407 408// UserAgent constructs the user agent to be used in this session. 409func (s *Session) UserAgent() (useragent string) { 410 useragent += s.softwareName + "/" + s.softwareVersion 411 useragent += " ooniprobe-engine/" + version.Version 412 return 413} 414 415// MaybeUpdateResources updates the resources if needed. 416func (s *Session) MaybeUpdateResources(ctx context.Context) error { 417 return (&resources.Client{ 418 HTTPClient: s.DefaultHTTPClient(), 419 Logger: s.logger, 420 UserAgent: s.UserAgent(), 421 WorkDir: s.assetsDir, 422 }).Ensure(ctx) 423} 424 425func (s *Session) getAvailableProbeServices() []model.Service { 426 if len(s.availableProbeServices) > 0 { 427 return s.availableProbeServices 428 } 429 return probeservices.Default() 430} 431 432func (s *Session) initOrchestraClient( 433 ctx context.Context, clnt *probeservices.Client, 434 maybeLogin func(ctx context.Context) error, 435) (*probeservices.Client, error) { 436 // The original implementation has as its only use case that we 437 // were registering and logging in for sending an update regarding 438 // the probe whereabouts. Yet here in probe-engine, the orchestra 439 // is currently only used to fetch inputs. For this purpose, we don't 440 // need to communicate any specific information. The code that will 441 // perform an update used to be responsible of doing that. Now, we 442 // are not using orchestra for this purpose anymore. 443 meta := probeservices.Metadata{ 444 Platform: "miniooni", 445 ProbeASN: "AS0", 446 ProbeCC: "ZZ", 447 SoftwareName: "miniooni", 448 SoftwareVersion: "0.1.0-dev", 449 SupportedTests: []string{"web_connectivity"}, 450 } 451 if err := clnt.MaybeRegister(ctx, meta); err != nil { 452 return nil, err 453 } 454 if err := maybeLogin(ctx); err != nil { 455 return nil, err 456 } 457 return clnt, nil 458} 459 460// LookupASN maps an IP address to its ASN and network name. This method implements 461// LocationLookupASNLookupper.LookupASN. 462func (s *Session) LookupASN(dbPath, ip string) (uint, string, error) { 463 return geolocate.LookupASN(dbPath, ip) 464} 465 466// ErrAllProbeServicesFailed indicates all probe services failed. 467var ErrAllProbeServicesFailed = errors.New("all available probe services failed") 468 469func (s *Session) maybeLookupBackends(ctx context.Context) error { 470 // TODO(bassosimone): do we need a mutex here? 471 if s.selectedProbeService != nil { 472 return nil 473 } 474 s.queryProbeServicesCount.Add(1) 475 candidates := probeservices.TryAll(ctx, s, s.getAvailableProbeServices()) 476 selected := probeservices.SelectBest(candidates) 477 if selected == nil { 478 return ErrAllProbeServicesFailed 479 } 480 s.logger.Infof("session: using probe services: %+v", selected.Endpoint) 481 s.selectedProbeService = &selected.Endpoint 482 s.availableTestHelpers = selected.TestHelpers 483 return nil 484} 485 486// LookupLocationContext performs a location lookup. If you want memoisation 487// of the results, you should use MaybeLookupLocationContext. 488func (s *Session) LookupLocationContext(ctx context.Context) (*geolocate.Results, error) { 489 // Implementation note: we don't perform the lookup of the resolver IP 490 // when we are using a proxy because that might leak information. 491 task := geolocate.Must(geolocate.NewTask(geolocate.Config{ 492 EnableResolverLookup: s.proxyURL == nil, 493 HTTPClient: s.DefaultHTTPClient(), 494 Logger: s.Logger(), 495 ResourcesManager: s, 496 UserAgent: s.UserAgent(), 497 })) 498 return task.Run(ctx) 499} 500 501// MaybeLookupLocationContext is like MaybeLookupLocation but with a context 502// that can be used to interrupt this long running operation. 503func (s *Session) MaybeLookupLocationContext(ctx context.Context) error { 504 if s.location == nil { 505 location, err := s.LookupLocationContext(ctx) 506 if err != nil { 507 return err 508 } 509 s.location = location 510 } 511 return nil 512} 513 514var _ model.ExperimentSession = &Session{} 515