1# frozen_string_literal: true 2 3# Keep separate directories for separate processes 4def prometheus_default_multiproc_dir 5 return unless Rails.env.development? || Rails.env.test? 6 7 if Gitlab::Runtime.sidekiq? 8 Rails.root.join('tmp/prometheus_multiproc_dir/sidekiq') 9 elsif Gitlab::Runtime.puma? 10 Rails.root.join('tmp/prometheus_multiproc_dir/puma') 11 else 12 Rails.root.join('tmp/prometheus_multiproc_dir') 13 end 14end 15 16::Prometheus::Client.configure do |config| 17 config.logger = Gitlab::AppLogger 18 19 config.initial_mmap_file_size = 4 * 1024 20 21 config.multiprocess_files_dir = ENV['prometheus_multiproc_dir'] || prometheus_default_multiproc_dir 22 23 config.pid_provider = ::Prometheus::PidProvider.method(:worker_id) 24end 25 26Gitlab::Application.configure do |config| 27 # 0 should be Sentry to catch errors in this middleware 28 config.middleware.insert_after(Labkit::Middleware::Rack, Gitlab::Metrics::RequestsRackMiddleware) 29end 30 31if Gitlab::Runtime.sidekiq? && (!ENV['SIDEKIQ_WORKER_ID'] || ENV['SIDEKIQ_WORKER_ID'] == '0') 32 # The single worker outside of a sidekiq-cluster, or the first worker (sidekiq_0) 33 # in a cluster of processes, is responsible for serving health checks. 34 # 35 # Do not clean the metrics directory here - the supervisor script should 36 # have already taken care of that. 37 Sidekiq.configure_server do |config| 38 config.on(:startup) do 39 # In https://gitlab.com/gitlab-org/gitlab/-/issues/345804 we are looking to 40 # only serve health-checks from a worker process; for backwards compatibility 41 # we still go through the metrics exporter server, but start to configure it 42 # with the new settings keys. 43 exporter_settings = Settings.monitoring.sidekiq_health_checks 44 Gitlab::Metrics::Exporter::SidekiqExporter.instance(exporter_settings).start 45 end 46 end 47end 48 49if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled? 50 # When running Puma in a Single mode, `on_master_start` and `on_worker_start` are the same. 51 # Thus, we order these events to run `reinitialize_on_pid_change` with `force: true` first. 52 Gitlab::Cluster::LifecycleEvents.on_master_start do 53 ::Prometheus::Client.reinitialize_on_pid_change(force: true) 54 55 if Gitlab::Runtime.puma? 56 Gitlab::Metrics::Samplers::PumaSampler.instance.start 57 end 58 59 Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION, revision: Gitlab.revision }, 1) 60 61 if Gitlab::Runtime.web_server? 62 Gitlab::Metrics::RequestsRackMiddleware.initialize_metrics 63 end 64 65 Gitlab::Ci::Parsers.instrument! 66 rescue IOError => e 67 Gitlab::ErrorTracking.track_exception(e) 68 Gitlab::Metrics.error_detected! 69 end 70 71 Gitlab::Cluster::LifecycleEvents.on_worker_start do 72 defined?(::Prometheus::Client.reinitialize_on_pid_change) && ::Prometheus::Client.reinitialize_on_pid_change 73 74 Gitlab::Metrics::Samplers::RubySampler.initialize_instance.start 75 Gitlab::Metrics::Samplers::DatabaseSampler.initialize_instance.start 76 Gitlab::Metrics::Samplers::ThreadsSampler.initialize_instance.start 77 78 if Gitlab::Runtime.web_server? 79 Gitlab::Metrics::Samplers::ActionCableSampler.instance.start 80 end 81 82 if Gitlab.ee? && Gitlab::Runtime.sidekiq? 83 Gitlab::Metrics::Samplers::GlobalSearchSampler.instance.start 84 end 85 86 Gitlab::Ci::Parsers.instrument! 87 rescue IOError => e 88 Gitlab::ErrorTracking.track_exception(e) 89 Gitlab::Metrics.error_detected! 90 end 91end 92 93if Gitlab::Runtime.web_server? 94 Gitlab::Cluster::LifecycleEvents.on_master_start do 95 Gitlab::Metrics::Exporter::WebExporter.instance.start 96 end 97 98 # DEPRECATED: TO BE REMOVED 99 # This is needed to implement blackout period of `web_exporter` 100 # https://gitlab.com/gitlab-org/gitlab/issues/35343#note_238479057 101 Gitlab::Cluster::LifecycleEvents.on_before_blackout_period do 102 Gitlab::Metrics::Exporter::WebExporter.instance.mark_as_not_running! 103 end 104 105 Gitlab::Cluster::LifecycleEvents.on_before_graceful_shutdown do 106 # We need to ensure that before we re-exec or shutdown server 107 # we do stop the exporter 108 Gitlab::Metrics::Exporter::WebExporter.instance.stop 109 end 110 111 Gitlab::Cluster::LifecycleEvents.on_before_master_restart do 112 # We need to ensure that before we re-exec server 113 # we do stop the exporter 114 # 115 # We do it again, for being extra safe, 116 # but it should not be needed 117 Gitlab::Metrics::Exporter::WebExporter.instance.stop 118 end 119 120 Gitlab::Cluster::LifecycleEvents.on_worker_start do 121 # The `#close_on_exec=` takes effect only on `execve` 122 # but this does not happen for Ruby fork 123 # 124 # This does stop server, as it is running on master. 125 Gitlab::Metrics::Exporter::WebExporter.instance.stop 126 end 127end 128