Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: move self monitor to init func #2057

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

chore: move self monitor to init func #2057

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 37 additions & 23 deletions core/prometheus/PrometheusInputRunner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,24 +51,6 @@ PrometheusInputRunner::PrometheusInputRunner()
mEventPool(true),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

现在模式有点混乱，先让玄飏梳理下模式吧。

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

指标操作参考该PR修改一下：
#2091

mUnRegisterMs(0) {
mTimer = std::make_shared<Timer>();

// self monitor
MetricLabels labels;
labels.emplace_back(METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS);
labels.emplace_back(METRIC_LABEL_KEY_INSTANCE_ID, Application::GetInstance()->GetInstanceId());
labels.emplace_back(METRIC_LABEL_KEY_POD_NAME, mPodName);
labels.emplace_back(METRIC_LABEL_KEY_SERVICE_HOST, mServiceHost);
labels.emplace_back(METRIC_LABEL_KEY_SERVICE_PORT, ToString(mServicePort));

DynamicMetricLabels dynamicLabels;
dynamicLabels.emplace_back(METRIC_LABEL_KEY_PROJECT, [this]() -> std::string { return this->GetAllProjects(); });

WriteMetrics::GetInstance()->PrepareMetricsRecordRef(
mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_RUNNER, std::move(labels), std::move(dynamicLabels));

mPromRegisterState = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_CLIENT_REGISTER_STATE);
mPromJobNum = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_JOBS_TOTAL);
mPromRegisterRetryTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL);
}

/// @brief receive scrape jobs from input plugins and update scrape jobs
Expand Down Expand Up @@ -102,7 +84,9 @@ void PrometheusInputRunner::UpdateScrapeInput(std::shared_ptr<TargetSubscriberSc
targetSubscriber->ScheduleNext();
{
ReadLock lock(mSubscriberMapRWLock);
mPromJobNum->Set(mTargetSubscriberSchedulerMap.size());
if (mPromJobNum != nullptr) {
mPromJobNum->Set(mTargetSubscriberSchedulerMap.size());
}
}
// 3. add project name to mJobNameToProjectNameMap for self monitor
{
Expand All @@ -117,7 +101,9 @@ void PrometheusInputRunner::RemoveScrapeInput(const std::string& jobName) {
if (mTargetSubscriberSchedulerMap.count(jobName)) {
mTargetSubscriberSchedulerMap[jobName]->Cancel();
mTargetSubscriberSchedulerMap.erase(jobName);
mPromJobNum->Set(mTargetSubscriberSchedulerMap.size());
if (mPromJobNum != nullptr) {
mPromJobNum->Set(mTargetSubscriberSchedulerMap.size());
}
}
}
{
Expand All @@ -137,6 +123,28 @@ void PrometheusInputRunner::Init() {
LOG_INFO(sLogger, ("PrometheusInputRunner", "Start"));
mIsStarted = true;

if (!mStartSelfMonitor) {
mStartSelfMonitor = true;
// self monitor
MetricLabels labels;
labels.emplace_back(METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROMETHEUS);
labels.emplace_back(METRIC_LABEL_KEY_INSTANCE_ID, Application::GetInstance()->GetInstanceId());
labels.emplace_back(METRIC_LABEL_KEY_POD_NAME, mPodName);
labels.emplace_back(METRIC_LABEL_KEY_SERVICE_HOST, mServiceHost);
labels.emplace_back(METRIC_LABEL_KEY_SERVICE_PORT, ToString(mServicePort));

DynamicMetricLabels dynamicLabels;
dynamicLabels.emplace_back(METRIC_LABEL_KEY_PROJECT,
[this]() -> std::string { return this->GetAllProjects(); });

WriteMetrics::GetInstance()->PrepareMetricsRecordRef(
mMetricsRecordRef, MetricCategory::METRIC_CATEGORY_RUNNER, std::move(labels), std::move(dynamicLabels));

mPromRegisterState = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_CLIENT_REGISTER_STATE);
mPromJobNum = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_JOBS_TOTAL);
mPromRegisterRetryTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_CLIENT_REGISTER_RETRY_TOTAL);
}

#ifndef APSARA_UNIT_TEST_MAIN
mTimer->Init();
AsynCurlRunner::GetInstance()->Init();
Expand All @@ -153,7 +161,9 @@ void PrometheusInputRunner::Init() {
++retry;
auto httpResponse = SendRegisterMessage(prometheus::REGISTER_COLLECTOR_PATH);
if (httpResponse.GetStatusCode() != 200) {
mPromRegisterRetryTotal->Add(1);
if (mPromRegisterRetryTotal != nullptr) {
mPromRegisterRetryTotal->Add(1);
}
if (retry % 10 == 0) {
LOG_INFO(sLogger,
("register failed, retried", retry)("statusCode", httpResponse.GetStatusCode()));
Expand All @@ -179,7 +189,9 @@ void PrometheusInputRunner::Init() {
}
}
}
mPromRegisterState->Set(1);
if (mPromRegisterState != nullptr) {
mPromRegisterState->Set(1);
}
LOG_INFO(sLogger, ("Register Success", mPodName));
// subscribe immediately
SubscribeOnce();
Expand Down Expand Up @@ -228,7 +240,9 @@ void PrometheusInputRunner::Stop() {
LOG_ERROR(sLogger, ("unregister failed, statusCode", httpResponse.GetStatusCode()));
} else {
LOG_INFO(sLogger, ("Unregister Success", mPodName));
mPromRegisterState->Set(0);
if (mPromRegisterState != nullptr) {
mPromRegisterState->Set(0);
}
break;
}
std::this_thread::sleep_for(std::chrono::seconds(1));
Expand Down
1 change: 1 addition & 0 deletions core/prometheus/PrometheusInputRunner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ class PrometheusInputRunner : public InputRunner {
std::atomic<uint64_t> mUnRegisterMs;

// self monitor
bool mStartSelfMonitor = false;
ReadWriteLock mProjectRWLock;
std::map<std::string, std::string> mJobNameToProjectNameMap;
MetricsRecordRef mMetricsRecordRef;
Expand Down
Loading