Skip to content

Commit

Permalink
[#10605] Added cleanup inactive applications batch job
Browse files Browse the repository at this point in the history
  • Loading branch information
youngjin.kim2 authored and smilu97 committed Jan 26, 2024
1 parent 91074a4 commit 31e3616
Show file tree
Hide file tree
Showing 15 changed files with 564 additions and 7 deletions.
15 changes: 15 additions & 0 deletions batch/src/main/java/com/navercorp/pinpoint/batch/BatchModule.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@
package com.navercorp.pinpoint.batch;

import com.navercorp.pinpoint.batch.alarm.AlarmSenderConfiguration;
import com.navercorp.pinpoint.batch.common.BatchJobLauncher;
import com.navercorp.pinpoint.batch.common.StartupJobLauncher;
import com.navercorp.pinpoint.batch.configuration.AlarmJobModule;
import com.navercorp.pinpoint.batch.configuration.CleanupInactiveApplicationsJobConfig;
import com.navercorp.pinpoint.common.server.config.CommonCacheManagerConfiguration;
import com.navercorp.pinpoint.common.server.config.RestTemplateConfiguration;
import com.navercorp.pinpoint.common.server.util.DefaultTimeSlot;
Expand All @@ -31,11 +34,14 @@
import com.navercorp.pinpoint.web.component.config.ComponentConfiguration;
import com.navercorp.pinpoint.web.hyperlink.HyperLinkConfiguration;
import com.navercorp.pinpoint.web.webhook.WebhookModule;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.transaction.TransactionAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.ImportResource;

import java.util.List;

@ImportResource({
"classpath:applicationContext-batch-schedule.xml",

Expand Down Expand Up @@ -67,11 +73,20 @@
UriStatAlarmConfiguration.class,
AlarmSenderConfiguration.class,
CommonCacheManagerConfiguration.class,
CleanupInactiveApplicationsJobConfig.class,
})
public class BatchModule {

@Bean
public TimeSlot timeSlot() {
return new DefaultTimeSlot();
}

@Bean
StartupJobLauncher startupJobLauncher(
BatchJobLauncher launcher,
@Value("${batch.startup.jobs:#{''}}") List<String> jobs
) {
return new StartupJobLauncher(launcher, jobs);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

package com.navercorp.pinpoint.batch.common;

import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.configuration.JobLocator;
Expand All @@ -36,8 +36,11 @@ public class BatchJobLauncher extends JobLaunchSupport {

private final BatchProperties batchProperties;

public BatchJobLauncher(@Qualifier("jobRegistry") JobLocator locator,
@Qualifier("jobLauncher") JobLauncher launcher, BatchProperties batchProperties) {
public BatchJobLauncher(
@Qualifier("jobRegistry") JobLocator locator,
@Qualifier("jobLauncher") JobLauncher launcher,
BatchProperties batchProperties
) {
super(locator, launcher);
this.batchProperties = Objects.requireNonNull(batchProperties, "batchProperties");
}
Expand All @@ -59,7 +62,7 @@ public void uriStatAlarmJob() {
}
}

private JobParameters createTimeParameter() {
public static JobParameters createTimeParameter() {
JobParametersBuilder builder = new JobParametersBuilder();
Date now = new Date();
builder.addDate("schedule.date", now);
Expand Down Expand Up @@ -89,4 +92,13 @@ public void cleanupInactiveAgentsJob() {
logger.debug("Skip cleanupInactiveAgentsJob, because 'enableCleanupInactiveAgentsJob' is disabled.");
}
}

public void cleanupInactiveApplicationsJob() {
if (batchProperties.isCleanupInactiveApplicationsJobEnable()) {
run("cleanupInactiveApplicationsJob", createTimeParameter());
} else {
logger.debug("Skip applicationCleanJob, because 'enableCleanupInactiveApplicationsJob' is disabled.");
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ public class BatchProperties {
@Value("${job.cleanup.inactive.agents.duration.days:30}")
private int cleanupInactiveAgentsDurationDays;

@Value("${job.cleanup.inactive.applications.enable:false}")
private boolean cleanupInactiveApplicationsJobEnable;

@Value("${job.cleanup.inactive.applications.cron}")
private String cleanupInactiveApplicationsJobCron;

private static final int MINIMUM_CLEANUP_INACTIVE_AGENTS_DURATION_DAYS = 7;

@PostConstruct
Expand Down Expand Up @@ -156,6 +162,14 @@ public int getCleanupInactiveAgentsDurationDays() {
return cleanupInactiveAgentsDurationDays;
}

public boolean isCleanupInactiveApplicationsJobEnable() {
return cleanupInactiveApplicationsJobEnable;
}

public String getCleanupInactiveApplicationsJobCron() {
return cleanupInactiveApplicationsJobCron;
}

@Override
public String toString() {
return "BatchProperties{" +
Expand All @@ -174,6 +188,8 @@ public String toString() {
", uriStatAlarmJobEnable=" + uriStatAlarmJobEnable +
", uriStatAlarmJobCron='" + uriStatAlarmJobCron + '\'' +
", cleanupInactiveAgentsDurationDays=" + cleanupInactiveAgentsDurationDays +
", cleanupInactiveApplicationsJobEnable=" + cleanupInactiveApplicationsJobEnable +
", cleanupInactiveApplicationsJobCron='" + cleanupInactiveApplicationsJobCron + '\'' +
'}';
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
* Copyright 2024 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.navercorp.pinpoint.batch.common;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;

import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
* @author youngjin.kim2
*/
public class StartupJobLauncher implements InitializingBean, DisposableBean {

private static final Logger logger = LogManager.getLogger(StartupJobLauncher.class);

private final BatchJobLauncher launcher;
private final List<String> jobs;

private final ExecutorService executor = Executors.newSingleThreadExecutor();

public StartupJobLauncher(BatchJobLauncher launcher, List<String> jobs) {
this.launcher = Objects.requireNonNull(launcher, "launcher");
this.jobs = Objects.requireNonNull(jobs, "jobs");
}

@Override
public void afterPropertiesSet() {
if (this.jobs.isEmpty()) {
logger.info("No startup jobs to launch");
return;
}

logger.info("Startup job launcher started");
this.executor.execute(() -> {
for (String job : jobs) {
logger.info("Launching job {}", job);
launcher.run(job, BatchJobLauncher.createTimeParameter());
}
logger.info("Startup job launcher finished");
});
}

@Override
public void destroy() {
this.executor.shutdownNow();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
* Copyright 2024 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.navercorp.pinpoint.batch.configuration;

import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.ImportResource;

/**
* @author youngjin.kim2
*/
@ImportResource({ "classpath:job/applicationContext-cleanupInactiveApplicationsJob.xml" })
@Configuration(proxyBeanMethods = false)
public class CleanupInactiveApplicationsJobConfig {
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,16 @@

import com.navercorp.pinpoint.web.dao.ApplicationIndexDao;
import com.navercorp.pinpoint.web.vo.Application;
import jakarta.annotation.Nonnull;
import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.StepExecutionListener;
import org.springframework.batch.item.ItemReader;

import jakarta.annotation.Nonnull;
import java.util.List;
import java.util.Objects;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.stream.Collectors;

/**
* @author youngjin.kim2
Expand All @@ -48,7 +47,7 @@ public void beforeStep(@Nonnull StepExecution stepExecution) {
List<String> applicationNames = applicationIndexDao.selectAllApplicationNames()
.stream()
.map(Application::getName)
.collect(Collectors.toUnmodifiableList());
.toList();
this.applicationNameQueue = new ConcurrentLinkedQueue<>(applicationNames);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* Copyright 2024 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.navercorp.pinpoint.batch.job;

import com.navercorp.pinpoint.batch.service.ApplicationService;
import jakarta.annotation.Nonnull;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.batch.item.ItemProcessor;

import java.time.Duration;
import java.util.Objects;

/**
* @author youngjin.kim2
*/
public class ApplicationEmptyFilter implements ItemProcessor<String, String> {

private static final Logger logger = LogManager.getLogger(ApplicationEmptyFilter.class);

private final ApplicationService applicationService;
private final Duration emptyDurationThreshold;

public ApplicationEmptyFilter(ApplicationService applicationService, Duration emptyDurationThreshold) {
this.applicationService = Objects.requireNonNull(applicationService, "applicationService");
this.emptyDurationThreshold = Objects.requireNonNull(emptyDurationThreshold, "emptyDurationThreshold");
}

@Override
public String process(@Nonnull String s) throws Exception {
if (isApplicationEmpty(s)) {
logger.info("Application is empty: {}", s);
return s;
} else {
logger.info("Application is not empty: {}", s);
return null;
}
}

private boolean isApplicationEmpty(String applicationName) {
return this.applicationService.isApplicationEmpty(applicationName, this.emptyDurationThreshold);
}
}
Loading

0 comments on commit 31e3616

Please sign in to comment.