[feat_1192][taier-metrics][add a monitor metrics with taier #1192]

pom.xml

@@ -41,6 +41,7 @@
         <module>taier-data-develop</module>
         <module>taier-worker</module>
         <module>taier-datasource</module>
+        <module>taier-metrics</module>
     </modules>
 
     <dependencyManagement>

taier-common/src/main/java/com/dtstack/taier/common/client/ClientProxy.java

@@ -20,7 +20,7 @@
 
 import com.dtstack.taier.common.exception.LimitResourceException;
 import com.dtstack.taier.common.exception.TaierDefineException;
-import com.dtstack.taier.pluginapi.CustomThreadFactory;
+import com.dtstack.taier.metrics.collect.em.QueueTypeEnum;
 import com.dtstack.taier.pluginapi.JobClient;
 import com.dtstack.taier.pluginapi.JobIdentifier;
 import com.dtstack.taier.pluginapi.callback.CallBack;
@@ -29,9 +29,9 @@
 import com.dtstack.taier.pluginapi.enums.TaskStatus;
 import com.dtstack.taier.pluginapi.exception.ClientArgumentException;
 import com.dtstack.taier.pluginapi.exception.ExceptionUtil;
+import com.dtstack.taier.pluginapi.metrics.DynamicMetricsThreadPoolUtil;
 import com.dtstack.taier.pluginapi.pojo.CheckResult;
 import com.dtstack.taier.pluginapi.pojo.ComponentTestResult;
-import com.dtstack.taier.pluginapi.pojo.FileResult;
 import com.dtstack.taier.pluginapi.pojo.JobResult;
 import com.dtstack.taier.pluginapi.pojo.JudgeResult;
 
@@ -41,8 +41,6 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
@@ -64,8 +62,12 @@ public class ClientProxy implements IClient {
 
     public ClientProxy(IClient targetClient) {
         this.targetClient = targetClient;
-        executorService = new ThreadPoolExecutor(1, 10, 0L, TimeUnit.MILLISECONDS,
-                new SynchronousQueue<>(), new CustomThreadFactory(targetClient.getClass().getSimpleName() + "_" + this.getClass().getSimpleName()));
+
+        executorService = DynamicMetricsThreadPoolUtil.buildDynamicThreadPool(
+                1, 10, 0L, TimeUnit.MILLISECONDS,
+                "ClientProxy", QueueTypeEnum.SYNCHRONOUS_QUEUE.getName(), null, false, this.timeout,
+                targetClient.getClass().getSimpleName() + "_" + this.getClass().getSimpleName(),
+                null, false);
     }
 
     @Override

taier-common/src/main/java/com/dtstack/taier/common/env/EnvironmentContext.java

@@ -35,7 +35,11 @@
 import org.springframework.core.env.Environment;
 import org.springframework.stereotype.Component;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 
 /**
@@ -271,6 +275,12 @@ public class EnvironmentContext implements InitializingBean {
     @Value("${schedule.scanning.cycle.day:1}")
     private Integer scanningCycleJobDay;
 
+    @Value("${engine.monitor.metrics.interval:5}")
+    private Long monitorMetricsInterval;
+
+    @Value("${engine.monitor.metrics.delay:30}")
+    private Long monitorMetricsDelay;
+
     @Override
     public void afterPropertiesSet() throws Exception {
         // 读取全局配置并初始化 datasource
@@ -621,4 +631,63 @@ public Integer getTempSelectExpireTime() {
     public int getScanningCycleJobDay() {
         return scanningCycleJobDay;
     }
+
+    public long getMonitorMetricsInterval() {
+        return monitorMetricsInterval;
+    }
+
+    public long getMonitorMetricsDelay() {
+        return monitorMetricsDelay;
+    }
+
+    /**
+     * 默认支持的监控指标推送类型
+     * 平台内置的支持两种: micrometer, logging, output 默认logging
+     */
+    public List<String> getSupportMonitorMetricsType() {
+        String property = environment.getProperty("taier.monitor.metrics.support.type", "logging");
+        if (StringUtils.isEmpty(property)) {
+            return Collections.emptyList();
+        }
+        return Arrays.stream(StringUtils.split(property, ",")).map(String::valueOf)
+                .collect(Collectors.toList());
+    }
+
+    /**
+     */
+    public boolean getIsMonitorMetrics() {
+        return Boolean.parseBoolean(environment.getProperty("taier.monitor.metrics.enabled", "true"));
+    }
+
+    /**
+     * push gateway prometheus 默认url
+     */
+    public String getPrometheusPushGatewayUrl() {
+        return environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.url", "");
+    }
+
+    /**
+     * push gateway prometheus 默认search参数
+     */
+    public String getPrometheusPushGatewaySearch() {
+        return environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.search", "");
+    }
+
+    /**
+     * push gateway 超时时间, 单位s 默认5s
+     */
+    public long getPrometheusPushGatewayTimeout() {
+        return Long.parseLong(environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.timeout", "5"));
+    }
+
+    /**
+     * push gateway 默认间隔时间,单位s, 默认60s
+     */
+    public long getPrometheusPushGatewayInterval() {
+        return Long.parseLong(environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.interval", "60"));
+    }
+
+
+
+
 }

taier-metrics/pom.xml

@@ -0,0 +1,134 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.dtstack.taier</groupId>
+        <artifactId>taier</artifactId>
+        <version>1.0.0</version>
+    </parent>
+
+    <artifactId>taier-metrics</artifactId>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <!-- micrometer 监控 -->
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-registry-prometheus</artifactId>
+            <version>1.10.12</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_pushgateway</artifactId>
+            <version>0.16.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.dropwizard.metrics</groupId>
+            <artifactId>metrics-core</artifactId>
+            <version>4.2.37</version>
+        </dependency>
+
+        <!-- 不考虑引入spring boot starter actuator-->
+        <!--<dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-actuator</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.springframework.boot</groupId>
+                    <artifactId>spring-boot-starter-logging</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>-->
+
+        <!-- lombok 配置依赖-->
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.36</version>
+        </dependency>
+
+        <!-- 可选依赖,主要包含logging,guava,commons-lang3,commons-logging,commons-collections-->
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+            <version>1.2</version>
+            <optional>true</optional>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-collections</groupId>
+            <artifactId>commons-collections</artifactId>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                    <encoding>UTF-8</encoding>
+                    <annotationProcessorPaths>
+                        <path>
+                            <groupId>org.projectlombok</groupId>
+                            <artifactId>lombok</artifactId>
+                            <version>1.18.36</version>
+                        </path>
+                    </annotationProcessorPaths>
+                </configuration>
+            </plugin>
+            <!-- 要将源码放上去，需要加入这个插件 -->
+            <plugin>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>3.0.1</version>
+                <configuration>
+                    <attach>true</attach>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>compile</phase>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>3.0.1</version>
+                <configuration>
+                    <reportOutputDirectory>../javadocs</reportOutputDirectory>
+                    <destDir>taier-api-client</destDir>
+                    <additionalJOptions>
+                        <additionalJOption>-Xdoclint:none</additionalJOption>
+                    </additionalJOptions>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>