alibaba · zhangyongxianggithub · Sep 4, 2024
diff --git a/restwriter/doc/restwriter.md b/restwriter/doc/restwriter.md
@@ -0,0 +1,242 @@
+# DataX REstWriter 说明
+
+------------
+
+## 1 快速介绍
+
+RestWriter提供了Restful接口写入数据的能力。RestWriter服务的用户主要在于DataX开发、测试同学。
+
+## 2 功能与限制
+
+RestWriter实现了向Restful接口同步数据的通用能力，RestWriter如下几个方面约定:
+
+1. 通过请求题写入数据，支持post、put、patch3种http方法
+
+2. 通过返回的http状态码判断写入成功或者失败
+
+3. 支持失败重试与限流
+
+我们不能做到：
+
+1. 不支持嵌套对象。
+
+## 3 功能说明
+
+### 3.1 配置样例
+
+```json
+{
+  "setting": {},
+  "job": {
+    "setting": {
+      "speed": {
+        "channel": 2
+      }
+    },
+    "content": [
+      {
+        "reader": {
+          "name": "txtfilereader",
+          "parameter": {
+            "path": [
+              "/home/haiwei.luo/case00/data"
+            ],
+            "encoding": "UTF-8",
+            "column": [
+              {
+                "index": 0,
+                "type": "long"
+              },
+              {
+                "index": 1,
+                "type": "boolean"
+              },
+              {
+                "index": 2,
+                "type": "double"
+              },
+              {
+                "index": 3,
+                "type": "string"
+              },
+              {
+                "index": 4,
+                "type": "date",
+                "format": "yyyy.MM.dd"
+              }
+            ],
+            "fieldDelimiter": ","
+          }
+        },
+        "writer": {
+          "name": "restwriter",
+          "parameter": {
+            "url": "http://localhost:8080/echo",
+            "method": "post",
+            "ssl": false,
+            "headers": {
+              "aaa": "bbbb"
+            },
+            "query": {
+              "test": "test"
+            },
+            "maxRetries": 3,
+            "batch": false,
+            "batchSize": 1000,
+            "fields": [
+              {
+                "name": "id"
+              },
+              {
+                "name": "jobGroup"
+              },
+              {
+                "name": "jobId"
+              },
+              {
+                "name": "executorAddress"
+              },
+              {
+                "name": "executorHandler"
+              },
+              {
+                "name": "executorParam"
+              },
+              {
+                "name": "executorShardingParam"
+              },
+              {
+                "name": "executorFailRetryCount"
+              },
+              {
+                "name": "triggerTime",
+                "type": "java.time.LocalDateTime"
+              },
+              {
+                "name": "triggerCode"
+              },
+              {
+                "name": "triggerMsg"
+              },
+              {
+                "name": "handleTime",
+                "type": "java.time.LocalDateTime"
+              },
+              {
+                "name": "handleCode"
+              },
+              {
+                "name": "handleMsg"
+              },
+              {
+                "name": "alarmStatus"
+              }
+            ],
+            "print": true,
+            "failFast": false,
+            "rate-per-task": 10
+          }
+        }
+      }
+    ]
+  }
+}
+```
+
+### 3.2 参数说明
+
+* **url**
+
+    * 描述：restful API URL，经过转义后的URL，RestWriter不负责转义特殊字符，比如空格等。 <br />
+
+    * 必选：是 <br />
+
+    * 默认值：无 <br />
+
+* **method**
+
+    * 描述：http method <br />
+
+    * 必选：是 <br />
+
+    * 默认值：无 <br />
+
+* **ssl**
+
+    * 描述：restful api是https/http，如果在url中给出protocol，则以url中为准<br />
+
+    * 必选：否 <br />
+
+    * 默认值：false <br />
+
+* **headers**
+
+    * 描述：http请求头 <br />
+
+    * 必选：否 <br />
+
+    * 默认值：无 <br />
+
+* **query**
+
+    * 描述：查询参数。 <br />
+
+    * 必选：否 <br />
+
+    * 默认值：无 <br />
+
+* **maxRetries**
+
+    * 描述：最大失败重试次数。<br />
+
+    * 必选：否 <br />
+
+    * 默认值：3 <br />
+
+
+* **batch**
+
+    * 描述：是否批量处理<br />
+
+    * 必选：否 <br />
+
+    * 默认值：false <br />
+
+* **batchSize**
+
+    * 描述：批量处理最大条数<br />
+
+    * 必选：否 <br />
+
+    * 默认值：100 <br />
+
+* **fields**
+
+    * 描述字段信息<br />
+
+    * 必选：是 <br />
+
+    * 默认值：无 <br />
+
+* **print**
+
+    * 描述：是否打印debug信息。<br />
+
+    * 必选：否 <br />
+
+    * 默认值：false <br />
+
+### 3.3 类型转换
+
+
+## 4 性能报告
+
+## 5 约束限制
+
+略
+
+## 6 FAQ
+
+略
+
+
diff --git a/restwriter/pom.xml b/restwriter/pom.xml
@@ -0,0 +1,99 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>com.alibaba.datax</groupId>
+		<artifactId>datax-all</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+
+	<artifactId>restwriter</artifactId>
+	<name>restwriter</name>
+	<description>RestWriter提供了通过HTTP请求写入的功能</description>
+	<packaging>jar</packaging>
+
+	<dependencies>
+		<dependency>
+			<groupId>com.alibaba.datax</groupId>
+			<artifactId>datax-common</artifactId>
+			<version>${datax-project-version}</version>
+			<exclusions>
+				<exclusion>
+					<artifactId>slf4j-log4j12</artifactId>
+					<groupId>org.slf4j</groupId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-classic</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.projectlombok</groupId>
+			<artifactId>lombok</artifactId>
+			<version>1.18.28</version>
+		</dependency>
+		<dependency>
+			<groupId>com.google.guava</groupId>
+			<artifactId>guava</artifactId>
+			<version>32.1.2-jre</version>
+		</dependency>
+		<dependency>
+			<groupId>com.konghq</groupId>
+			<artifactId>unirest-java</artifactId>
+			<version>3.14.5</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-collections4</artifactId>
+			<version>4.4</version>
+		</dependency>
+		<dependency>
+			<groupId>dev.failsafe</groupId>
+			<artifactId>failsafe</artifactId>
+			<version>3.3.2</version>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-expression</artifactId>
+			<version>5.3.30</version>
+		</dependency>
+
+	</dependencies>
+
+	<build>
+		<plugins>
+			<!-- compiler plugin -->
+			<plugin>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<configuration>
+					<source>${jdk-version}</source>
+					<target>${jdk-version}</target>
+					<encoding>${project-sourceEncoding}</encoding>
+				</configuration>
+			</plugin>
+			<plugin>
+				<artifactId>maven-assembly-plugin</artifactId>
+				<configuration>
+					<descriptors>
+						<descriptor>src/main/assembly/package.xml</descriptor>
+					</descriptors>
+					<finalName>datax</finalName>
+				</configuration>
+				<executions>
+					<execution>
+						<id>dwzip</id>
+						<phase>package</phase>
+						<goals>
+							<goal>single</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+</project>
diff --git a/restwriter/src/main/assembly/package.xml b/restwriter/src/main/assembly/package.xml
@@ -0,0 +1,35 @@
+<assembly
+	xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+	<id></id>
+	<formats>
+		<format>dir</format>
+	</formats>
+	<includeBaseDirectory>false</includeBaseDirectory>
+	<fileSets>
+		<fileSet>
+			<directory>src/main/resources</directory>
+			<includes>
+				<include>plugin.json</include>
+				<include>plugin_job_template.json</include>
+			</includes>
+			<outputDirectory>plugin/writer/restwriter</outputDirectory>
+		</fileSet>
+		<fileSet>
+			<directory>target/</directory>
+			<includes>
+				<include>restwriter-0.0.1-SNAPSHOT.jar</include>
+			</includes>
+			<outputDirectory>plugin/writer/restwriter</outputDirectory>
+		</fileSet>
+	</fileSets>
+
+	<dependencySets>
+		<dependencySet>
+			<useProjectArtifact>false</useProjectArtifact>
+			<outputDirectory>plugin/writer/restwriter/libs</outputDirectory>
+			<scope>runtime</scope>
+		</dependencySet>
+	</dependencySets>
+</assembly>