From 10bb95c11a5e43da4b963809e75c5c2c8a9f3998 Mon Sep 17 00:00:00 2001 From: fangjian0423 Date: Thu, 22 Nov 2018 19:18:40 +0800 Subject: [PATCH 1/4] sentinel datasource refactor: including code refactor and update examples, docs --- .../src/main/asciidoc-zh/sentinel.adoc | 256 ++++++++++ .../sentinel-core-example/pom.xml | 18 + .../sentinel-core-example/readme-zh.md | 98 ++-- .../sentinel-core-example/readme.md | 83 +--- ...er.java => JsonFlowRuleListConverter.java} | 2 +- .../cloud/examples/ServiceApplication.java | 2 +- .../src/main/resources/application.properties | 16 +- .../src/main/resources/degraderule.json | 16 + .../src/main/resources/flowrule.json | 26 + .../src/main/resources/flowrule.xml | 21 + .../pom.xml | 19 + .../sentinel/datasource/DataSourceLoader.java | 152 ------ .../SentinelDataSourcePostProcessor.java | 278 ----------- .../SentinelDataSourceRegistry.java | 68 --- .../annotation/SentinelDataSource.java | 50 -- .../config/AbstractDataSourceProperties.java | 38 ++ .../config/ApolloDataSourceProperties.java | 44 ++ .../DataSourcePropertiesConfiguration.java | 75 +++ .../config/FileDataSourceProperties.java | 53 ++ .../config/NacosDataSourceProperties.java | 44 ++ .../config/ZookeeperDataSourceProperties.java | 56 +++ .../datasource/converter/JsonConverter.java | 161 ++++++ .../datasource/converter/XmlConverter.java | 160 ++++++ .../datasource/util/PropertySourcesUtils.java | 75 --- spring-cloud-alibaba-sentinel/pom.xml | 6 + .../alibaba/sentinel/SentinelProperties.java | 459 ++++++++++-------- .../custom/SentinelAutoConfiguration.java | 232 +++++---- .../custom/SentinelBeanPostProcessor.java | 2 +- .../custom/SentinelDataSourceHandler.java | 353 ++++++++++++++ .../sentinel/endpoint/SentinelEndpoint.java | 30 +- 30 files changed, 1808 insertions(+), 1085 deletions(-) rename spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/{JsonFlowRuleListParser.java => JsonFlowRuleListConverter.java} (83%) create mode 100644 spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/degraderule.json create mode 100644 spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/flowrule.json create mode 100644 spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/flowrule.xml delete mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/DataSourceLoader.java delete mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/SentinelDataSourcePostProcessor.java delete mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/SentinelDataSourceRegistry.java delete mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/annotation/SentinelDataSource.java create mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/AbstractDataSourceProperties.java create mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/ApolloDataSourceProperties.java create mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/DataSourcePropertiesConfiguration.java create mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/FileDataSourceProperties.java create mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/NacosDataSourceProperties.java create mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/ZookeeperDataSourceProperties.java create mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/converter/JsonConverter.java create mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/converter/XmlConverter.java delete mode 100644 spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/util/PropertySourcesUtils.java create mode 100644 spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelDataSourceHandler.java diff --git a/spring-cloud-alibaba-docs/src/main/asciidoc-zh/sentinel.adoc b/spring-cloud-alibaba-docs/src/main/asciidoc-zh/sentinel.adoc index 59eaa589c..e4c9817fc 100644 --- a/spring-cloud-alibaba-docs/src/main/asciidoc-zh/sentinel.adoc +++ b/spring-cloud-alibaba-docs/src/main/asciidoc-zh/sentinel.adoc @@ -1 +1,257 @@ == Spring Cloud Alibaba Sentinel + +### Sentinel介绍 + +随着微服务的流行，服务和服务之间的稳定性变得越来越重要。Sentinel 以流量为切入点，从流量控制、熔断降级、系统负载保护等多个维度保护服务的稳定性。 + +Sentinel 具有以下特征: + + +* *丰富的应用场景*：Sentinel 承接了阿里巴巴近 10 年的双十一大促流量的核心场景，例如秒杀（即突发流量控制在系统容量可以承受的范围）、消息削峰填谷、实时熔断下游不可用应用等。 +* *完备的实时监控*：Sentinel 同时提供实时的监控功能。您可以在控制台中看到接入应用的单台机器秒级数据，甚至 500 台以下规模的集群的汇总运行情况。 +* *广泛的开源生态*：Sentinel 提供开箱即用的与其它开源框架/库的整合模块，例如与 Spring Cloud、Dubbo、gRPC 的整合。您只需要引入相应的依赖并进行简单的配置即可快速地接入 Sentinel。 +* *完善的 SPI 扩展点*：Sentinel 提供简单易用、完善的 SPI 扩展点。您可以通过实现扩展点，快速的定制逻辑。例如定制规则管理、适配数据源等。 + +### 如何使用Sentinel + +如果要在您的项目中引入Sentinel，使用group ID为 `org.springframework.cloud` 和artifact ID为 `spring-cloud-starter-alibaba-sentinel` 的starter。 + +```xml + + org.springframework.cloud + spring-cloud-starter-alibaba-sentinel + +``` + +下面这个例子就是一个最简单的使用Sentinel的例子: + +```java +@SpringBootApplication +public class Application { + + public static void main(String[] args) { + SpringApplication.run(ServiceApplication.class, args); + } + +} + +@RestController +public class TestController { + + @GetMapping(value = "/hello") + @SentinelResource("hello") + public String hello() { + return "Hello Sentinel"; + } + +} +``` + +@SentinelResource注解用来标识资源是否被限流、降级。上述例子上该注解的属性'hello'表示资源名。 + +@SentinelResource还提供了其它额外的属性如 `blockHandler`，`blockHandlerClass`，`fallback` 用于表示限流或降级的操作，更多内容可以参考 https://github.com/alibaba/Sentinel/wiki/%E6%B3%A8%E8%A7%A3%E6%94%AF%E6%8C%81[Sentinel注解支持]。 + +##### Sentinel 控制台 + +Sentinel 控制台提供一个轻量级的控制台，它提供机器发现、单机资源实时监控、集群资源汇总，以及规则管理的功能。您只需要对应用进行简单的配置，就可以使用这些功能。 + +*注意*: 集群资源汇总仅支持 500 台以下的应用集群，有大概 1 - 2 秒的延时。 + +.Sentinel Dashboard +image::https://github.com/alibaba/Sentinel/wiki/image/dashboard.png[] + +开启该功能需要3个步骤： + +###### 获取控制台 + +您可以从 https://github.com/alibaba/Sentinel/releases[release 页面] 下载最新版本的控制台 jar 包。 + +您也可以从最新版本的源码自行构建 Sentinel 控制台： + +* 下载 https://github.com/alibaba/Sentinel/tree/master/sentinel-dashboard[控制台] 工程 +* 使用以下命令将代码打包成一个 fat jar: `mvn clean package` + + +###### 启动控制台 + +Sentinel 控制台是一个标准的SpringBoot应用，以SpringBoot的方式运行jar包即可。 + +```shell +java -Dserver.port=8080 -Dcsp.sentinel.dashboard.server=localhost:8080 -Dproject.name=sentinel-dashboard -jar sentinel-dashboard.jar +``` + +如若8080端口冲突，可使用 `-Dserver.port=新端口` 进行设置。 + +#### 配置控制台信息 + +.application.yml +---- +spring: + cloud: + sentinel: + transport: + port: 8719 + dashboard: localhost:8080 +---- + +这里的 `spring.cloud.sentinel.transport.port` 端口配置会在应用对应的机器上启动一个Http Server，该Serve会与Sentinel控制台做交互。比如Sentinel控制台添加了1个限流规则，会把规则数据push给这个Http Server接受，Http Server再将规则注册到Sentinel中。 + +更多Sentinel控制台的使用及问题参考： https://github.com/alibaba/Sentinel/wiki/%E6%8E%A7%E5%88%B6%E5%8F%B0[Sentinel控制台] + +### Feign支持 + +DOING + +### RestTemplate支持 + +Spring Cloud Alibaba Sentinel支持对 `RestTemplate` 的服务调用使用Sentinel进行保护，在构造 `RestTemplate` bean的时候需要加上 `@SentinelProtect` 注解。 + +```java +@Bean +@SentinelProtect(blockHandler = "handleException", blockHandlerClass = ExceptionUtil.class) +public RestTemplate restTemplate() { + return new RestTemplate(); +} +``` + +`@SentinelProtect` 注解的参数跟 `@SentinelResource` 一致，使用方式也一致。 + +限流的资源规则提供两种粒度： + +* `schema://host:port/path`：协议、主机、端口和路径 + +* `schema://host:port`：协议、主机和端口 + +NOTE: 以 `https://www.taobao.com/test` 这个url为例。对应的资源名有两种粒度，分别是 `https://www.taobao.com:80` 以及 `https://www.taobao.com:80/test` + + +### 动态数据源支持 + +*在版本 0.2.0.RELEASE 或 0.1.0.RELEASE 之前*，要在Spring Cloud Alibaba Sentinel下使用动态数据源，需要3个步骤： + +* 配置文件中定义数据源信息。比如使用文件： + +.application.properties +---- +spring.cloud.sentinel.datasource.type=file +spring.cloud.sentinel.datasource.recommendRefreshMs=3000 +spring.cloud.sentinel.datasource.bufSize=4056196 +spring.cloud.sentinel.datasource.charset=utf-8 +spring.cloud.sentinel.datasource.converter=flowConverter +spring.cloud.sentinel.datasource.file=/Users/you/yourrule.json +---- + +* 创建一个Converter类实现 `com.alibaba.csp.sentinel.datasource.Converter` 接口，并且需要在 `ApplicationContext` 中有该类的一个Bean + +```java +@Component("flowConverter") +public class JsonFlowRuleListParser implements Converter> { + @Override + public List convert(String source) { + return JSON.parseObject(source, new TypeReference>() { + }); + } +} +``` + +这个Converter的bean name需要跟 `application.properties` 配置文件中的converter配置一致 + +* 在任意一个 Spring Bean 中定义一个被 `@SentinelDataSource` 注解修饰的 `ReadableDataSource` 属性 + +```java +@SentinelDataSource("spring.cloud.sentinel.datasource") +private ReadableDataSource dataSource; +``` + +`@SentinelDataSource` 注解的value属性表示数据源在 `application.properties` 配置文件中前缀。 该在例子中，前缀为 `spring.cloud.sentinel.datasource` 。 + +如果 `ApplicationContext` 中存在超过1个 `ReadableDataSource` bean，那么不会加载这些 `ReadableDataSource` 中的任意一个。 只有在 `ApplicationContext` 存在一个 `ReadableDataSource` 的情况下才会生效。 + +如果数据源生效并且规则成功加载，控制台会打印类似如下信息： + +``` +[Sentinel Starter] load 3 flow rules +``` + +*在版本 0.2.0.RELEASE 或 0.1.0.RELEASE 之后*，要在Spring Cloud Alibaba Sentinel下使用动态数据源，只需要1个步骤： + +* 直接在 `application.properties` 配置文件中配置数据源信息即可 + +比如配置了4个数据源： + +``` +spring.cloud.sentinel.datasource.ds1.file.file=classpath: degraderule.json + +spring.cloud.sentinel.datasource.ds2.nacos.server-addr=localhost:8848 +spring.cloud.sentinel.datasource.ds2.nacos.dataId=sentinel +spring.cloud.sentinel.datasource.ds2.nacos.groupId=DEFAULT_GROUP +spring.cloud.sentinel.datasource.ds2.nacos.data-type=json + +spring.cloud.sentinel.datasource.ds3.zk.path = /Sentinel-Demo/SYSTEM-CODE-DEMO-FLOW +spring.cloud.sentinel.datasource.ds3.zk.server-addr = localhost:2181 + +spring.cloud.sentinel.datasource.ds4.apollo.namespace-name = application +spring.cloud.sentinel.datasource.ds4.apollo.flow-rules-key = sentinel +spring.cloud.sentinel.datasource.ds4.apollo.default-flow-rule-value = test + +``` + +这样配置方式参考了Spring Cloud Stream Binder的配置，内部使用了 `TreeMap` 进行存储，comparator为 `String.CASE_INSENSITIVE_ORDER` 。 + +NOTE: d1, ds2, ds3, ds4 是 `ReadableDataSource` 的名字，可随意编写。后面的 `file` ，`zk` ，`nacos` , `apollo` 就是对应具体的数据源。 它们后面的配置就是这些数据源各自的配置。 + +每种数据源都有两个共同的配置项： `data-type` 和 `converter-class` 。 + +`data-type` 配置项表示 `Converter`，Spring Cloud Alibaba Sentinel默认提供两种内置的值，分别是 `json` 和 `xml` (不填默认是json)。 如果不想使用内置的 `json` 或 `xml` 这两种 `Converter`，可以填写 `custom` 表示自定义 `Converter`，然后再配置 `converter-class` 配置项，该配置项需要写类的全路径名。 + +这两种内置的 `Converter` 只支持解析 json数组 或 xml数组。内部解析的时候会自动判断每个json对象或xml对象属于哪4种Sentinel规则(`FlowRule`，`DegradeRule`，`SystemRule`，`AuthorityRule`)。 + +比如10个规则数组里解析出5个限流规则和5个降级规则。 这种情况下该数据源不会注册，日志里页会进行警告。 + +如果10个规则里有9个限流规则，1个解析报错了。这种情况下日志会警告有个规则格式错误，另外9个限流规则会注册上去。 + +这里json或xml解析用的是 `jackson`。`ObjectMapper` 或 `XmlMapper` 使用默认的配置，遇到不认识的字段会解析报错。 因为不这样做的话限流 json 也会解析成 系统规则(系统规则所有的配置都有默认值)，所以需要这样严格解析。 + +当然还有一种情况是json对象或xml对象可能会匹配上所有4种规则(比如json对象里只配了 `resource` 字段，那么会匹配上4种规则)，这种情况下日志里会警告，并且过滤掉这个对象。 + +用户使用这种配置的时候只需要填写正确的json或xml就行，有任何不合理的信息都会在日志里打印出来。 + +如果数据源生效并且规则成功加载，控制台会打印类似如下信息： + +``` +[Sentinel Starter] DataSource ds1-sentinel-file-datasource load 3 DegradeRule +[Sentinel Starter] DataSource ds2-sentinel-nacos-datasource load 2 FlowRule +``` + +NOTE: 默认情况下，xml格式是不支持的。需要添加 `jackson-dataformat-xml` 依赖后才会自动生效。 + +关于Sentinel动态数据源的实现原理，参考： https://github.com/alibaba/Sentinel/wiki/%E5%8A%A8%E6%80%81%E8%A7%84%E5%88%99%E6%89%A9%E5%B1%95[动态规则扩展] + +### Endpoint支持 + +在使用Endpoint特性之前需要在 Maven 中添加 `spring-boot-starter-actuator` 依赖，并在配置中允许 Endpoints 的访问。 + +* Spring Boot 1.x 中添加配置 `management.security.enabled=false`。暴露的endpoint路径为 `/sentinel` +* Spring Boot 2.x 中添加配置 `management.endpoints.web.exposure.include=*`。暴露的endpoint路径为 `/actuator/sentinel` + +### More + +下表显示 Spring Cloud Alibaba Sentinel 的所有配置信息： + +:frame: topbot +[width="60%",options="header"] +|==== +^|配置项 ^|含义 ^|默认值 +|`spring.cloud.sentinel.enabled`|Sentinel自动化配置是否生效|true +|`spring.cloud.sentinel.eager`|取消Sentinel控制台懒加载|false +|`spring.cloud.sentinel.charset`|metric文件字符集|UTF-8 +|`spring.cloud.sentinel.transport.port`|应用与Sentinel控制台交互的端口，应用本地会起一个该端口占用的HttpServer|8721 +|`spring.cloud.sentinel.transport.dashboard`|Sentinel 控制台地址| +|`spring.cloud.sentinel.transport.heartbeatIntervalMs`|应用与Sentinel控制台的心跳间隔时间| +|`spring.cloud.sentinel.filter.order`|Servlet Filter的加载顺序。Starter内部会构造这个filter|Integer.MIN_VALUE +|`spring.cloud.sentinel.filter.spring.url-patterns`|数据类型是数组。表示Servlet Filter的url pattern集合|/* +|`spring.cloud.sentinel.metric.fileSingleSize`|Sentinel metric 单个文件的大小| +|`spring.cloud.sentinel.metric.fileTotalCount`|Sentinel metric 总文件数量| +|`spring.cloud.sentinel.servlet.blockPage`| 自定义的跳转 URL，当请求被限流时会自动跳转至设定好的 URL | +|`spring.cloud.sentinel.flow.coldFactor`| https://github.com/alibaba/Sentinel/wiki/%E9%99%90%E6%B5%81---%E5%86%B7%E5%90%AF%E5%8A%A8[冷启动因子] |3 +|==== diff --git a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/pom.xml b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/pom.xml index e8ffc6852..3fe3b98b2 100644 --- a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/pom.xml +++ b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/pom.xml @@ -31,6 +31,24 @@ spring-boot-starter-actuator + + + + + + + + + + + + + + + + + + diff --git a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/readme-zh.md b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/readme-zh.md index c07c20e37..af535e6da 100644 --- a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/readme-zh.md +++ b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/readme-zh.md @@ -196,77 +196,33 @@ Sentinel 控制台支持实时监控查看，您可以通过 Sentinel 控制台 Sentinel 内部提供了[动态规则的扩展实现 ReadableDataSource](https://github.com/alibaba/Sentinel/wiki/%E5%8A%A8%E6%80%81%E8%A7%84%E5%88%99%E6%89%A9%E5%B1%95#datasource-%E6%89%A9%E5%B1%95)。 -Sentinel starter 整合了目前存在的几类 DataSource。只需要在配置文件中进行相关配置，即可在 Spring 容器中自动注册 DataSource。 - -比如要定义一个 `FileRefreshableDataSource`，配置如下： - - spring.cloud.sentinel.datasource.type=file - spring.cloud.sentinel.datasource.recommendRefreshMs=2000 - spring.cloud.sentinel.datasource.bufSize=2048 - spring.cloud.sentinel.datasource.charset=utf-8 - spring.cloud.sentinel.datasource.converter=myParser - spring.cloud.sentinel.datasource.file=/Users/you/rule.json - -然后使用`@SentinelDataSource` 注解修饰 DataSource 即可注入： - - @SentinelDataSource("spring.cloud.sentinel.datasource") - private ReadableDataSource dataSource; - -`@SentinelDataSource` 注解的 value 属性可以不填。默认值就是 `spring.cloud.sentinel.datasource`。 - -`value` 属性代表配置前缀。示例中会去找 `spring.cloud.sentinel.datasource.xxx` 相关的配置。 - -`spring.cloud.sentinel.datasource.type` 就是对应的 DataSource 类型。 - -`spring.cloud.sentinel.datasource.recommendRefreshMs` 里的 `recommendRefreshMs` 对应相关 DataSource 的属性。 - -`spring.cloud.sentinel.datasource.converter`代表 `Converter` 在 Spring 容器里的 name。如果没找到，会抛出异常。 - -type目前支持file, nacos, zk, apollo。其中nacos，zk，apollo的使用需要加上对应的依赖`sentinel-datasource-nacos`, `sentinel-datasource-zookeeper`, `sentinel-datasource-apollo` - -### 自定义DataSource - -自定义DataSource只需要两步。 - -1. 定义DataSource - - public class CustomDataSource implements ReadableDataSource { - private String fieldA; - private String fieldB; - ... - } - -2. 装配DataSource。有两种方式处理。 - - * 直接构造DataSource - - @Bean - public CustomDataSource customDataSource() { - CustomDataSource customDataSource = - new CustomDataSource(); - customDataSource.setFieldA("valueA"); - customDataSource.setFieldB("valueB"); - ... - return customDataSource; - } - - * 在classpath:/META-INF/sentinel-datasource.properties中管理DataSource信息 - - custom = yourpackage.CustomDataSource - - 在application.properties中定义DataSource - - spring.cloud.sentinel.datasource.type = custom - spring.cloud.sentinel.datasource.fieldA = valueA - spring.cloud.sentinel.datasource.fieldB = valueB - - 注意：由于目前Sentinel的AbstractDataSource需要有个Converter作为构造函数中的参数，并且它的子类的构造都是通过多个参数的构造函数构造的。 - 所以目前所有的Sentinel starter中的DataSource都是基于FactoryBean并且通过设置属性构造的。如果有这方面的需求，需要再多加一个registerFactoryBean过程。 - - SentinelDataSourceRegistry.registerFactoryBean("custeom", CustomDataSourceFactoryBean.class); - - 如果自定义DataSource可以注入属性，那么没有必要使用SentinelDataSourceRegistry注册FactoryBean。 - +Sentinel starter 整合了目前存在的几类 ReadableDataSource。只需要在配置文件中进行相关配置，即可在 Spring 容器中自动注册 DataSource。 + +比如要定义两个ReadableDataSource，分别是 `FileRefreshableDataSource` 和 `NacosDataSource`，配置如下： + +```properties +spring.cloud.sentinel.datasource.ds1.file.file=classpath: degraderule.json +spring.cloud.sentinel.datasource.ds1.file.data-type=json + +spring.cloud.sentinel.datasource.ds2.nacos.server-addr=localhost:8848 +spring.cloud.sentinel.datasource.ds2.nacos.dataId=sentinel +spring.cloud.sentinel.datasource.ds2.nacos.groupId=DEFAULT_GROUP +spring.cloud.sentinel.datasource.ds2.nacos.data-type=json +``` + +`ds1` 和 `ds2` 表示ReadableDataSource的名称，可随意编写。`ds1` 和 `ds2` 后面的 `file` 和 `nacos` 表示ReadableDataSource的类型。 + +目前支持`file`, `nacos`, `zk`, `apollo` 这4种类型。 + +其中`nacos`，`zk`，`apollo`这3种类型的使用需要加上对应的依赖`sentinel-datasource-nacos`, `sentinel-datasource-zookeeper`, `sentinel-datasource-apollo`。 + +当ReadableDataSource加载规则数据成功的时候，控制台会打印出相应的日志信息： + +``` +[Sentinel Starter] DataSource ds1-sentinel-file-datasource load 3 DegradeRule +[Sentinel Starter] DataSource ds2-sentinel-nacos-datasource load 2 FlowRule +``` + ## More Sentinel 是一款功能强大的中间件，从流量控制，熔断降级，系统负载保护等多个维度保护服务的稳定性。此 Demo 仅演示了 使用 Sentinel 作为限流工具的使用，更多 Sentinel 相关的信息，请参考 [Sentinel 项目](https://github.com/alibaba/Sentinel)。 diff --git a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/readme.md b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/readme.md index 157ad0e88..e31357295 100644 --- a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/readme.md +++ b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/readme.md @@ -71,7 +71,7 @@ Before we start the demo, let's learn how to connect Sentinel to a Spring Cloud 2. Start the application in IDE or by building a fatjar. 1. Start in IDE: Find main class `ServiceApplication`, and execute the main method. - 2. Build a fatjar：Execute command `mvn clean package` to build a fatjar，and run command `java -jar sentinel-core-example.jar` to start the application. + 2. Build a fatjar：Execute command `mvn clean package` to build a fatjar, and run command `java -jar sentinel-core-example.jar` to start the application. ### Invoke Service @@ -86,7 +86,7 @@ The screenshot belows shows invoke success: 1. Open http://localhost:8080 in browser, and you can find a Sentinel-Example Application has been registered to the dashboard. - **Note: If you can't find your application in the dashboard, invoke a method that has been defined as a Sentinel Resource，for Sentinel uses lazy load strategy.** + **Note: If you can't find your application in the dashboard, invoke a method that has been defined as a Sentinel Resource, for Sentinel uses lazy load strategy.**

@@ -125,8 +125,8 @@ The screenshot belows shows invoke success: 2. When a custom resource is blocked by Sentinel, the default logic is throw BlockException. - If you want to customize your flow control logic, implement interface `SentinelExceptionHandler`, set @SentinelResource's blockHandler() and blockHandlerClass(). See the code below: - + If you want to customize your flow control logic, implement interface `SentinelExceptionHandler`, set @SentinelResource's blockHandler() and blockHandlerClass(). See the code below: + @SentinelResource(value = "resource", blockHandler = "", blockHandlerClass = ExceptionUtil.class) public String hello() { return "Hello"; @@ -171,71 +171,30 @@ Sentinel provide [ReadableDataSource](https://github.com/alibaba/Sentinel/blob/m Sentinel starter integrated 4 DataSources provided by Sentinel. It will be register into Spring Context if you write some configs in `application.properties`. -If you want to define FileRefreshableDataSource: +If you want to define `FileRefreshableDataSource` and `NacosDataSource`, see the code below: - spring.cloud.sentinel.datasource.type=file - spring.cloud.sentinel.datasource.recommendRefreshMs=2000 - spring.cloud.sentinel.datasource.bufSize=2048 - spring.cloud.sentinel.datasource.charset=utf-8 - spring.cloud.sentinel.datasource.converter=myParser - spring.cloud.sentinel.datasource.file=/Users/you/rule.json - -then use `@SentinelDataSource` to annotate DataSource: - - @SentinelDataSource("spring.cloud.sentinel.datasource") - private ReadableDataSource dataSource; - -The value() of `@SentinelDataSource` is not required, it means the prefix of configuration. Default value is `spring.cloud.sentinel.datasource`. +```properties +spring.cloud.sentinel.datasource.ds1.file.file=classpath: degraderule.json +spring.cloud.sentinel.datasource.ds1.file.data-type=json -spring.cloud.sentinel.datasource.type means the type of DataSource. +spring.cloud.sentinel.datasource.ds2.nacos.server-addr=localhost:8848 +spring.cloud.sentinel.datasource.ds2.nacos.dataId=sentinel +spring.cloud.sentinel.datasource.ds2.nacos.groupId=DEFAULT_GROUP +spring.cloud.sentinel.datasource.ds2.nacos.data-type=json +``` -spring.cloud.sentinel.datasource.recommendRefreshMs means the recommendRefreshMs property of specified DataSource. +`ds1` and `ds2` means the name of ReadableDataSource, you can write whatever you want. The `file` and `nacos` after name `ds1` and `ds2` means the type of ReadableDataSource. -spring.cloud.sentinel.datasource.converter means the name of spring bean that type is Converter. If the bean is not exists, will throw exception. - -Now datasource type support 4 categories: file, nacos, zk, apollo. If you want to using nacos, zk or apollo, you should add `sentinel-datasource-nacos`, `sentinel-datasource-zookeeper` or `sentinel-datasource-apollo` dependency. +Now ReadableDataSource type support 4 categories: `file`, `nacos`, `zk` and `apollo`. -### User-defined DataSource +If you want to use `nacos`, `zk` or `apollo` ReadableDataSource, you could add `sentinel-datasource-nacos`, `sentinel-datasource-zookeeper` or `sentinel-datasource-apollo` dependency. -User-defined DataSource need 2 steps. +When ReadableDataSource load rule data successfully, console will print some logs: -1. Define DataSource - - public class CustomDataSource implements ReadableDataSource { - private String fieldA; - private String fieldB; - ... - } - -2. Assemble DataSource. There are 2 ways to do this. - - * Construct DataSource directly - - @Bean - public CustomDataSource customDataSource() { - CustomDataSource customDataSource = new CustomDataSource(); - customDataSource.setFieldA("valueA"); - customDataSource.setFieldB("valueB"); - ... - return customDataSource; - } - - * define DataSource metadata in `classpath:/META-INF/sentinel-datasource.properties` - - custom = yourpackage.CustomDataSource - - define configuration in `application.properties` - - spring.cloud.sentinel.datasource.type = custom - spring.cloud.sentinel.datasource.fieldA = valueA - spring.cloud.sentinel.datasource.fieldB = valueB - -Note: The AbstractDataSource of Sentinel need a Converter as a constructor param and the subclass of AbstractDataSource was construct by multi-param constructor. -Now All DataSources in starter was construct by FactoryBean. If you want to do it in this way, you should register FactoryBean by SentinelDataSourceRegistry. - - SentinelDataSourceRegistry.registerFactoryBean("custeom", CustomDataSourceFactoryBean.class); - -It is no need to using SentinelDataSourceRegistry to register FactoryBean if your User-defined DataSource can inject fields. +``` +[Sentinel Starter] DataSource ds1-sentinel-file-datasource load 3 DegradeRule +[Sentinel Starter] DataSource ds2-sentinel-nacos-datasource load 2 FlowRule +``` ## More For more information about Sentinel, see [Sentinel Project](https://github.com/alibaba/Sentinel). diff --git a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/JsonFlowRuleListParser.java b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/JsonFlowRuleListConverter.java similarity index 83% rename from spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/JsonFlowRuleListParser.java rename to spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/JsonFlowRuleListConverter.java index b205c96c5..c5cea94df 100644 --- a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/JsonFlowRuleListParser.java +++ b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/JsonFlowRuleListConverter.java @@ -10,7 +10,7 @@ import com.alibaba.fastjson.TypeReference; /** * @author fangjian */ -public class JsonFlowRuleListParser implements Converter> { +public class JsonFlowRuleListConverter implements Converter> { @Override public List convert(String source) { return JSON.parseObject(source, new TypeReference>() { diff --git a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/ServiceApplication.java b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/ServiceApplication.java index 8ced5aff1..93183382c 100644 --- a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/ServiceApplication.java +++ b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/ServiceApplication.java @@ -27,7 +27,7 @@ public class ServiceApplication { @Bean public Converter myParser() { - return new JsonFlowRuleListParser(); + return new JsonFlowRuleListConverter(); } public static void main(String[] args) { diff --git a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/application.properties b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/application.properties index b61171907..19c2868e8 100644 --- a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/application.properties +++ b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/application.properties @@ -1,14 +1,16 @@ spring.application.name=sentinel-example server.port=18083 management.security.enabled=false -spring.cloud.sentinel.transport.port=8721 spring.cloud.sentinel.transport.dashboard=localhost:8080 +spring.cloud.sentinel.eager=true +spring.cloud.sentinel.datasource.ds1.file.file=classpath: flowrule.json +spring.cloud.sentinel.datasource.ds1.file.data-type=json -spring.cloud.sentinel.datasource.type=file -spring.cloud.sentinel.datasource.recommendRefreshMs=3000 -spring.cloud.sentinel.datasource.bufSize=4056196 -spring.cloud.sentinel.datasource.charset=utf-8 -spring.cloud.sentinel.datasource.converter=myParser -spring.cloud.sentinel.datasource.file=/Users/you/rule.json \ No newline at end of file +#spring.cloud.sentinel.datasource.ds1.file.file=classpath: flowrule.json +#spring.cloud.sentinel.datasource.ds1.file.data-type=custom +#spring.cloud.sentinel.datasource.ds1.file.converter-class=org.springframework.cloud.alibaba.cloud.examples.JsonFlowRuleListConverter + +spring.cloud.sentinel.datasource.ds2.file.file=classpath: degraderule.json +spring.cloud.sentinel.datasource.ds2.file.data-type=json \ No newline at end of file diff --git a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/degraderule.json b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/degraderule.json new file mode 100644 index 000000000..5977c5fcf --- /dev/null +++ b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/degraderule.json @@ -0,0 +1,16 @@ +[ + { + "resource": "abc0", + "count": 20.0, + "grade": 0, + "passCount": 0, + "timeWindow": 10 + }, + { + "resource": "abc1", + "count": 15.0, + "grade": 0, + "passCount": 0, + "timeWindow": 10 + } +] \ No newline at end of file diff --git a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/flowrule.json b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/flowrule.json new file mode 100644 index 000000000..679dff4a4 --- /dev/null +++ b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/flowrule.json @@ -0,0 +1,26 @@ +[ + { + "resource": "resource", + "controlBehavior": 0, + "count": 1, + "grade": 1, + "limitApp": "default", + "strategy": 0 + }, + { + "resource": "p", + "controlBehavior": 0, + "count": 1, + "grade": 1, + "limitApp": "default", + "strategy": 0 + }, + { + "resource": "abc", + "controlBehavior": 0, + "count": 1, + "grade": 1, + "limitApp": "default", + "strategy": 0 + } +] diff --git a/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/flowrule.xml b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/flowrule.xml new file mode 100644 index 000000000..7f1926d3e --- /dev/null +++ b/spring-cloud-alibaba-examples/sentinel-example/sentinel-core-example/src/main/resources/flowrule.xml @@ -0,0 +1,21 @@ + + + + resource + 0 + 1 + 1 + default + 0 + + + test + 0 + 1 + 1 + default + 0 + + + + diff --git a/spring-cloud-alibaba-sentinel-datasource/pom.xml b/spring-cloud-alibaba-sentinel-datasource/pom.xml index af959677a..db4ed9974 100644 --- a/spring-cloud-alibaba-sentinel-datasource/pom.xml +++ b/spring-cloud-alibaba-sentinel-datasource/pom.xml @@ -41,8 +41,27 @@ true + + com.fasterxml.jackson.core + jackson-databind + provided + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + provided + + + + org.springframework.boot + spring-boot-configuration-processor + provided + true + + org.springframework.boot spring-boot diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/DataSourceLoader.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/DataSourceLoader.java deleted file mode 100644 index da894ae5e..000000000 --- a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/DataSourceLoader.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (C) 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.alibaba.sentinel.datasource; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import com.alibaba.csp.sentinel.datasource.ReadableDataSource; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.core.io.Resource; -import org.springframework.core.io.support.PathMatchingResourcePatternResolver; -import org.springframework.core.io.support.PropertiesLoaderUtils; -import org.springframework.core.io.support.ResourcePatternResolver; -import org.springframework.util.Assert; -import org.springframework.util.ClassUtils; - -import static org.springframework.core.io.support.ResourcePatternResolver.CLASSPATH_ALL_URL_PREFIX; - -/** - * {@link ReadableDataSource} Loader - * - * @author Jim - */ -public class DataSourceLoader { - - private static final Logger logger = LoggerFactory.getLogger(DataSourceLoader.class); - - private final static String PROPERTIES_RESOURCE_LOCATION = "META-INF/sentinel-datasource.properties"; - - private final static String ALL_PROPERTIES_RESOURCES_LOCATION = CLASSPATH_ALL_URL_PREFIX - + PROPERTIES_RESOURCE_LOCATION; - - private final static ConcurrentMap> dataSourceClassesCache - = new ConcurrentHashMap>( - 4); - - static void loadAllDataSourceClassesCache() { - Map> dataSourceClassesMap = loadAllDataSourceClassesCache( - ALL_PROPERTIES_RESOURCES_LOCATION); - - dataSourceClassesCache.putAll(dataSourceClassesMap); - } - - static Map> loadAllDataSourceClassesCache( - String resourcesLocation) { - - Map> dataSourcesMap - = new HashMap>( - 4); - - ClassLoader classLoader = DataSourceLoader.class.getClassLoader(); - - ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); - - try { - - Resource[] resources = resolver.getResources(resourcesLocation); - - for (Resource resource : resources) { - if (resource.exists()) { - Properties properties = PropertiesLoaderUtils - .loadProperties(resource); - for (Map.Entry entry : properties.entrySet()) { - - String type = (String)entry.getKey(); - String className = (String)entry.getValue(); - - if (!ClassUtils.isPresent(className, classLoader)) { - if (logger.isDebugEnabled()) { - logger.debug( - "Sentinel DataSource implementation [ type : " - + type + ": , class : " + className - + " , url : " + resource.getURL() - + "] was not present in current classpath , " - + "thus loading will be ignored , please add dependency if required !"); - } - continue; - } - - Assert.isTrue(!dataSourcesMap.containsKey(type), - "The duplicated type[" + type - + "] of SentinelDataSource were found in " - + "resource [" + resource.getURL() + "]"); - - Class dataSourceClass = ClassUtils.resolveClassName(className, - classLoader); - Assert.isAssignable(ReadableDataSource.class, dataSourceClass); - - dataSourcesMap.put(type, - (Class)dataSourceClass); - - if (logger.isDebugEnabled()) { - logger.debug("Sentinel DataSource implementation [ type : " - + type + ": , class : " + className - + "] was loaded."); - } - } - } - } - - } catch (IOException e) { - if (logger.isErrorEnabled()) { - logger.error(e.getMessage(), e); - } - } - - return dataSourcesMap; - } - - public static Class loadClass(String type) - throws IllegalArgumentException { - - Class dataSourceClass = dataSourceClassesCache.get(type); - - if (dataSourceClass == null) { - if (dataSourceClassesCache.isEmpty()) { - loadAllDataSourceClassesCache(); - dataSourceClass = dataSourceClassesCache.get(type); - } - } - - if (dataSourceClass == null) { - throw new IllegalArgumentException( - "Sentinel DataSource implementation [ type : " + type - + " ] can't be found!"); - } - - return dataSourceClass; - - } - -} \ No newline at end of file diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/SentinelDataSourcePostProcessor.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/SentinelDataSourcePostProcessor.java deleted file mode 100644 index 73ece0ec4..000000000 --- a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/SentinelDataSourcePostProcessor.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (C) 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.alibaba.sentinel.datasource; - -import java.beans.PropertyDescriptor; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import com.alibaba.csp.sentinel.datasource.Converter; -import com.alibaba.csp.sentinel.datasource.ReadableDataSource; -import com.alibaba.csp.sentinel.property.SentinelProperty; -import com.alibaba.csp.sentinel.slots.block.authority.AuthorityRule; -import com.alibaba.csp.sentinel.slots.block.authority.AuthorityRuleManager; -import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule; -import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; -import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; -import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleManager; -import com.alibaba.csp.sentinel.slots.system.SystemRule; -import com.alibaba.csp.sentinel.slots.system.SystemRuleManager; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.PropertyValues; -import org.springframework.beans.factory.BeanCreationException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.config.InstantiationAwareBeanPostProcessorAdapter; -import org.springframework.beans.factory.support.BeanDefinitionBuilder; -import org.springframework.beans.factory.support.DefaultListableBeanFactory; -import org.springframework.beans.factory.support.MergedBeanDefinitionPostProcessor; -import org.springframework.beans.factory.support.RootBeanDefinition; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.cloud.alibaba.sentinel.datasource.annotation.SentinelDataSource; -import org.springframework.cloud.alibaba.sentinel.datasource.util.PropertySourcesUtils; -import org.springframework.context.ApplicationContext; -import org.springframework.context.event.EventListener; -import org.springframework.core.env.ConfigurableEnvironment; -import org.springframework.util.ReflectionUtils; -import org.springframework.util.StringUtils; - -import static org.springframework.core.annotation.AnnotationUtils.getAnnotation; - -/** - * {@link SentinelDataSource @SentinelDataSource} Post Processor - * - * @author Jim - * @see ReadableDataSource - * @see SentinelDataSource - */ -public class SentinelDataSourcePostProcessor - extends InstantiationAwareBeanPostProcessorAdapter - implements MergedBeanDefinitionPostProcessor { - - private static final Logger logger = LoggerFactory - .getLogger(SentinelDataSourcePostProcessor.class); - - @Autowired - private ApplicationContext applicationContext; - - @Autowired - private ConfigurableEnvironment environment; - - private final Map> dataSourceFieldCache = new ConcurrentHashMap<>( - 64); - - @Override - public void postProcessMergedBeanDefinition(RootBeanDefinition beanDefinition, - Class beanType, final String beanName) { - // find all fields using by @SentinelDataSource annotation - ReflectionUtils.doWithFields(beanType, new ReflectionUtils.FieldCallback() { - @Override - public void doWith(Field field) - throws IllegalArgumentException, IllegalAccessException { - SentinelDataSource annotation = getAnnotation(field, - SentinelDataSource.class); - if (annotation != null) { - if (Modifier.isStatic(field.getModifiers())) { - if (logger.isWarnEnabled()) { - logger.warn( - "@SentinelDataSource annotation is not supported on static fields: " - + field); - } - return; - } - if (dataSourceFieldCache.containsKey(beanName)) { - dataSourceFieldCache.get(beanName) - .add(new SentinelDataSourceField(annotation, field)); - } else { - List list = new ArrayList<>(); - list.add(new SentinelDataSourceField(annotation, field)); - dataSourceFieldCache.put(beanName, list); - } - } - } - }); - } - - @Override - public PropertyValues postProcessPropertyValues(PropertyValues pvs, - PropertyDescriptor[] pds, Object bean, String beanName) - throws BeanCreationException { - if (dataSourceFieldCache.containsKey(beanName)) { - List sentinelDataSourceFields = dataSourceFieldCache - .get(beanName); - for (SentinelDataSourceField sentinelDataSourceField : sentinelDataSourceFields) { - try { - // construct DataSource field annotated by @SentinelDataSource - Field field = sentinelDataSourceField.getField(); - ReflectionUtils.makeAccessible(field); - String dataSourceBeanName = constructDataSource( - sentinelDataSourceField.getSentinelDataSource()); - field.set(bean, applicationContext.getBean(dataSourceBeanName)); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - return pvs; - } - - private String constructDataSource(SentinelDataSource annotation) { - String prefix = annotation.value(); - if (StringUtils.isEmpty(prefix)) { - prefix = SentinelDataSourceConstants.PROPERTY_DATASOURCE_PREFIX; - } - Map propertyMap = PropertySourcesUtils - .getSubProperties(environment.getPropertySources(), prefix); - String alias = propertyMap.get("type").toString(); - Class dataSourceClass = DataSourceLoader.loadClass(alias); - - String beanName = StringUtils.isEmpty(annotation.name()) - ? StringUtils.uncapitalize(dataSourceClass.getSimpleName()) + "_" + prefix - : annotation.name(); - if (applicationContext.containsBean(beanName)) { - return beanName; - } - - Class targetClass = null; - // if alias exists in SentinelDataSourceRegistry, wired properties into - // FactoryBean - if (SentinelDataSourceRegistry.checkFactoryBean(alias)) { - targetClass = SentinelDataSourceRegistry.getFactoryBean(alias); - } else { - // if alias not exists in SentinelDataSourceRegistry, wired properties into - // raw class - targetClass = dataSourceClass; - } - - registerDataSource(beanName, targetClass, propertyMap); - - return beanName; - } - - private void registerDataSource(String beanName, Class targetClass, - Map propertyMap) { - BeanDefinitionBuilder builder = BeanDefinitionBuilder - .genericBeanDefinition(targetClass); - for (String propertyName : propertyMap.keySet()) { - Field field = ReflectionUtils.findField(targetClass, propertyName); - if (field != null) { - if (field.getType().isAssignableFrom(Converter.class)) { - // Converter get from ApplicationContext - builder.addPropertyReference(propertyName, - propertyMap.get(propertyName).toString()); - } else { - // wired properties - builder.addPropertyValue(propertyName, propertyMap.get(propertyName)); - } - } - } - - DefaultListableBeanFactory beanFactory = (DefaultListableBeanFactory)applicationContext - .getAutowireCapableBeanFactory(); - beanFactory.registerBeanDefinition(beanName, builder.getBeanDefinition()); - } - - @EventListener(classes = ApplicationReadyEvent.class) - public void appStartedListener(ApplicationReadyEvent event) throws Exception { - logger.info("[Sentinel Starter] Start to find ReadableDataSource"); - Map dataSourceMap = event.getApplicationContext().getBeansOfType( - ReadableDataSource.class); - if (dataSourceMap.size() == 1) { - logger.info("[Sentinel Starter] There exists only one ReadableDataSource named {}, start to load rules", - dataSourceMap.keySet().iterator().next()); - ReadableDataSource dataSource = dataSourceMap.values().iterator().next(); - Object ruleConfig = dataSource.loadConfig(); - SentinelProperty sentinelProperty = dataSource.getProperty(); - Integer rulesNum; - if ((rulesNum = checkRuleType(ruleConfig, FlowRule.class)) > 0) { - FlowRuleManager.register2Property(sentinelProperty); - logger.info("[Sentinel Starter] load {} flow rules", rulesNum); - } - if ((rulesNum = checkRuleType(ruleConfig, DegradeRule.class)) > 0) { - DegradeRuleManager.register2Property(sentinelProperty); - logger.info("[Sentinel Starter] load {} degrade rules", rulesNum); - } - if ((rulesNum = checkRuleType(ruleConfig, SystemRule.class)) > 0) { - SystemRuleManager.register2Property(sentinelProperty); - logger.info("[Sentinel Starter] load {} system rules", rulesNum); - } - if ((rulesNum = checkRuleType(ruleConfig, AuthorityRule.class)) > 0) { - AuthorityRuleManager.register2Property(sentinelProperty); - logger.info("[Sentinel Starter] load {} authority rules", rulesNum); - } - } else if (dataSourceMap.size() > 1) { - logger.warn( - "[Sentinel Starter] There exists more than one ReadableDataSource, can not choose which one to load"); - } else { - logger.warn( - "[Sentinel Starter] No ReadableDataSource exists"); - } - } - - private Integer checkRuleType(Object ruleConfig, Class type) { - if (ruleConfig.getClass() == type) { - return 1; - } else if (ruleConfig instanceof List) { - List ruleList = (List)ruleConfig; - List checkList = new ArrayList(); - for (Object rule : ruleList) { - if (rule.getClass() == type) { - checkList.add(rule); - } - } - if (ruleList.size() == checkList.size()) { - return ruleList.size(); - } - } - return -1; - } - - class SentinelDataSourceField { - private SentinelDataSource sentinelDataSource; - private Field field; - - public SentinelDataSourceField(SentinelDataSource sentinelDataSource, - Field field) { - this.sentinelDataSource = sentinelDataSource; - this.field = field; - } - - public SentinelDataSource getSentinelDataSource() { - return sentinelDataSource; - } - - public void setSentinelDataSource(SentinelDataSource sentinelDataSource) { - this.sentinelDataSource = sentinelDataSource; - } - - public Field getField() { - return field; - } - - public void setField(Field field) { - this.field = field; - } - } - -} diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/SentinelDataSourceRegistry.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/SentinelDataSourceRegistry.java deleted file mode 100644 index 5108196b5..000000000 --- a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/SentinelDataSourceRegistry.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.alibaba.sentinel.datasource; - -import java.util.HashMap; - -import com.alibaba.csp.sentinel.datasource.ReadableDataSource; - -import org.springframework.beans.factory.FactoryBean; -import org.springframework.cloud.alibaba.sentinel.datasource.factorybean.ApolloDataSourceFactoryBean; -import org.springframework.cloud.alibaba.sentinel.datasource.factorybean.FileRefreshableDataSourceFactoryBean; -import org.springframework.cloud.alibaba.sentinel.datasource.factorybean.NacosDataSourceFactoryBean; -import org.springframework.cloud.alibaba.sentinel.datasource.factorybean.ZookeeperDataSourceFactoryBean; - -/** - * Registry to save DataSource FactoryBean - * - * @author Jim - * @see ReadableDataSource - * @see FileRefreshableDataSourceFactoryBean - * @see ZookeeperDataSourceFactoryBean - * @see NacosDataSourceFactoryBean - * @see ApolloDataSourceFactoryBean - */ -public class SentinelDataSourceRegistry { - - private static HashMap> cache = new HashMap<>( - 32); - - static { - SentinelDataSourceRegistry.registerFactoryBean("file", - FileRefreshableDataSourceFactoryBean.class); - SentinelDataSourceRegistry.registerFactoryBean("zk", - ZookeeperDataSourceFactoryBean.class); - SentinelDataSourceRegistry.registerFactoryBean("nacos", - NacosDataSourceFactoryBean.class); - SentinelDataSourceRegistry.registerFactoryBean("apollo", - ApolloDataSourceFactoryBean.class); - } - - public static synchronized void registerFactoryBean(String alias, - Class clazz) { - cache.put(alias, clazz); - } - - public static Class getFactoryBean(String alias) { - return cache.get(alias); - } - - public static boolean checkFactoryBean(String alias) { - return cache.containsKey(alias); - } - -} diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/annotation/SentinelDataSource.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/annotation/SentinelDataSource.java deleted file mode 100644 index 1925fbef5..000000000 --- a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/annotation/SentinelDataSource.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.alibaba.sentinel.datasource.annotation; - -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -import com.alibaba.csp.sentinel.datasource.ReadableDataSource; - -import org.springframework.core.annotation.AliasFor; - -/** - * An annotation to inject {@link ReadableDataSource} instance - * into a Spring Bean. The Properties of DataSource bean get from config files with - * specific prefix. - * - * @author Jim - * @see ReadableDataSource - */ -@Target({ElementType.FIELD}) -@Retention(RetentionPolicy.RUNTIME) -@Documented -public @interface SentinelDataSource { - - @AliasFor("prefix") - String value() default ""; - - @AliasFor("value") - String prefix() default ""; - - String name() default ""; // spring bean name - -} diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/AbstractDataSourceProperties.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/AbstractDataSourceProperties.java new file mode 100644 index 000000000..3aa05be63 --- /dev/null +++ b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/AbstractDataSourceProperties.java @@ -0,0 +1,38 @@ +package org.springframework.cloud.alibaba.sentinel.datasource.config; + +/** + * Abstract class Using by {@link DataSourcePropertiesConfiguration} + * + * @author Jim + */ +public class AbstractDataSourceProperties { + + private String dataType = "json"; + private String converterClass; + private final String factoryBeanName; + + public AbstractDataSourceProperties(String factoryBeanName) { + this.factoryBeanName = factoryBeanName; + } + + public String getDataType() { + return dataType; + } + + public void setDataType(String dataType) { + this.dataType = dataType; + } + + public String getConverterClass() { + return converterClass; + } + + public void setConverterClass(String converterClass) { + this.converterClass = converterClass; + } + + public String getFactoryBeanName() { + return factoryBeanName; + } + +} diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/ApolloDataSourceProperties.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/ApolloDataSourceProperties.java new file mode 100644 index 000000000..28d803eaf --- /dev/null +++ b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/ApolloDataSourceProperties.java @@ -0,0 +1,44 @@ +package org.springframework.cloud.alibaba.sentinel.datasource.config; + +import org.springframework.cloud.alibaba.sentinel.datasource.factorybean.ApolloDataSourceFactoryBean; + +/** + * Apollo Properties class Using by {@link DataSourcePropertiesConfiguration} and + * {@link ApolloDataSourceFactoryBean} + * + * @author Jim + */ +public class ApolloDataSourceProperties extends AbstractDataSourceProperties { + + private String namespaceName; + private String flowRulesKey; + private String defaultFlowRuleValue; + + public ApolloDataSourceProperties() { + super(ApolloDataSourceFactoryBean.class.getName()); + } + + public String getNamespaceName() { + return namespaceName; + } + + public void setNamespaceName(String namespaceName) { + this.namespaceName = namespaceName; + } + + public String getFlowRulesKey() { + return flowRulesKey; + } + + public void setFlowRulesKey(String flowRulesKey) { + this.flowRulesKey = flowRulesKey; + } + + public String getDefaultFlowRuleValue() { + return defaultFlowRuleValue; + } + + public void setDefaultFlowRuleValue(String defaultFlowRuleValue) { + this.defaultFlowRuleValue = defaultFlowRuleValue; + } +} diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/DataSourcePropertiesConfiguration.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/DataSourcePropertiesConfiguration.java new file mode 100644 index 000000000..e6bf51bd6 --- /dev/null +++ b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/DataSourcePropertiesConfiguration.java @@ -0,0 +1,75 @@ +package org.springframework.cloud.alibaba.sentinel.datasource.config; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.util.ObjectUtils; + +/** + * Using By ConfigurationProperties. + * + * @author Jim + * @see NacosDataSourceProperties + * @see ApolloDataSourceProperties + * @see ZookeeperDataSourceProperties + * @see FileDataSourceProperties + */ +public class DataSourcePropertiesConfiguration { + + private FileDataSourceProperties file; + + private NacosDataSourceProperties nacos; + + private ZookeeperDataSourceProperties zk; + + private ApolloDataSourceProperties apollo; + + public FileDataSourceProperties getFile() { + return file; + } + + public void setFile(FileDataSourceProperties file) { + this.file = file; + } + + public NacosDataSourceProperties getNacos() { + return nacos; + } + + public void setNacos(NacosDataSourceProperties nacos) { + this.nacos = nacos; + } + + public ZookeeperDataSourceProperties getZk() { + return zk; + } + + public void setZk(ZookeeperDataSourceProperties zk) { + this.zk = zk; + } + + public ApolloDataSourceProperties getApollo() { + return apollo; + } + + public void setApollo(ApolloDataSourceProperties apollo) { + this.apollo = apollo; + } + + public List getInvalidField() { + List fieldList = new ArrayList<>(); + for (Field field : this.getClass().getDeclaredFields()) { + try { + if (!ObjectUtils.isEmpty(field.get(this))) { + fieldList.add(field.getName()); + } + } + catch (IllegalAccessException e) { + // won't happen + } + } + return fieldList; + } + +} diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/FileDataSourceProperties.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/FileDataSourceProperties.java new file mode 100644 index 000000000..ab70b8164 --- /dev/null +++ b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/FileDataSourceProperties.java @@ -0,0 +1,53 @@ +package org.springframework.cloud.alibaba.sentinel.datasource.config; + +import org.springframework.cloud.alibaba.sentinel.datasource.factorybean.FileRefreshableDataSourceFactoryBean; + +/** + * File Properties class Using by {@link DataSourcePropertiesConfiguration} and + * {@link FileRefreshableDataSourceFactoryBean} + * + * @author Jim + */ +public class FileDataSourceProperties extends AbstractDataSourceProperties { + + private String file; + private String charset = "utf-8"; + private long recommendRefreshMs = 3000L; + private int bufSize = 1024 * 1024; + + public FileDataSourceProperties() { + super(FileRefreshableDataSourceFactoryBean.class.getName()); + } + + public String getFile() { + return file; + } + + public void setFile(String file) { + this.file = file; + } + + public String getCharset() { + return charset; + } + + public void setCharset(String charset) { + this.charset = charset; + } + + public long getRecommendRefreshMs() { + return recommendRefreshMs; + } + + public void setRecommendRefreshMs(long recommendRefreshMs) { + this.recommendRefreshMs = recommendRefreshMs; + } + + public int getBufSize() { + return bufSize; + } + + public void setBufSize(int bufSize) { + this.bufSize = bufSize; + } +} diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/NacosDataSourceProperties.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/NacosDataSourceProperties.java new file mode 100644 index 000000000..3448b410f --- /dev/null +++ b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/NacosDataSourceProperties.java @@ -0,0 +1,44 @@ +package org.springframework.cloud.alibaba.sentinel.datasource.config; + +import org.springframework.cloud.alibaba.sentinel.datasource.factorybean.NacosDataSourceFactoryBean; + +/** + * Nacos Properties class Using by {@link DataSourcePropertiesConfiguration} and + * {@link NacosDataSourceFactoryBean} + * + * @author Jim + */ +public class NacosDataSourceProperties extends AbstractDataSourceProperties { + + private String serverAddr; + private String groupId; + private String dataId; + + public NacosDataSourceProperties() { + super(NacosDataSourceFactoryBean.class.getName()); + } + + public String getServerAddr() { + return serverAddr; + } + + public void setServerAddr(String serverAddr) { + this.serverAddr = serverAddr; + } + + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + public String getDataId() { + return dataId; + } + + public void setDataId(String dataId) { + this.dataId = dataId; + } +} diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/ZookeeperDataSourceProperties.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/ZookeeperDataSourceProperties.java new file mode 100644 index 000000000..a1d616674 --- /dev/null +++ b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/config/ZookeeperDataSourceProperties.java @@ -0,0 +1,56 @@ +package org.springframework.cloud.alibaba.sentinel.datasource.config; + +import org.springframework.cloud.alibaba.sentinel.datasource.factorybean.ZookeeperDataSourceFactoryBean; + +/** + * Zookeeper Properties class Using by {@link DataSourcePropertiesConfiguration} and + * {@link ZookeeperDataSourceFactoryBean} + * + * @author Jim + */ +public class ZookeeperDataSourceProperties extends AbstractDataSourceProperties { + + public ZookeeperDataSourceProperties() { + super(ZookeeperDataSourceFactoryBean.class.getName()); + } + + private String serverAddr; + + private String path; + + private String groupId; + + private String dataId; + + public String getServerAddr() { + return serverAddr; + } + + public void setServerAddr(String serverAddr) { + this.serverAddr = serverAddr; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + public String getDataId() { + return dataId; + } + + public void setDataId(String dataId) { + this.dataId = dataId; + } +} diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/converter/JsonConverter.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/converter/JsonConverter.java new file mode 100644 index 000000000..b05626a5a --- /dev/null +++ b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/converter/JsonConverter.java @@ -0,0 +1,161 @@ +package org.springframework.cloud.alibaba.sentinel.datasource.converter; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.ObjectUtils; +import org.springframework.util.StringUtils; + +import com.alibaba.csp.sentinel.datasource.Converter; +import com.alibaba.csp.sentinel.slots.block.AbstractRule; +import com.alibaba.csp.sentinel.slots.block.authority.AuthorityRule; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleManager; +import com.alibaba.csp.sentinel.slots.system.SystemRule; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Convert sentinel rules for json array Using strict mode to parse json + * + * @author Jim + * @see FlowRule + * @see DegradeRule + * @see SystemRule + * @see AuthorityRule + * @see ObjectMapper + */ +public class JsonConverter implements Converter> { + + private static final Logger logger = LoggerFactory.getLogger(JsonConverter.class); + + private final ObjectMapper objectMapper; + + public JsonConverter(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + public List convert(String source) { + List ruleList = new ArrayList<>(); + if (StringUtils.isEmpty(source)) { + logger.info( + "Sentinel JsonConverter can not convert rules because source is empty"); + return ruleList; + } + try { + List jsonArray = objectMapper.readValue(source, + new TypeReference>() { + }); + + for (Object obj : jsonArray) { + String itemJson = null; + try { + itemJson = objectMapper.writeValueAsString(obj); + } + catch (JsonProcessingException e) { + // won't be happen + } + + List rules = Arrays.asList(convertFlowRule(itemJson), + convertDegradeRule(itemJson), convertSystemRule(itemJson), + convertAuthorityRule(itemJson)); + + List convertRuleList = new ArrayList<>(); + + for (AbstractRule rule : rules) { + if (!ObjectUtils.isEmpty(rule)) { + convertRuleList.add(rule); + } + } + + if (convertRuleList.size() == 0) { + logger.warn( + "Sentinel JsonConverter can not convert {} to any rules, ignore", + itemJson); + } + else if (convertRuleList.size() > 1) { + logger.warn( + "Sentinel JsonConverter convert {} and match multi rules, ignore", + itemJson); + } + else { + ruleList.add(convertRuleList.get(0)); + } + + } + + if (jsonArray.size() != ruleList.size()) { + logger.warn( + "Sentinel JsonConverter Source list size is not equals to Target List, maybe a " + + "part of json is invalid. Source List: " + jsonArray + + ", Target List: " + ruleList); + } + } + catch (Exception e) { + logger.error("Sentinel JsonConverter convert error: " + e.getMessage()); + throw new RuntimeException( + "Sentinel JsonConverter convert error: " + e.getMessage(), e); + } + logger.info("Sentinel JsonConverter convert {} rules: {}", ruleList.size(), + ruleList); + return ruleList; + } + + private FlowRule convertFlowRule(String json) { + try { + FlowRule rule = objectMapper.readValue(json, FlowRule.class); + if (FlowRuleManager.isValidRule(rule)) { + return rule; + } + } + catch (Exception e) { + // ignore + } + return null; + } + + private DegradeRule convertDegradeRule(String json) { + try { + DegradeRule rule = objectMapper.readValue(json, DegradeRule.class); + if (DegradeRuleManager.isValidRule(rule)) { + return rule; + } + } + catch (Exception e) { + // ignore + } + return null; + } + + private SystemRule convertSystemRule(String json) { + SystemRule rule = null; + try { + rule = objectMapper.readValue(json, SystemRule.class); + } + catch (Exception e) { + // ignore + } + return rule; + } + + private AuthorityRule convertAuthorityRule(String json) { + AuthorityRule rule = null; + try { + rule = objectMapper.readValue(json, AuthorityRule.class); + } + catch (Exception e) { + // ignore + } + return rule; + } + +} diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/converter/XmlConverter.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/converter/XmlConverter.java new file mode 100644 index 000000000..c67cc6457 --- /dev/null +++ b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/converter/XmlConverter.java @@ -0,0 +1,160 @@ +package org.springframework.cloud.alibaba.sentinel.datasource.converter; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.ObjectUtils; +import org.springframework.util.StringUtils; + +import com.alibaba.csp.sentinel.datasource.Converter; +import com.alibaba.csp.sentinel.slots.block.AbstractRule; +import com.alibaba.csp.sentinel.slots.block.authority.AuthorityRule; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleManager; +import com.alibaba.csp.sentinel.slots.system.SystemRule; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; + +/** + * Convert sentinel rules for xml array Using strict mode to parse xml + * + * @author Jim + * @see FlowRule + * @see DegradeRule + * @see SystemRule + * @see AuthorityRule + * @see XmlMapper + */ +public class XmlConverter implements Converter> { + + private static final Logger logger = LoggerFactory.getLogger(XmlConverter.class); + + private final XmlMapper xmlMapper; + + public XmlConverter(XmlMapper xmlMapper) { + this.xmlMapper = xmlMapper; + } + + @Override + public List convert(String source) { + List ruleList = new ArrayList<>(); + if (StringUtils.isEmpty(source)) { + logger.info( + "Sentinel XmlConverter can not convert rules because source is empty"); + return ruleList; + } + try { + List xmlArray = xmlMapper.readValue(source, + new TypeReference>() { + }); + + for (Object obj : xmlArray) { + String itemXml = null; + try { + itemXml = xmlMapper.writeValueAsString(obj); + } + catch (JsonProcessingException e) { + // won't be happen + } + + List rules = Arrays.asList(convertFlowRule(itemXml), + convertDegradeRule(itemXml), convertSystemRule(itemXml), + convertAuthorityRule(itemXml)); + + List convertRuleList = new ArrayList<>(); + + for (AbstractRule rule : rules) { + if (!ObjectUtils.isEmpty(rule)) { + convertRuleList.add(rule); + } + } + + if (convertRuleList.size() == 0) { + logger.warn( + "Sentinel XmlConverter can not convert {} to any rules, ignore", + itemXml); + } + else if (convertRuleList.size() > 1) { + logger.warn( + "Sentinel XmlConverter convert {} and match multi rules, ignore", + itemXml); + } + else { + ruleList.add(convertRuleList.get(0)); + } + } + + if (xmlArray.size() != ruleList.size()) { + logger.warn( + "Sentinel XmlConverter Source list size is not equals to Target List, maybe a " + + "part of xml is invalid. Source List: " + xmlArray + + ", Target List: " + ruleList); + } + } + catch (Exception e) { + logger.error("Sentinel XmlConverter convert error: " + e.getMessage()); + throw new RuntimeException( + "Sentinel XmlConverter convert error: " + e.getMessage(), e); + } + logger.info("Sentinel XmlConverter convert {} rules: {}", ruleList.size(), + ruleList); + return ruleList; + } + + private FlowRule convertFlowRule(String xml) { + try { + FlowRule rule = xmlMapper.readValue(xml, FlowRule.class); + if (FlowRuleManager.isValidRule(rule)) { + return rule; + } + } + catch (Exception e) { + // ignore + } + return null; + } + + private DegradeRule convertDegradeRule(String xml) { + try { + DegradeRule rule = xmlMapper.readValue(xml, DegradeRule.class); + if (DegradeRuleManager.isValidRule(rule)) { + return rule; + } + } + catch (Exception e) { + // ignore + } + return null; + } + + private SystemRule convertSystemRule(String xml) { + SystemRule rule = null; + try { + rule = xmlMapper.readValue(xml, SystemRule.class); + } + catch (Exception e) { + // ignore + } + return rule; + } + + private AuthorityRule convertAuthorityRule(String xml) { + AuthorityRule rule = null; + try { + rule = xmlMapper.readValue(xml, AuthorityRule.class); + } + catch (Exception e) { + // ignore + } + return rule; + } + +} diff --git a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/util/PropertySourcesUtils.java b/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/util/PropertySourcesUtils.java deleted file mode 100644 index fde8fc44f..000000000 --- a/spring-cloud-alibaba-sentinel-datasource/src/main/java/org/springframework/cloud/alibaba/sentinel/datasource/util/PropertySourcesUtils.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (C) 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.alibaba.sentinel.datasource.util; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Properties; - -import org.springframework.core.env.EnumerablePropertySource; -import org.springframework.core.env.PropertySource; -import org.springframework.core.env.PropertySources; - -/** - * {@link PropertySources} Utilities - * - * @author Mercy - */ -public abstract class PropertySourcesUtils { - - /** - * Get Sub {@link Properties} - * - * @param propertySources {@link PropertySource} Iterable - * @param prefix the prefix of property name - * @return Map - * @see Properties - */ - public static Map getSubProperties(Iterable> propertySources, String prefix) { - - Map subProperties = new LinkedHashMap(); - - String normalizedPrefix = normalizePrefix(prefix); - - for (PropertySource source : propertySources) { - if (source instanceof EnumerablePropertySource) { - for (String name : ((EnumerablePropertySource)source).getPropertyNames()) { - if (!subProperties.containsKey(name) && name.startsWith(normalizedPrefix)) { - String subName = name.substring(normalizedPrefix.length()); - if (!subProperties.containsKey(subName)) { // take first one - Object value = source.getProperty(name); - subProperties.put(subName, value); - } - } - } - } - } - - return subProperties; - - } - - /** - * Normalize the prefix - * - * @param prefix the prefix - * @return the prefix - */ - public static String normalizePrefix(String prefix) { - return prefix.endsWith(".") ? prefix : prefix + "."; - } -} diff --git a/spring-cloud-alibaba-sentinel/pom.xml b/spring-cloud-alibaba-sentinel/pom.xml index a06d6689c..31824be27 100644 --- a/spring-cloud-alibaba-sentinel/pom.xml +++ b/spring-cloud-alibaba-sentinel/pom.xml @@ -40,6 +40,12 @@ spring-cloud-alibaba-sentinel-datasource + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + provided + + diff --git a/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/SentinelProperties.java b/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/SentinelProperties.java index 49460a984..feac8acc5 100644 --- a/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/SentinelProperties.java +++ b/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/SentinelProperties.java @@ -17,256 +17,285 @@ package org.springframework.cloud.alibaba.sentinel; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.NestedConfigurationProperty; +import org.springframework.cloud.alibaba.sentinel.datasource.config.DataSourcePropertiesConfiguration; import org.springframework.core.Ordered; +import com.alibaba.csp.sentinel.config.SentinelConfig; +import com.alibaba.csp.sentinel.transport.config.TransportConfig; + /** * @author xiaojing * @author hengyunabc + * @author jiashuai.xie + * @author Jim */ @ConfigurationProperties(prefix = SentinelConstants.PROPERTY_PREFIX) public class SentinelProperties { - /** - * 是否提前初始化心跳连接 - */ - private boolean eager = false; - - /** - * Enable sentinel auto configure, the default value is true - */ - private boolean enabled = true; - - /** - * 字符编码集 - */ - private String charset = "UTF-8"; - - /** - * 通信相关配置 - */ - @NestedConfigurationProperty - private Transport transport = new Transport(); - - /** - * 监控数据相关配置 - */ - @NestedConfigurationProperty - private Metric metric = new Metric(); - - /** - * web 相关配置 - */ - @NestedConfigurationProperty - private Servlet servlet = new Servlet(); - - /** - * 限流相关 - */ - @NestedConfigurationProperty - private Filter filter = new Filter(); - - @NestedConfigurationProperty - private Flow flow = new Flow(); - - public boolean isEager() { - return eager; - } - - public void setEager(boolean eager) { - this.eager = eager; - } - - public Flow getFlow() { - return flow; - } - - public void setFlow(Flow flow) { - this.flow = flow; - } - - public String getCharset() { - return charset; - } - - public void setCharset(String charset) { - this.charset = charset; - } - - public Transport getTransport() { - return transport; - } - - public void setTransport(Transport transport) { - this.transport = transport; - } - - public Metric getMetric() { - return metric; - } - - public void setMetric(Metric metric) { - this.metric = metric; - } - - public Servlet getServlet() { - return servlet; - } - - public void setServlet(Servlet servlet) { - this.servlet = servlet; - } - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public Filter getFilter() { - return filter; - } - - public void setFilter(Filter filter) { - this.filter = filter; - } - - public static class Flow { - - /** - * 限流冷启动因子 - */ - private String coldFactor = "3"; - - public String getColdFactor() { - return coldFactor; - } - - public void setColdFactor(String coldFactor) { - this.coldFactor = coldFactor; - } + /** + * earlier initialize heart-beat when the spring container starts when the + * transport dependency is on classpath ,the configuration is effective + */ + private boolean eager = false; + + /** + * enable sentinel auto configure, the default value is true + */ + private boolean enabled = true; + + /** + * charset when sentinel write or search metric file {@link SentinelConfig#CHARSET} + */ + private String charset = "UTF-8"; + + /** + * configurations about datasource, like 'nacos', 'apollo', 'file', 'zookeeper' + */ + private Map datasource = new TreeMap<>( + String.CASE_INSENSITIVE_ORDER); + + /** + * transport configuration about dashboard and client + */ + @NestedConfigurationProperty + private Transport transport = new Transport(); + + /** + * metric configuration about resource + */ + @NestedConfigurationProperty + private Metric metric = new Metric(); + + /** + * web servlet configuration when the application is web ,the configuration is + * effective + */ + @NestedConfigurationProperty + private Servlet servlet = new Servlet(); + + /** + * sentinel filter when the application is web ,the configuration is effective + * + */ + @NestedConfigurationProperty + private Filter filter = new Filter(); + + /** + * flow configuration + */ + @NestedConfigurationProperty + private Flow flow = new Flow(); + + public boolean isEager() { + return eager; + } + + public void setEager(boolean eager) { + this.eager = eager; + } + + public Flow getFlow() { + return flow; + } + + public void setFlow(Flow flow) { + this.flow = flow; + } + + public String getCharset() { + return charset; + } + + public void setCharset(String charset) { + this.charset = charset; + } + + public Transport getTransport() { + return transport; + } + + public void setTransport(Transport transport) { + this.transport = transport; + } + + public Metric getMetric() { + return metric; + } + + public void setMetric(Metric metric) { + this.metric = metric; + } + + public Servlet getServlet() { + return servlet; + } + + public void setServlet(Servlet servlet) { + this.servlet = servlet; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public Filter getFilter() { + return filter; + } + + public void setFilter(Filter filter) { + this.filter = filter; + } + + public Map getDatasource() { + return datasource; + } + + public void setDatasource(Map datasource) { + this.datasource = datasource; + } + + public static class Flow { + + /** + * the cold factor {@link SentinelConfig#COLD_FACTOR} + */ + private String coldFactor = "3"; + + public String getColdFactor() { + return coldFactor; + } + + public void setColdFactor(String coldFactor) { + this.coldFactor = coldFactor; + } - } - - public static class Servlet { + } + + public static class Servlet { - /** - * url 限流后的处理页面 - */ - private String blockPage; + /** + * The process page when the flow control is triggered + */ + private String blockPage; - public String getBlockPage() { - return blockPage; - } + public String getBlockPage() { + return blockPage; + } - public void setBlockPage(String blockPage) { - this.blockPage = blockPage; - } - } + public void setBlockPage(String blockPage) { + this.blockPage = blockPage; + } + } - public static class Metric { + public static class Metric { - /** - * 监控数据写磁盘时单个文件的大小 - */ - private String fileSingleSize; + /** + * the metric file size {@link SentinelConfig#SINGLE_METRIC_FILE_SIZE} + */ + private String fileSingleSize; - /** - * 监控数据在磁盘上的总数量 - */ - private String fileTotalCount; + /** + * the total metric file count {@link SentinelConfig#TOTAL_METRIC_FILE_COUNT} + */ + private String fileTotalCount; - public String getFileSingleSize() { - return fileSingleSize; - } + public String getFileSingleSize() { + return fileSingleSize; + } - public void setFileSingleSize(String fileSingleSize) { - this.fileSingleSize = fileSingleSize; - } + public void setFileSingleSize(String fileSingleSize) { + this.fileSingleSize = fileSingleSize; + } - public String getFileTotalCount() { - return fileTotalCount; - } + public String getFileTotalCount() { + return fileTotalCount; + } - public void setFileTotalCount(String fileTotalCount) { - this.fileTotalCount = fileTotalCount; - } - } + public void setFileTotalCount(String fileTotalCount) { + this.fileTotalCount = fileTotalCount; + } + } - public static class Transport { + public static class Transport { - /** - * sentinel api port,default value is 8721 - */ - private String port = "8721"; + /** + * sentinel api port,default value is 8721 {@link TransportConfig#SERVER_PORT} + */ + private String port = "8721"; - /** - * Sentinel dashboard address, won't try to connect dashboard when address is - * empty - */ - private String dashboard = ""; + /** + * sentinel dashboard address, won't try to connect dashboard when address is + * empty {@link TransportConfig#CONSOLE_SERVER} + */ + private String dashboard = ""; - /** - * 客户端和DashBord心跳发送时间 - */ - private String heartbeatIntervalMs; + /** + * send heartbeat interval millisecond + * {@link TransportConfig#HEARTBEAT_INTERVAL_MS} + */ + private String heartbeatIntervalMs; - public String getHeartbeatIntervalMs() { - return heartbeatIntervalMs; - } + public String getHeartbeatIntervalMs() { + return heartbeatIntervalMs; + } - public void setHeartbeatIntervalMs(String heartbeatIntervalMs) { - this.heartbeatIntervalMs = heartbeatIntervalMs; - } + public void setHeartbeatIntervalMs(String heartbeatIntervalMs) { + this.heartbeatIntervalMs = heartbeatIntervalMs; + } - public String getPort() { - return port; - } + public String getPort() { + return port; + } - public void setPort(String port) { - this.port = port; - } + public void setPort(String port) { + this.port = port; + } - public String getDashboard() { - return dashboard; - } + public String getDashboard() { + return dashboard; + } - public void setDashboard(String dashboard) { - this.dashboard = dashboard; - } + public void setDashboard(String dashboard) { + this.dashboard = dashboard; + } - } + } - public static class Filter { + public static class Filter { - /** - * Sentinel filter chain order. - */ - private int order = Ordered.HIGHEST_PRECEDENCE; + /** + * sentinel filter chain order. + */ + private int order = Ordered.HIGHEST_PRECEDENCE; - /** - * URL pattern for sentinel filter,default is /* - */ - private List urlPatterns; - - public int getOrder() { - return this.order; - } - - public void setOrder(int order) { - this.order = order; - } - - public List getUrlPatterns() { - return urlPatterns; - } - - public void setUrlPatterns(List urlPatterns) { - this.urlPatterns = urlPatterns; - } - } + /** + * URL pattern for sentinel filter,default is /* + */ + private List urlPatterns; + + public int getOrder() { + return this.order; + } + + public void setOrder(int order) { + this.order = order; + } + + public List getUrlPatterns() { + return urlPatterns; + } + + public void setUrlPatterns(List urlPatterns) { + this.urlPatterns = urlPatterns; + } + } } diff --git a/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelAutoConfiguration.java b/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelAutoConfiguration.java index 9112781b4..ea74c449b 100644 --- a/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelAutoConfiguration.java +++ b/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelAutoConfiguration.java @@ -18,125 +18,155 @@ package org.springframework.cloud.alibaba.sentinel.custom; import javax.annotation.PostConstruct; -import com.alibaba.csp.sentinel.adapter.servlet.callback.UrlBlockHandler; -import com.alibaba.csp.sentinel.adapter.servlet.callback.UrlCleaner; -import com.alibaba.csp.sentinel.adapter.servlet.callback.WebCallbackManager; -import com.alibaba.csp.sentinel.adapter.servlet.config.WebServletConfig; -import com.alibaba.csp.sentinel.annotation.aspectj.SentinelResourceAspect; -import com.alibaba.csp.sentinel.config.SentinelConfig; -import com.alibaba.csp.sentinel.init.InitExecutor; -import com.alibaba.csp.sentinel.transport.config.TransportConfig; -import com.alibaba.csp.sentinel.util.AppNameUtil; - import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.cloud.alibaba.sentinel.SentinelProperties; -import org.springframework.cloud.alibaba.sentinel.datasource.SentinelDataSourcePostProcessor; +import org.springframework.cloud.alibaba.sentinel.datasource.converter.JsonConverter; +import org.springframework.cloud.alibaba.sentinel.datasource.converter.XmlConverter; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.util.StringUtils; +import com.alibaba.csp.sentinel.adapter.servlet.callback.UrlBlockHandler; +import com.alibaba.csp.sentinel.adapter.servlet.callback.UrlCleaner; +import com.alibaba.csp.sentinel.adapter.servlet.callback.WebCallbackManager; +import com.alibaba.csp.sentinel.adapter.servlet.config.WebServletConfig; +import com.alibaba.csp.sentinel.annotation.aspectj.SentinelResourceAspect; +import com.alibaba.csp.sentinel.config.SentinelConfig; +import com.alibaba.csp.sentinel.init.InitExecutor; +import com.alibaba.csp.sentinel.transport.config.TransportConfig; +import com.alibaba.csp.sentinel.util.AppNameUtil; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; + /** * @author xiaojing + * @author Jim */ @Configuration @ConditionalOnProperty(name = "spring.cloud.sentinel.enabled", matchIfMissing = true) @EnableConfigurationProperties(SentinelProperties.class) public class SentinelAutoConfiguration { - @Value("${project.name:${spring.application.name:}}") - private String projectName; - - @Autowired - private SentinelProperties properties; - - @Autowired(required = false) - private UrlCleaner urlCleaner; - - @Autowired(required = false) - private UrlBlockHandler urlBlockHandler; - - @PostConstruct - private void init() { - if (StringUtils.isEmpty(System.getProperty(AppNameUtil.APP_NAME)) - && StringUtils.hasText(projectName)) { - System.setProperty(AppNameUtil.APP_NAME, projectName); - } - if (StringUtils.isEmpty(System.getProperty(TransportConfig.SERVER_PORT)) - && StringUtils.hasText(properties.getTransport().getPort())) { - System.setProperty(TransportConfig.SERVER_PORT, - properties.getTransport().getPort()); - } - if (StringUtils.isEmpty(System.getProperty(TransportConfig.CONSOLE_SERVER)) - && StringUtils.hasText(properties.getTransport().getDashboard())) { - System.setProperty(TransportConfig.CONSOLE_SERVER, - properties.getTransport().getDashboard()); - } - if (StringUtils.isEmpty(System.getProperty(TransportConfig.HEARTBEAT_INTERVAL_MS)) - && StringUtils - .hasText(properties.getTransport().getHeartbeatIntervalMs())) { - System.setProperty(TransportConfig.HEARTBEAT_INTERVAL_MS, - properties.getTransport().getHeartbeatIntervalMs()); - } - if (StringUtils.isEmpty(System.getProperty(SentinelConfig.CHARSET)) - && StringUtils.hasText(properties.getCharset())) { - System.setProperty(SentinelConfig.CHARSET, properties.getCharset()); - } - if (StringUtils - .isEmpty(System.getProperty(SentinelConfig.SINGLE_METRIC_FILE_SIZE)) - && StringUtils.hasText(properties.getMetric().getFileSingleSize())) { - System.setProperty(SentinelConfig.SINGLE_METRIC_FILE_SIZE, - properties.getMetric().getFileSingleSize()); - } - if (StringUtils - .isEmpty(System.getProperty(SentinelConfig.TOTAL_METRIC_FILE_COUNT)) - && StringUtils.hasText(properties.getMetric().getFileTotalCount())) { - System.setProperty(SentinelConfig.TOTAL_METRIC_FILE_COUNT, - properties.getMetric().getFileTotalCount()); - } - if (StringUtils.isEmpty(System.getProperty(SentinelConfig.COLD_FACTOR)) - && StringUtils.hasText(properties.getFlow().getColdFactor())) { - System.setProperty(SentinelConfig.COLD_FACTOR, - properties.getFlow().getColdFactor()); - } - - if (StringUtils.hasText(properties.getServlet().getBlockPage())) { - WebServletConfig.setBlockPage(properties.getServlet().getBlockPage()); - } - if (urlBlockHandler != null) { - WebCallbackManager.setUrlBlockHandler(urlBlockHandler); - } - if (urlCleaner != null) { - WebCallbackManager.setUrlCleaner(urlCleaner); - } - - // earlier initialize - if (properties.isEager()) { - InitExecutor.doInit(); - } - } - - @Bean - @ConditionalOnMissingBean - public SentinelResourceAspect sentinelResourceAspect() { - return new SentinelResourceAspect(); - } - - @Bean - @ConditionalOnMissingBean - @ConditionalOnClass(name = "org.springframework.web.client.RestTemplate") - public SentinelBeanPostProcessor sentinelBeanPostProcessor() { - return new SentinelBeanPostProcessor(); - } - - @Bean - @ConditionalOnMissingBean - public SentinelDataSourcePostProcessor sentinelDataSourcePostProcessor() { - return new SentinelDataSourcePostProcessor(); - } + @Value("${project.name:${spring.application.name:}}") + private String projectName; + + @Autowired + private SentinelProperties properties; + + @Autowired(required = false) + private UrlCleaner urlCleaner; + + @Autowired(required = false) + private UrlBlockHandler urlBlockHandler; + + @PostConstruct + private void init() { + if (StringUtils.isEmpty(System.getProperty(AppNameUtil.APP_NAME)) + && StringUtils.hasText(projectName)) { + System.setProperty(AppNameUtil.APP_NAME, projectName); + } + if (StringUtils.isEmpty(System.getProperty(TransportConfig.SERVER_PORT)) + && StringUtils.hasText(properties.getTransport().getPort())) { + System.setProperty(TransportConfig.SERVER_PORT, + properties.getTransport().getPort()); + } + if (StringUtils.isEmpty(System.getProperty(TransportConfig.CONSOLE_SERVER)) + && StringUtils.hasText(properties.getTransport().getDashboard())) { + System.setProperty(TransportConfig.CONSOLE_SERVER, + properties.getTransport().getDashboard()); + } + if (StringUtils.isEmpty(System.getProperty(TransportConfig.HEARTBEAT_INTERVAL_MS)) + && StringUtils + .hasText(properties.getTransport().getHeartbeatIntervalMs())) { + System.setProperty(TransportConfig.HEARTBEAT_INTERVAL_MS, + properties.getTransport().getHeartbeatIntervalMs()); + } + if (StringUtils.isEmpty(System.getProperty(SentinelConfig.CHARSET)) + && StringUtils.hasText(properties.getCharset())) { + System.setProperty(SentinelConfig.CHARSET, properties.getCharset()); + } + if (StringUtils + .isEmpty(System.getProperty(SentinelConfig.SINGLE_METRIC_FILE_SIZE)) + && StringUtils.hasText(properties.getMetric().getFileSingleSize())) { + System.setProperty(SentinelConfig.SINGLE_METRIC_FILE_SIZE, + properties.getMetric().getFileSingleSize()); + } + if (StringUtils + .isEmpty(System.getProperty(SentinelConfig.TOTAL_METRIC_FILE_COUNT)) + && StringUtils.hasText(properties.getMetric().getFileTotalCount())) { + System.setProperty(SentinelConfig.TOTAL_METRIC_FILE_COUNT, + properties.getMetric().getFileTotalCount()); + } + if (StringUtils.isEmpty(System.getProperty(SentinelConfig.COLD_FACTOR)) + && StringUtils.hasText(properties.getFlow().getColdFactor())) { + System.setProperty(SentinelConfig.COLD_FACTOR, + properties.getFlow().getColdFactor()); + } + + if (StringUtils.hasText(properties.getServlet().getBlockPage())) { + WebServletConfig.setBlockPage(properties.getServlet().getBlockPage()); + } + if (urlBlockHandler != null) { + WebCallbackManager.setUrlBlockHandler(urlBlockHandler); + } + if (urlCleaner != null) { + WebCallbackManager.setUrlCleaner(urlCleaner); + } + + // earlier initialize + if (properties.isEager()) { + InitExecutor.doInit(); + } + } + + @Bean + @ConditionalOnMissingBean + public SentinelResourceAspect sentinelResourceAspect() { + return new SentinelResourceAspect(); + } + + @Bean + @ConditionalOnMissingBean + @ConditionalOnClass(name = "org.springframework.web.client.RestTemplate") + public SentinelBeanPostProcessor sentinelBeanPostProcessor() { + return new SentinelBeanPostProcessor(); + } + + @Bean + public SentinelDataSourceHandler sentinelDataSourceHandler() { + return new SentinelDataSourceHandler(); + } + + @Bean("sentinel-json-converter") + public JsonConverter jsonConverter( + @Qualifier("sentinel-object-mapper") ObjectMapper objectMapper) { + return new JsonConverter(objectMapper); + } + + @Bean("sentinel-object-mapper") + public ObjectMapper objectMapper() { + return new ObjectMapper(); + } + + @ConditionalOnClass(XmlMapper.class) + protected static class SentinelXmlConfiguration { + @Bean("sentinel-xml-converter") + public XmlConverter xmlConverter( + @Qualifier("sentinel-xml-mapper") XmlMapper xmlMapper) { + return new XmlConverter(xmlMapper); + } + + @Bean("sentinel-xml-mapper") + public XmlMapper xmlMapper() { + return new XmlMapper(); + } + } } diff --git a/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelBeanPostProcessor.java b/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelBeanPostProcessor.java index 609cc8d24..fdad96641 100644 --- a/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelBeanPostProcessor.java +++ b/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelBeanPostProcessor.java @@ -34,7 +34,7 @@ import org.springframework.web.client.RestTemplate; /** * PostProcessor handle @SentinelProtect Annotation, add interceptor for RestTemplate * - * @author fangjian + * @author Jim * @see SentinelProtect * @see SentinelProtectInterceptor */ diff --git a/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelDataSourceHandler.java b/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelDataSourceHandler.java new file mode 100644 index 000000000..4121d1989 --- /dev/null +++ b/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/custom/SentinelDataSourceHandler.java @@ -0,0 +1,353 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.alibaba.sentinel.custom; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.support.DefaultListableBeanFactory; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.cloud.alibaba.sentinel.SentinelProperties; +import org.springframework.cloud.alibaba.sentinel.datasource.config.AbstractDataSourceProperties; +import org.springframework.cloud.alibaba.sentinel.datasource.config.DataSourcePropertiesConfiguration; +import org.springframework.cloud.alibaba.sentinel.datasource.converter.JsonConverter; +import org.springframework.cloud.alibaba.sentinel.datasource.converter.XmlConverter; +import org.springframework.context.event.EventListener; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ReflectionUtils; +import org.springframework.util.ResourceUtils; +import org.springframework.util.StringUtils; + +import com.alibaba.csp.sentinel.datasource.ReadableDataSource; +import com.alibaba.csp.sentinel.property.SentinelProperty; +import com.alibaba.csp.sentinel.slots.block.AbstractRule; +import com.alibaba.csp.sentinel.slots.block.authority.AuthorityRule; +import com.alibaba.csp.sentinel.slots.block.authority.AuthorityRuleManager; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleManager; +import com.alibaba.csp.sentinel.slots.system.SystemRule; +import com.alibaba.csp.sentinel.slots.system.SystemRuleManager; + +/** + * Sentinel {@link ReadableDataSource} Handler Handle the configurations of + * 'spring.cloud.sentinel.datasource' + * + * @author Jim + * @see SentinelProperties#datasource + * @see JsonConverter + * @see XmlConverter + */ +public class SentinelDataSourceHandler { + + private static final Logger logger = LoggerFactory + .getLogger(SentinelDataSourceHandler.class); + + private List dataTypeList = Arrays.asList("json", "xml"); + + private List> rulesList = Arrays.asList(FlowRule.class, + DegradeRule.class, SystemRule.class, AuthorityRule.class); + + private List dataSourceBeanNameList = Collections + .synchronizedList(new ArrayList()); + + private final String DATATYPE_FIELD = "dataType"; + private final String CUSTOM_DATATYPE = "custom"; + private final String CONVERTERCLASS_FIELD = "converterClass"; + + @Autowired + private SentinelProperties sentinelProperties; + + @EventListener(classes = ApplicationReadyEvent.class) + public void buildDataSource(ApplicationReadyEvent event) throws Exception { + + DefaultListableBeanFactory beanFactory = (DefaultListableBeanFactory) event + .getApplicationContext().getAutowireCapableBeanFactory(); + + for (Map.Entry entry : sentinelProperties + .getDatasource().entrySet()) { + String dataSourceName = entry.getKey(); + DataSourcePropertiesConfiguration dataSourceProperties = entry.getValue(); + + if (dataSourceProperties.getInvalidField().size() != 1) { + logger.error("[Sentinel Starter] DataSource " + dataSourceName + + " multi datasource active and won't loaded: " + + dataSourceProperties.getInvalidField()); + return; + } + + if (dataSourceProperties.getFile() != null) { + try { + dataSourceProperties.getFile() + .setFile(ResourceUtils + .getFile(StringUtils.trimAllWhitespace( + dataSourceProperties.getFile().getFile())) + .getAbsolutePath()); + } + catch (IOException e) { + logger.error("[Sentinel Starter] DataSource " + dataSourceName + + " handle file error: " + e.getMessage()); + throw new RuntimeException("[Sentinel Starter] DataSource " + + dataSourceName + " handle file error: " + e.getMessage(), + e); + } + registerBean(beanFactory, dataSourceProperties.getFile(), + dataSourceName + "-sentinel-file-datasource"); + } + + if (dataSourceProperties.getNacos() != null) { + registerBean(beanFactory, dataSourceProperties.getNacos(), + dataSourceName + "-sentinel-nacos-datasource"); + } + + if (dataSourceProperties.getApollo() != null) { + registerBean(beanFactory, dataSourceProperties.getApollo(), + dataSourceName + "-sentinel-apollo-datasource"); + } + + if (dataSourceProperties.getZk() != null) { + registerBean(beanFactory, dataSourceProperties.getZk(), + dataSourceName + "-sentinel-zk-datasource"); + } + + } + + for (String beanName : dataSourceBeanNameList) { + ReadableDataSource dataSource = beanFactory.getBean(beanName, + ReadableDataSource.class); + Object ruleConfig; + try { + logger.info("[Sentinel Starter] DataSource " + beanName + + " start to loadConfig"); + ruleConfig = dataSource.loadConfig(); + } + catch (Exception e) { + logger.error("[Sentinel Starter] DataSource " + beanName + + " loadConfig error: " + e.getMessage(), e); + return; + } + SentinelProperty sentinelProperty = dataSource.getProperty(); + Class ruleType = getAndCheckRuleType(ruleConfig, beanName); + if (ruleType != null) { + if (ruleType == FlowRule.class) { + FlowRuleManager.register2Property(sentinelProperty); + } + else if (ruleType == DegradeRule.class) { + DegradeRuleManager.register2Property(sentinelProperty); + } + else if (ruleType == SystemRule.class) { + SystemRuleManager.register2Property(sentinelProperty); + } + else { + AuthorityRuleManager.register2Property(sentinelProperty); + } + } + } + } + + private void registerBean(DefaultListableBeanFactory beanFactory, + final AbstractDataSourceProperties dataSourceProperties, + String dataSourceName) { + + Map propertyMap = new HashMap<>(); + for (Field field : dataSourceProperties.getClass().getDeclaredFields()) { + try { + field.setAccessible(true); + Object fieldVal = field.get(dataSourceProperties); + if (fieldVal != null) { + propertyMap.put(field.getName(), fieldVal); + } + } + catch (IllegalAccessException e) { + logger.error("[Sentinel Starter] DataSource " + dataSourceName + + " field: " + field.getName() + " invoke error"); + throw new RuntimeException("[Sentinel Starter] DataSource " + + dataSourceName + " field: " + field.getName() + " invoke error", + e); + } + } + propertyMap.put(CONVERTERCLASS_FIELD, dataSourceProperties.getConverterClass()); + propertyMap.put(DATATYPE_FIELD, dataSourceProperties.getDataType()); + + BeanDefinitionBuilder builder = BeanDefinitionBuilder + .genericBeanDefinition(dataSourceProperties.getFactoryBeanName()); + + for (Map.Entry entry : propertyMap.entrySet()) { + String propertyName = entry.getKey(); + Object propertyValue = entry.getValue(); + Field field = ReflectionUtils.findField(dataSourceProperties.getClass(), + propertyName); + if (field != null) { + if (DATATYPE_FIELD.equals(propertyName)) { + String dataType = StringUtils + .trimAllWhitespace(propertyValue.toString()); + if (CUSTOM_DATATYPE.equals(dataType)) { + try { + if (StringUtils + .isEmpty(dataSourceProperties.getConverterClass())) { + throw new RuntimeException( + "[Sentinel Starter] DataSource " + dataSourceName + + "dataType is custom, please set converter-class " + + "property"); + } + // construct custom Converter with 'converterClass' + // configuration and register + String customConvertBeanName = "sentinel-" + + dataSourceProperties.getConverterClass(); + if (!beanFactory.containsBean(customConvertBeanName)) { + beanFactory.registerBeanDefinition(customConvertBeanName, + BeanDefinitionBuilder + .genericBeanDefinition( + Class.forName(dataSourceProperties + .getConverterClass())) + .getBeanDefinition()); + } + builder.addPropertyReference("converter", + customConvertBeanName); + } + catch (ClassNotFoundException e) { + logger.error("[Sentinel Starter] DataSource " + dataSourceName + + " handle " + + dataSourceProperties.getClass().getSimpleName() + + " error, class name: " + + dataSourceProperties.getConverterClass()); + throw new RuntimeException( + "[Sentinel Starter] DataSource " + dataSourceName + + " handle " + + dataSourceProperties.getClass() + .getSimpleName() + + " error, class name: " + + dataSourceProperties.getConverterClass(), + e); + } + } + else { + if (!dataTypeList.contains(StringUtils + .trimAllWhitespace(propertyValue.toString()))) { + throw new RuntimeException("[Sentinel Starter] DataSource " + + dataSourceName + " dataType: " + propertyValue + + " is not support now. please using these types: " + + dataTypeList.toString()); + } + // converter type now support xml or json. + // The bean name of these converters wrapped by + // 'sentinel-{converterType}-converter' + builder.addPropertyReference("converter", + "sentinel-" + propertyValue.toString() + "-converter"); + } + } + else if (CONVERTERCLASS_FIELD.equals(propertyName)) { + continue; + } + else { + // wired properties + builder.addPropertyValue(propertyName, propertyValue); + } + } + } + + beanFactory.registerBeanDefinition(dataSourceName, builder.getBeanDefinition()); + // init in Spring + beanFactory.getBean(dataSourceName); + dataSourceBeanNameList.add(dataSourceName); + } + + private Class getAndCheckRuleType(Object ruleConfig, String dataSourceName) { + if (rulesList.contains(ruleConfig.getClass())) { + logger.info("[Sentinel Starter] DataSource {} load {} {}", dataSourceName, 1, + ruleConfig.getClass().getSimpleName()); + return ruleConfig.getClass(); + } + else if (ruleConfig instanceof List) { + List convertedRuleList = (List) ruleConfig; + if (CollectionUtils.isEmpty(convertedRuleList)) { + logger.warn("[Sentinel Starter] DataSource {} rule list is empty.", + dataSourceName); + return null; + } + + if (checkRuleTypeValid(convertedRuleList)) { + if (checkAllRuleTypeSame(convertedRuleList)) { + logger.info("[Sentinel Starter] DataSource {} load {} {}", + dataSourceName, convertedRuleList.size(), + convertedRuleList.get(0).getClass().getSimpleName()); + return convertedRuleList.get(0).getClass(); + } + else { + logger.warn( + "[Sentinel Starter] DataSource {} all rules are not same rule type and it will not be used. " + + "Rule List: {}", + dataSourceName, convertedRuleList.toString()); + } + } + else { + List classList = new ArrayList<>(); + for (Object rule : convertedRuleList) { + classList.add(rule.getClass()); + } + logger.error("[Sentinel Starter] DataSource " + dataSourceName + + " rule class is invalid. Class List: " + classList); + throw new RuntimeException( + "[Sentinel Starter] DataSource " + dataSourceName + + " rule class is invalid. Class List: " + classList); + } + } + else { + logger.error("[Sentinel Starter] DataSource " + dataSourceName + + " rule class is invalid. Class: " + ruleConfig.getClass()); + throw new RuntimeException("[Sentinel Starter] DataSource " + dataSourceName + + " rule class is invalid. Class: " + ruleConfig.getClass()); + } + return null; + } + + private boolean checkRuleTypeValid(List convertedRuleList) { + int matchCount = 0; + for (Object rule : convertedRuleList) { + if (rulesList.contains(rule.getClass())) { + matchCount++; + } + } + return matchCount == convertedRuleList.size(); + } + + private boolean checkAllRuleTypeSame(List convertedRuleList) { + int matchCount = 0; + for(Object rule : convertedRuleList) { + if(rulesList.contains(rule.getClass()) && rule.getClass() == convertedRuleList.get(0).getClass()) { + matchCount ++; + } + } + return matchCount == convertedRuleList.size(); + } + + public List getDataSourceBeanNameList() { + return dataSourceBeanNameList; + } + +} diff --git a/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/endpoint/SentinelEndpoint.java b/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/endpoint/SentinelEndpoint.java index 2a71f0b5c..05262314d 100644 --- a/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/endpoint/SentinelEndpoint.java +++ b/spring-cloud-alibaba-sentinel/src/main/java/org/springframework/cloud/alibaba/sentinel/endpoint/SentinelEndpoint.java @@ -20,15 +20,19 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.actuate.endpoint.AbstractEndpoint; +import org.springframework.cloud.alibaba.sentinel.SentinelProperties; +import org.springframework.cloud.alibaba.sentinel.custom.SentinelDataSourceHandler; +import org.springframework.context.ApplicationContext; + +import com.alibaba.csp.sentinel.datasource.ReadableDataSource; import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule; import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleManager; import com.alibaba.csp.sentinel.slots.system.SystemRule; import com.alibaba.csp.sentinel.slots.system.SystemRuleManager; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.actuate.endpoint.AbstractEndpoint; -import org.springframework.cloud.alibaba.sentinel.SentinelProperties; /** * Endpoint for Sentinel, contains ans properties and rules @@ -39,6 +43,12 @@ public class SentinelEndpoint extends AbstractEndpoint> { @Autowired private SentinelProperties sentinelProperties; + @Autowired + private SentinelDataSourceHandler dataSourceHandler; + + @Autowired + private ApplicationContext applicationContext; + public SentinelEndpoint() { super("sentinel"); } @@ -54,6 +64,20 @@ public class SentinelEndpoint extends AbstractEndpoint> { result.put("FlowRules", flowRules); result.put("DegradeRules", degradeRules); result.put("SystemRules", systemRules); + result.put("datasources", new HashMap()); + + for (String dataSourceBeanName : dataSourceHandler.getDataSourceBeanNameList()) { + ReadableDataSource dataSource = applicationContext.getBean(dataSourceBeanName, + ReadableDataSource.class); + try { + ((HashMap) result.get("datasources")).put(dataSourceBeanName, + dataSource.loadConfig()); + } + catch (Exception e) { + ((HashMap) result.get("datasources")).put(dataSourceBeanName, + "load error: " + e.getMessage()); + } + } return result; } From 3263d203c2e2cc99ef595c14efd0266abd97f23a Mon Sep 17 00:00:00 2001 From: fangjian0423 Date: Tue, 27 Nov 2018 11:36:29 +0800 Subject: [PATCH 2/4] add rocketmq-binder and starter --- pom.xml | 1 + spring-cloud-alibaba-dependencies/pom.xml | 26 ++ spring-cloud-starter-alibaba/pom.xml | 1 + .../pom.xml | 20 ++ spring-cloud-stream-binder-rocketmq/pom.xml | 71 ++++ .../rocketmq/RocketMQBinderConstants.java | 42 +++ .../RocketMQMessageChannelBinder.java | 117 +++++++ .../RocketMQMessageHeaderAccessor.java | 123 +++++++ .../actuator/RocketMQBinderEndpoint.java | 58 ++++ .../RocketMQBinderHealthIndicator.java | 66 ++++ .../RocketMQBinderAutoConfiguration.java | 70 ++++ ...cketMQBinderEndpointAutoConfiguration.java | 52 +++ .../rocketmq/consuming/Acknowledgement.java | 94 ++++++ .../rocketmq/consuming/ConsumersManager.java | 122 +++++++ .../RocketMQInboundChannelAdapter.java | 314 ++++++++++++++++++ .../integration/RocketMQMessageHandler.java | 160 +++++++++ .../metrics/ConsumerGroupInstrumentation.java | 33 ++ .../metrics/ConsumerInstrumentation.java | 53 +++ .../rocketmq/metrics/Instrumentation.java | 66 ++++ .../metrics/InstrumentationManager.java | 89 +++++ .../metrics/ProducerInstrumentation.java | 53 +++ ...RocketMQBinderConfigurationProperties.java | 48 +++ .../properties/RocketMQBindingProperties.java | 44 +++ .../RocketMQConsumerProperties.java | 95 ++++++ .../RocketMQExtendedBindingProperties.java | 80 +++++ .../RocketMQProducerProperties.java | 51 +++ .../RocketMQTopicProvisioner.java | 105 ++++++ .../main/resources/META-INF/spring.binders | 1 + .../main/resources/META-INF/spring.factories | 2 + .../RocketMQAutoConfigurationTests.java | 73 ++++ 30 files changed, 2130 insertions(+) create mode 100644 spring-cloud-starter-alibaba/spring-cloud-starter-stream-rocketmq/pom.xml create mode 100644 spring-cloud-stream-binder-rocketmq/pom.xml create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQBinderConstants.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQMessageChannelBinder.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQMessageHeaderAccessor.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/actuator/RocketMQBinderEndpoint.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/actuator/RocketMQBinderHealthIndicator.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/config/RocketMQBinderAutoConfiguration.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/config/RocketMQBinderEndpointAutoConfiguration.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/consuming/Acknowledgement.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/consuming/ConsumersManager.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQInboundChannelAdapter.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQMessageHandler.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ConsumerGroupInstrumentation.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ConsumerInstrumentation.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/Instrumentation.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/InstrumentationManager.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ProducerInstrumentation.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQBinderConfigurationProperties.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQBindingProperties.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQConsumerProperties.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQExtendedBindingProperties.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQProducerProperties.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/provisioning/RocketMQTopicProvisioner.java create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/resources/META-INF/spring.binders create mode 100644 spring-cloud-stream-binder-rocketmq/src/main/resources/META-INF/spring.factories create mode 100644 spring-cloud-stream-binder-rocketmq/src/test/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQAutoConfigurationTests.java diff --git a/pom.xml b/pom.xml index 70dc9d4a6..053e6e20a 100644 --- a/pom.xml +++ b/pom.xml @@ -80,6 +80,7 @@ spring-cloud-alibaba-sentinel-datasource spring-cloud-alibaba-nacos-config spring-cloud-alibaba-nacos-discovery + spring-cloud-stream-binder-rocketmq spring-cloud-alibaba-examples spring-cloud-alibaba-test spring-cloud-alibaba-docs diff --git a/spring-cloud-alibaba-dependencies/pom.xml b/spring-cloud-alibaba-dependencies/pom.xml index c89ae4889..38c62d125 100644 --- a/spring-cloud-alibaba-dependencies/pom.xml +++ b/spring-cloud-alibaba-dependencies/pom.xml @@ -24,6 +24,8 @@ 4.0.1 1.0.0 2.16.0 + 4.3.1 + 3.2.6 @@ -115,6 +117,11 @@ sentinel-dubbo-api ${project.version} + + org.apache.rocketmq + rocketmq-client + ${rocketmq.version} + @@ -166,6 +173,11 @@ spring-cloud-alicloud-context ${project.version} + + org.springframework.cloud + spring-cloud-stream-binder-rocketmq + ${project.version} + @@ -202,6 +214,20 @@ spring-cloud-starter-alicloud-acm ${project.version} + + + org.springframework.cloud + spring-cloud-starter-stream-rocketmq + ${project.version} + + + + + io.dropwizard.metrics + metrics-core + ${metrics.core} + + diff --git a/spring-cloud-starter-alibaba/pom.xml b/spring-cloud-starter-alibaba/pom.xml index d455bf87c..09aa60bd4 100644 --- a/spring-cloud-starter-alibaba/pom.xml +++ b/spring-cloud-starter-alibaba/pom.xml @@ -17,6 +17,7 @@ spring-cloud-starter-alibaba-nacos-config spring-cloud-starter-alibaba-nacos-discovery spring-cloud-starter-alibaba-sentinel + spring-cloud-starter-stream-rocketmq diff --git a/spring-cloud-starter-alibaba/spring-cloud-starter-stream-rocketmq/pom.xml b/spring-cloud-starter-alibaba/spring-cloud-starter-stream-rocketmq/pom.xml new file mode 100644 index 000000000..d4f2eedde --- /dev/null +++ b/spring-cloud-starter-alibaba/spring-cloud-starter-stream-rocketmq/pom.xml @@ -0,0 +1,20 @@ + + 4.0.0 + + + org.springframework.cloud + spring-cloud-starter-alibaba + 0.1.1.BUILD-SNAPSHOT + + spring-cloud-starter-stream-rocketmq + Spring Cloud Starter Stream RocketMQ + + + + org.springframework.cloud + spring-cloud-stream-binder-rocketmq + + + + diff --git a/spring-cloud-stream-binder-rocketmq/pom.xml b/spring-cloud-stream-binder-rocketmq/pom.xml new file mode 100644 index 000000000..4b534d4b8 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/pom.xml @@ -0,0 +1,71 @@ + + + + + org.springframework.cloud + spring-cloud-alibaba + 0.1.1.BUILD-SNAPSHOT + + 4.0.0 + + org.springframework.cloud + spring-cloud-stream-binder-rocketmq + Spring Cloud Alibaba RocketMQ Binder + + + + + org.springframework.cloud + spring-cloud-stream + + + + io.dropwizard.metrics + metrics-core + + + + org.apache.rocketmq + rocketmq-client + + + + org.springframework.boot + spring-boot-configuration-processor + provided + true + + + + org.springframework.boot + spring-boot + provided + true + + + + org.springframework.boot + spring-boot-autoconfigure + provided + true + + + + org.springframework.boot + spring-boot-actuator + provided + true + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQBinderConstants.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQBinderConstants.java new file mode 100644 index 000000000..cb35b5a65 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQBinderConstants.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq; + +/** + * @author Jim + */ +public interface RocketMQBinderConstants { + + /** + * Header key + */ + String ORIGINAL_ROCKET_MESSAGE = "ORIGINAL_ROCKETMQ_MESSAGE"; + + String ROCKET_FLAG = "ROCKETMQ_FLAG"; + + String ROCKET_SEND_RESULT = "ROCKETMQ_SEND_RESULT"; + + String ACKNOWLEDGEMENT_KEY = "ACKNOWLEDGEMENT"; + + /** + * Instrumentation key + */ + String LASTSEND_TIMESTAMP = "lastSend.timestamp"; + + String ENDPOINT_ID = "rocketmq_binder"; + +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQMessageChannelBinder.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQMessageChannelBinder.java new file mode 100644 index 000000000..4bed86afc --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQMessageChannelBinder.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.cloud.stream.binder.AbstractMessageChannelBinder; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.ExtendedPropertiesBinder; +import org.springframework.cloud.stream.binder.rocketmq.consuming.ConsumersManager; +import org.springframework.cloud.stream.binder.rocketmq.integration.RocketMQInboundChannelAdapter; +import org.springframework.cloud.stream.binder.rocketmq.integration.RocketMQMessageHandler; +import org.springframework.cloud.stream.binder.rocketmq.metrics.InstrumentationManager; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQConsumerProperties; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQExtendedBindingProperties; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQProducerProperties; +import org.springframework.cloud.stream.binder.rocketmq.provisioning.RocketMQTopicProvisioner; +import org.springframework.cloud.stream.provisioning.ConsumerDestination; +import org.springframework.cloud.stream.provisioning.ProducerDestination; +import org.springframework.integration.core.MessageProducer; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; + +/** + * @author Timur Valiev + * @author Jim + */ +public class RocketMQMessageChannelBinder extends + AbstractMessageChannelBinder, + ExtendedProducerProperties, RocketMQTopicProvisioner> + implements ExtendedPropertiesBinder { + + private static final Logger logger = LoggerFactory.getLogger(RocketMQMessageChannelBinder.class); + + private final RocketMQExtendedBindingProperties extendedBindingProperties; + private final RocketMQTopicProvisioner rocketTopicProvisioner; + private final RocketMQBinderConfigurationProperties rocketBinderConfigurationProperties; + private final InstrumentationManager instrumentationManager; + private final ConsumersManager consumersManager; + + public RocketMQMessageChannelBinder(ConsumersManager consumersManager, + RocketMQExtendedBindingProperties extendedBindingProperties, + RocketMQTopicProvisioner provisioningProvider, + RocketMQBinderConfigurationProperties rocketBinderConfigurationProperties, + InstrumentationManager instrumentationManager) { + super(true, null, provisioningProvider); + this.consumersManager = consumersManager; + this.extendedBindingProperties = extendedBindingProperties; + this.rocketTopicProvisioner = provisioningProvider; + this.rocketBinderConfigurationProperties = rocketBinderConfigurationProperties; + this.instrumentationManager = instrumentationManager; + } + + @Override + protected MessageHandler createProducerMessageHandler(ProducerDestination destination, + ExtendedProducerProperties + producerProperties, + MessageChannel errorChannel) throws Exception { + if (producerProperties.getExtension().getEnabled()) { + return new RocketMQMessageHandler(destination.getName(), producerProperties.getExtension(), + rocketBinderConfigurationProperties, instrumentationManager); + } else { + throw new RuntimeException( + "Binding for channel " + destination.getName() + "has been disabled, message can't be delivered"); + } + } + + @Override + protected MessageProducer createConsumerEndpoint(ConsumerDestination destination, String group, + ExtendedConsumerProperties + consumerProperties) + throws Exception { + if (group == null || "".equals(group)) { + throw new RuntimeException("'group' must be configured for channel + " + destination.getName()); + } + + RocketMQInboundChannelAdapter rocketInboundChannelAdapter = new RocketMQInboundChannelAdapter(consumersManager, + consumerProperties, destination.getName(), group, instrumentationManager); + + ErrorInfrastructure errorInfrastructure = registerErrorInfrastructure(destination, group, + consumerProperties); + if (consumerProperties.getMaxAttempts() > 1) { + rocketInboundChannelAdapter.setRetryTemplate(buildRetryTemplate(consumerProperties)); + rocketInboundChannelAdapter.setRecoveryCallback(errorInfrastructure.getRecoverer()); + } else { + rocketInboundChannelAdapter.setErrorChannel(errorInfrastructure.getErrorChannel()); + } + + return rocketInboundChannelAdapter; + } + + @Override + public RocketMQConsumerProperties getExtendedConsumerProperties(String channelName) { + return extendedBindingProperties.getExtendedConsumerProperties(channelName); + } + + @Override + public RocketMQProducerProperties getExtendedProducerProperties(String channelName) { + return extendedBindingProperties.getExtendedProducerProperties(channelName); + } +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQMessageHeaderAccessor.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQMessageHeaderAccessor.java new file mode 100644 index 000000000..fa480681a --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQMessageHeaderAccessor.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.springframework.cloud.stream.binder.rocketmq.consuming.Acknowledgement; +import org.springframework.integration.support.MutableMessage; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.MessageHeaderAccessor; + +import static org.springframework.cloud.stream.binder.rocketmq.RocketMQBinderConstants.ACKNOWLEDGEMENT_KEY; +import static org.springframework.cloud.stream.binder.rocketmq.RocketMQBinderConstants.ORIGINAL_ROCKET_MESSAGE; +import static org.springframework.cloud.stream.binder.rocketmq.RocketMQBinderConstants.ROCKET_FLAG; +import static org.springframework.cloud.stream.binder.rocketmq.RocketMQBinderConstants.ROCKET_SEND_RESULT; + +/** + * @author Timur Valiev + * @author Jim + */ +public class RocketMQMessageHeaderAccessor extends MessageHeaderAccessor { + + public RocketMQMessageHeaderAccessor() { + super(); + } + + public RocketMQMessageHeaderAccessor(Message message) { + super(message); + } + + public Acknowledgement getAcknowledgement(Message message) { + return message.getHeaders().get(ACKNOWLEDGEMENT_KEY, Acknowledgement.class); + } + + public RocketMQMessageHeaderAccessor withAcknowledgment(Acknowledgement acknowledgment) { + setHeader(ACKNOWLEDGEMENT_KEY, acknowledgment); + return this; + } + + public String getTags() { + return getMessageHeaders().get(MessageConst.PROPERTY_TAGS) == null ? "" : (String) getMessageHeaders().get(MessageConst.PROPERTY_TAGS); + } + + public RocketMQMessageHeaderAccessor withTags(String tag) { + setHeader(MessageConst.PROPERTY_TAGS, tag); + return this; + } + + public String getKeys() { + return getMessageHeaders().get(MessageConst.PROPERTY_KEYS) == null ? "" : (String) getMessageHeaders().get(MessageConst.PROPERTY_KEYS); + } + + public RocketMQMessageHeaderAccessor withKeys(String keys) { + setHeader(MessageConst.PROPERTY_KEYS, keys); + return this; + } + + public MessageExt getRocketMessage() { + return getMessageHeaders().get(ORIGINAL_ROCKET_MESSAGE, MessageExt.class); + } + + public RocketMQMessageHeaderAccessor withRocketMessage(MessageExt message) { + setHeader(ORIGINAL_ROCKET_MESSAGE, message); + return this; + } + + public Integer getDelayTimeLevel() { + return NumberUtils.toInt((String)getMessageHeaders().get(MessageConst.PROPERTY_DELAY_TIME_LEVEL), 0); + } + + public RocketMQMessageHeaderAccessor withDelayTimeLevel(Integer delayTimeLevel) { + setHeader(MessageConst.PROPERTY_DELAY_TIME_LEVEL, delayTimeLevel); + return this; + } + + public Integer getFlag() { + return NumberUtils.toInt((String)getMessageHeaders().get(ROCKET_FLAG), 0); + } + + public RocketMQMessageHeaderAccessor withFlag(Integer delayTimeLevel) { + setHeader(ROCKET_FLAG, delayTimeLevel); + return this; + } + + public SendResult getSendResult() { + return getMessageHeaders().get(ROCKET_SEND_RESULT, SendResult.class); + } + + public static void putSendResult(MutableMessage message, SendResult sendResult) { + message.getHeaders().put(ROCKET_SEND_RESULT, sendResult); + } + + public Map getUserProperties() { + Map result = new HashMap<>(); + for (Map.Entry entry : this.toMap().entrySet()) { + if (entry.getValue() instanceof String && !MessageConst.STRING_HASH_SET.contains(entry.getKey()) && !entry + .getKey().equals(MessageHeaders.CONTENT_TYPE)) { + result.put(entry.getKey(), (String)entry.getValue()); + } + } + return result; + } +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/actuator/RocketMQBinderEndpoint.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/actuator/RocketMQBinderEndpoint.java new file mode 100644 index 000000000..756924beb --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/actuator/RocketMQBinderEndpoint.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.actuator; + +import static org.springframework.cloud.stream.binder.rocketmq.RocketMQBinderConstants.ENDPOINT_ID; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.springframework.boot.actuate.endpoint.AbstractEndpoint; + +import com.codahale.metrics.MetricRegistry; + +/** + * @author Timur Valiev + * @author Jim + */ +public class RocketMQBinderEndpoint extends AbstractEndpoint> { + + private MetricRegistry metricRegistry = new MetricRegistry(); + private Map runtime = new ConcurrentHashMap<>(); + + public RocketMQBinderEndpoint() { + super(ENDPOINT_ID); + } + + @Override + public Map invoke() { + Map result = new HashMap<>(); + result.put("metrics", metricRegistry().getMetrics()); + result.put("runtime", runtime()); + return result; + } + + public MetricRegistry metricRegistry() { + return metricRegistry; + } + + public Map runtime() { + return runtime; + } + +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/actuator/RocketMQBinderHealthIndicator.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/actuator/RocketMQBinderHealthIndicator.java new file mode 100644 index 000000000..f23463acf --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/actuator/RocketMQBinderHealthIndicator.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.actuator; + +import org.springframework.boot.actuate.health.AbstractHealthIndicator; +import org.springframework.boot.actuate.health.Health; +import org.springframework.cloud.stream.binder.rocketmq.metrics.Instrumentation; +import org.springframework.cloud.stream.binder.rocketmq.metrics.InstrumentationManager; + +/** + * @author Timur Valiev + * @author Jim + */ +public class RocketMQBinderHealthIndicator extends AbstractHealthIndicator { + + private final InstrumentationManager instrumentationManager; + + public RocketMQBinderHealthIndicator(InstrumentationManager instrumentationManager) { + this.instrumentationManager = instrumentationManager; + } + + @Override + protected void doHealthCheck(Health.Builder builder) throws Exception { + int upCount = 0, outOfServiceCount = 0; + for (Instrumentation instrumentation : instrumentationManager + .getHealthInstrumentations()) { + if (instrumentation.isUp()) { + upCount++; + } + else if (instrumentation.isOutOfService()) { + upCount++; + } + } + if (upCount == instrumentationManager.getHealthInstrumentations().size()) { + builder.up(); + return; + } + else if (outOfServiceCount == instrumentationManager.getHealthInstrumentations() + .size()) { + builder.outOfService(); + return; + } + builder.down(); + + for (Instrumentation instrumentation : instrumentationManager + .getHealthInstrumentations()) { + if (!instrumentation.isStarted()) { + builder.withException(instrumentation.getStartException()); + } + } + } +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/config/RocketMQBinderAutoConfiguration.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/config/RocketMQBinderAutoConfiguration.java new file mode 100644 index 000000000..f2bb68cff --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/config/RocketMQBinderAutoConfiguration.java @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.config; + +import org.apache.rocketmq.client.log.ClientLogger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.stream.binder.rocketmq.RocketMQMessageChannelBinder; +import org.springframework.cloud.stream.binder.rocketmq.consuming.ConsumersManager; +import org.springframework.cloud.stream.binder.rocketmq.metrics.InstrumentationManager; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQExtendedBindingProperties; +import org.springframework.cloud.stream.binder.rocketmq.provisioning.RocketMQTopicProvisioner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author Timur Valiev + * @author Jim + */ +@Configuration +@EnableConfigurationProperties({RocketMQBinderConfigurationProperties.class, RocketMQExtendedBindingProperties.class}) +public class RocketMQBinderAutoConfiguration { + + private final RocketMQExtendedBindingProperties extendedBindingProperties; + + private final RocketMQBinderConfigurationProperties rocketBinderConfigurationProperties; + + @Autowired + public RocketMQBinderAutoConfiguration(RocketMQExtendedBindingProperties extendedBindingProperties, + RocketMQBinderConfigurationProperties rocketBinderConfigurationProperties) { + this.extendedBindingProperties = extendedBindingProperties; + this.rocketBinderConfigurationProperties = rocketBinderConfigurationProperties; + System.setProperty(ClientLogger.CLIENT_LOG_LEVEL, this.rocketBinderConfigurationProperties.getLogLevel()); + } + + @Bean + public RocketMQTopicProvisioner provisioningProvider() { + return new RocketMQTopicProvisioner(); + } + + @Bean + public RocketMQMessageChannelBinder rocketMessageChannelBinder(RocketMQTopicProvisioner provisioningProvider, + InstrumentationManager instrumentationManager, + ConsumersManager consumersManager) { + RocketMQMessageChannelBinder binder = new RocketMQMessageChannelBinder(consumersManager, extendedBindingProperties, + provisioningProvider, rocketBinderConfigurationProperties, instrumentationManager); + return binder; + } + + @Bean + public ConsumersManager consumersManager(InstrumentationManager instrumentationManager) { + return new ConsumersManager(instrumentationManager, rocketBinderConfigurationProperties); + } + +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/config/RocketMQBinderEndpointAutoConfiguration.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/config/RocketMQBinderEndpointAutoConfiguration.java new file mode 100644 index 000000000..a53865096 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/config/RocketMQBinderEndpointAutoConfiguration.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.config; + +import org.springframework.boot.actuate.autoconfigure.EndpointAutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.cloud.stream.binder.rocketmq.actuator.RocketMQBinderEndpoint; +import org.springframework.cloud.stream.binder.rocketmq.actuator.RocketMQBinderHealthIndicator; +import org.springframework.cloud.stream.binder.rocketmq.metrics.InstrumentationManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author Jim + */ +@Configuration +@AutoConfigureAfter(EndpointAutoConfiguration.class) +public class RocketMQBinderEndpointAutoConfiguration { + + @Bean + public RocketMQBinderEndpoint rocketBinderEndpoint() { + return new RocketMQBinderEndpoint(); + } + + @Bean + public RocketMQBinderHealthIndicator rocketBinderHealthIndicator( + InstrumentationManager instrumentationManager) { + return new RocketMQBinderHealthIndicator(instrumentationManager); + } + + @Bean + public InstrumentationManager instrumentationManager( + RocketMQBinderEndpoint rocketBinderEndpoint) { + return new InstrumentationManager(rocketBinderEndpoint.metricRegistry(), + rocketBinderEndpoint.runtime()); + } + +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/consuming/Acknowledgement.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/consuming/Acknowledgement.java new file mode 100644 index 000000000..207dbe50d --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/consuming/Acknowledgement.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.consuming; + +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; + +/** + * @author Timur Valiev + * @author Jim + */ +public class Acknowledgement { + + /** + * for {@link ConsumeConcurrentlyContext} using + */ + private ConsumeConcurrentlyStatus consumeConcurrentlyStatus = ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + /** + * Message consume retry strategy
+ * -1,no retry,put into DLQ directly
+ * 0,broker control retry frequency
+ * >0,client control retry frequency + */ + private Integer consumeConcurrentlyDelayLevel = 0; + + /** + * for {@link ConsumeOrderlyContext} using + */ + private ConsumeOrderlyStatus consumeOrderlyStatus = ConsumeOrderlyStatus.SUCCESS; + private Long consumeOrderlySuspendCurrentQueueTimeMill = -1L; + + public Acknowledgement setConsumeConcurrentlyStatus(ConsumeConcurrentlyStatus consumeConcurrentlyStatus) { + this.consumeConcurrentlyStatus = consumeConcurrentlyStatus; + return this; + } + + public ConsumeConcurrentlyStatus getConsumeConcurrentlyStatus() { + return consumeConcurrentlyStatus; + } + + public ConsumeOrderlyStatus getConsumeOrderlyStatus() { + return consumeOrderlyStatus; + } + + public Acknowledgement setConsumeOrderlyStatus(ConsumeOrderlyStatus consumeOrderlyStatus) { + this.consumeOrderlyStatus = consumeOrderlyStatus; + return this; + } + + public Integer getConsumeConcurrentlyDelayLevel() { + return consumeConcurrentlyDelayLevel; + } + + public void setConsumeConcurrentlyDelayLevel(Integer consumeConcurrentlyDelayLevel) { + this.consumeConcurrentlyDelayLevel = consumeConcurrentlyDelayLevel; + } + + public Long getConsumeOrderlySuspendCurrentQueueTimeMill() { + return consumeOrderlySuspendCurrentQueueTimeMill; + } + + public void setConsumeOrderlySuspendCurrentQueueTimeMill(Long consumeOrderlySuspendCurrentQueueTimeMill) { + this.consumeOrderlySuspendCurrentQueueTimeMill = consumeOrderlySuspendCurrentQueueTimeMill; + } + + public static Acknowledgement buildOrderlyInstance() { + Acknowledgement acknowledgement = new Acknowledgement(); + acknowledgement.setConsumeOrderlyStatus(ConsumeOrderlyStatus.SUCCESS); + return acknowledgement; + } + + public static Acknowledgement buildConcurrentlyInstance() { + Acknowledgement acknowledgement = new Acknowledgement(); + acknowledgement.setConsumeConcurrentlyStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS); + return acknowledgement; + } + +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/consuming/ConsumersManager.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/consuming/ConsumersManager.java new file mode 100644 index 000000000..6b9445f93 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/consuming/ConsumersManager.java @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.consuming; + +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.rocketmq.metrics.ConsumerGroupInstrumentation; +import org.springframework.cloud.stream.binder.rocketmq.metrics.InstrumentationManager; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQConsumerProperties; + +/** + * @author Timur Valiev + * @author Jim + */ +public class ConsumersManager { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final Map consumerGroups = new HashMap<>(); + private final Map started = new HashMap<>(); + private final Map, ExtendedConsumerProperties> propertiesMap + = new HashMap<>(); + private final InstrumentationManager instrumentationManager; + private final RocketMQBinderConfigurationProperties rocketBinderConfigurationProperties; + + public ConsumersManager(InstrumentationManager instrumentationManager, + RocketMQBinderConfigurationProperties rocketBinderConfigurationProperties) { + this.instrumentationManager = instrumentationManager; + this.rocketBinderConfigurationProperties = rocketBinderConfigurationProperties; + } + + public synchronized DefaultMQPushConsumer getOrCreateConsumer(String group, String topic, + ExtendedConsumerProperties consumerProperties) { + propertiesMap.put(new AbstractMap.SimpleEntry<>(group, topic), consumerProperties); + ConsumerGroupInstrumentation instrumentation = instrumentationManager.getConsumerGroupInstrumentation(group); + instrumentationManager.addHealthInstrumentation(instrumentation); + + if (consumerGroups.containsKey(group)) { + return consumerGroups.get(group); + } + + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group); + consumer.setNamesrvAddr(rocketBinderConfigurationProperties.getNamesrvAddr()); + consumerGroups.put(group, consumer); + started.put(group, false); + consumer.setConsumeThreadMax(consumerProperties.getConcurrency()); + consumer.setConsumeThreadMin(consumerProperties.getConcurrency()); + if (consumerProperties.getExtension().getBroadcasting()) { + consumer.setMessageModel(MessageModel.BROADCASTING); + } + logger.info("RocketMQ consuming for SCS group {} created", group); + return consumer; + } + + public synchronized void startConsumers() throws MQClientException { + for (String group : getConsumerGroups()) { + start(group); + } + } + + public synchronized void startConsumer(String group) throws MQClientException { + start(group); + } + + public synchronized void stopConsumer(String group) { + stop(group); + } + + private void stop(String group) { + if (consumerGroups.get(group) != null) { + consumerGroups.get(group).shutdown(); + started.put(group, false); + } + } + + private synchronized void start(String group) throws MQClientException { + if (started.get(group)) { + return; + } + ConsumerGroupInstrumentation groupInstrumentation = instrumentationManager.getConsumerGroupInstrumentation( + group); + instrumentationManager.addHealthInstrumentation(groupInstrumentation); + try { + consumerGroups.get(group).start(); + started.put(group, true); + groupInstrumentation.markStartedSuccessfully(); + } catch (MQClientException e) { + groupInstrumentation.markStartFailed(e); + logger.error("RocketMQ Consumer hasn't been started. Caused by " + e.getErrorMessage(), e); + throw e; + } + } + + public synchronized Set getConsumerGroups() { + return consumerGroups.keySet(); + } +} + diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQInboundChannelAdapter.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQInboundChannelAdapter.java new file mode 100644 index 000000000..9d8194621 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQInboundChannelAdapter.java @@ -0,0 +1,314 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.integration; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.lang3.ClassUtils; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListener; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.message.MessageExt; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.rocketmq.RocketMQMessageHeaderAccessor; +import org.springframework.cloud.stream.binder.rocketmq.consuming.Acknowledgement; +import org.springframework.cloud.stream.binder.rocketmq.consuming.ConsumersManager; +import org.springframework.cloud.stream.binder.rocketmq.metrics.ConsumerInstrumentation; +import org.springframework.cloud.stream.binder.rocketmq.metrics.InstrumentationManager; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQConsumerProperties; +import org.springframework.integration.endpoint.MessageProducerSupport; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.retry.RecoveryCallback; +import org.springframework.retry.RetryCallback; +import org.springframework.retry.RetryContext; +import org.springframework.retry.RetryListener; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.StringUtils; + +/** + * @author Jim + */ +public class RocketMQInboundChannelAdapter extends MessageProducerSupport { + + private static final Logger logger = LoggerFactory + .getLogger(RocketMQInboundChannelAdapter.class); + + private ConsumerInstrumentation consumerInstrumentation; + + private final ExtendedConsumerProperties consumerProperties; + + private final String destination; + + private final String group; + + private final InstrumentationManager instrumentationManager; + + private final ConsumersManager consumersManager; + + private RetryTemplate retryTemplate; + + private RecoveryCallback recoveryCallback; + + public RocketMQInboundChannelAdapter(ConsumersManager consumersManager, + ExtendedConsumerProperties consumerProperties, + String destination, String group, + InstrumentationManager instrumentationManager) { + this.consumersManager = consumersManager; + this.consumerProperties = consumerProperties; + this.destination = destination; + this.group = group; + this.instrumentationManager = instrumentationManager; + } + + @Override + protected void doStart() { + if (!consumerProperties.getExtension().getEnabled()) { + return; + } + + String tags = consumerProperties == null ? null + : consumerProperties.getExtension().getTags(); + Boolean isOrderly = consumerProperties == null ? false + : consumerProperties.getExtension().getOrderly(); + + DefaultMQPushConsumer consumer = consumersManager.getOrCreateConsumer(group, + destination, consumerProperties); + + final CloudStreamMessageListener listener = isOrderly + ? new CloudStreamMessageListenerOrderly(instrumentationManager) + : new CloudStreamMessageListenerConcurrently(instrumentationManager); + + if (retryTemplate != null) { + retryTemplate.registerListener(listener); + } + + Set tagsSet = new HashSet<>(); + if (!StringUtils.isEmpty(tags)) { + for (String tag : tags.split("\\|\\|")) { + tagsSet.add(tag.trim()); + } + } + + consumerInstrumentation = instrumentationManager + .getConsumerInstrumentation(destination); + instrumentationManager.addHealthInstrumentation(consumerInstrumentation); + + try { + if (!StringUtils.isEmpty(consumerProperties.getExtension().getSql())) { + consumer.subscribe(destination, MessageSelector + .bySql(consumerProperties.getExtension().getSql())); + } + else { + consumer.subscribe(destination, + org.apache.commons.lang3.StringUtils.join(tagsSet, " || ")); + } + consumerInstrumentation.markStartedSuccessfully(); + } + catch (MQClientException e) { + consumerInstrumentation.markStartFailed(e); + logger.error("RocketMQ Consumer hasn't been subscribed. Caused by " + + e.getErrorMessage(), e); + throw new RuntimeException("RocketMQ Consumer hasn't been subscribed.", e); + } + + consumer.registerMessageListener(listener); + + try { + consumersManager.startConsumer(group); + } + catch (MQClientException e) { + logger.error( + "RocketMQ Consumer startup failed. Caused by " + e.getErrorMessage(), + e); + throw new RuntimeException("RocketMQ Consumer startup failed.", e); + } + } + + @Override + protected void doStop() { + consumersManager.stopConsumer(group); + } + + public void setRetryTemplate(RetryTemplate retryTemplate) { + this.retryTemplate = retryTemplate; + } + + public void setRecoveryCallback(RecoveryCallback recoveryCallback) { + this.recoveryCallback = recoveryCallback; + } + + protected class CloudStreamMessageListener implements MessageListener, RetryListener { + + private final InstrumentationManager instrumentationManager; + + CloudStreamMessageListener(InstrumentationManager instrumentationManager) { + this.instrumentationManager = instrumentationManager; + } + + Acknowledgement consumeMessage(final List msgs) { + boolean enableRetry = RocketMQInboundChannelAdapter.this.retryTemplate != null; + try { + if (enableRetry) { + return RocketMQInboundChannelAdapter.this.retryTemplate.execute( + // (RetryCallback) + new RetryCallback() { + @Override + public Acknowledgement doWithRetry(RetryContext context) + throws Exception { + return doSendMsgs(msgs, context); + } + }, new RecoveryCallback() { + @Override + public Acknowledgement recover(RetryContext context) + throws Exception { + RocketMQInboundChannelAdapter.this.recoveryCallback + .recover(context); + if (ClassUtils.isAssignable(this.getClass(), + MessageListenerConcurrently.class)) { + return Acknowledgement + .buildConcurrentlyInstance(); + } + else { + return Acknowledgement.buildOrderlyInstance(); + } + } + }); + } + else { + Acknowledgement result = doSendMsgs(msgs, null); + instrumentationManager + .getConsumerInstrumentation( + RocketMQInboundChannelAdapter.this.destination) + .markConsumed(); + return result; + } + } + catch (Exception e) { + logger.error( + "Rocket Message hasn't been processed successfully. Caused by ", + e); + instrumentationManager + .getConsumerInstrumentation( + RocketMQInboundChannelAdapter.this.destination) + .markConsumedFailure(); + throw new RuntimeException( + "Rocket Message hasn't been processed successfully. Caused by ", + e); + } + } + + private Acknowledgement doSendMsgs(final List msgs, + RetryContext context) { + List acknowledgements = new ArrayList<>(); + for (MessageExt msg : msgs) { + String retryInfo = context == null ? "" + : "retryCount-" + String.valueOf(context.getRetryCount()) + "|"; + logger.debug(retryInfo + "consuming msg:\n" + msg); + logger.debug(retryInfo + "message body:\n" + new String(msg.getBody())); + Acknowledgement acknowledgement = new Acknowledgement(); + Message toChannel = MessageBuilder.withPayload(msg.getBody()) + .setHeaders(new RocketMQMessageHeaderAccessor() + .withAcknowledgment(acknowledgement) + .withTags(msg.getTags()).withKeys(msg.getKeys()) + .withFlag(msg.getFlag()).withRocketMessage(msg)) + .build(); + acknowledgements.add(acknowledgement); + RocketMQInboundChannelAdapter.this.sendMessage(toChannel); + } + return acknowledgements.get(0); + } + + @Override + public boolean open(RetryContext context, + RetryCallback callback) { + return true; + } + + @Override + public void close(RetryContext context, + RetryCallback callback, Throwable throwable) { + if (throwable != null) { + instrumentationManager + .getConsumerInstrumentation( + RocketMQInboundChannelAdapter.this.destination) + .markConsumedFailure(); + } + else { + instrumentationManager + .getConsumerInstrumentation( + RocketMQInboundChannelAdapter.this.destination) + .markConsumed(); + } + } + + @Override + public void onError(RetryContext context, + RetryCallback callback, Throwable throwable) { + } + } + + protected class CloudStreamMessageListenerConcurrently + extends CloudStreamMessageListener implements MessageListenerConcurrently { + + public CloudStreamMessageListenerConcurrently( + InstrumentationManager instrumentationManager) { + super(instrumentationManager); + } + + @Override + public ConsumeConcurrentlyStatus consumeMessage(final List msgs, + ConsumeConcurrentlyContext context) { + Acknowledgement acknowledgement = consumeMessage(msgs); + context.setDelayLevelWhenNextConsume( + acknowledgement.getConsumeConcurrentlyDelayLevel()); + return acknowledgement.getConsumeConcurrentlyStatus(); + } + } + + protected class CloudStreamMessageListenerOrderly extends CloudStreamMessageListener + implements MessageListenerOrderly { + + public CloudStreamMessageListenerOrderly( + InstrumentationManager instrumentationManager) { + super(instrumentationManager); + } + + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, + ConsumeOrderlyContext context) { + Acknowledgement acknowledgement = consumeMessage(msgs); + context.setSuspendCurrentQueueTimeMillis( + (acknowledgement.getConsumeOrderlySuspendCurrentQueueTimeMill())); + return acknowledgement.getConsumeOrderlyStatus(); + } + + } + +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQMessageHandler.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQMessageHandler.java new file mode 100644 index 000000000..88273e8c1 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQMessageHandler.java @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.integration; + +import java.util.Map; + +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.springframework.cloud.stream.binder.rocketmq.RocketMQBinderConstants; +import org.springframework.cloud.stream.binder.rocketmq.RocketMQMessageHeaderAccessor; +import org.springframework.cloud.stream.binder.rocketmq.metrics.InstrumentationManager; +import org.springframework.cloud.stream.binder.rocketmq.metrics.ProducerInstrumentation; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQProducerProperties; +import org.springframework.context.Lifecycle; +import org.springframework.integration.handler.AbstractMessageHandler; +import org.springframework.integration.support.MutableMessage; +import org.springframework.messaging.MessagingException; + +/** + * @author Jim + */ +public class RocketMQMessageHandler extends AbstractMessageHandler implements Lifecycle { + + private DefaultMQProducer producer; + + private ProducerInstrumentation producerInstrumentation; + + private final RocketMQProducerProperties producerProperties; + + private final String destination; + + private final RocketMQBinderConfigurationProperties rocketBinderConfigurationProperties; + + private final InstrumentationManager instrumentationManager; + + protected volatile boolean running = false; + + public RocketMQMessageHandler(String destination, + RocketMQProducerProperties producerProperties, + RocketMQBinderConfigurationProperties rocketBinderConfigurationProperties, + InstrumentationManager instrumentationManager) { + this.destination = destination; + this.producerProperties = producerProperties; + this.rocketBinderConfigurationProperties = rocketBinderConfigurationProperties; + this.instrumentationManager = instrumentationManager; + } + + @Override + public void start() { + producer = new DefaultMQProducer(destination); + + producerInstrumentation = instrumentationManager + .getProducerInstrumentation(destination); + instrumentationManager.addHealthInstrumentation(producerInstrumentation); + + producer.setNamesrvAddr(rocketBinderConfigurationProperties.getNamesrvAddr()); + + if (producerProperties.getMaxMessageSize() > 0) { + producer.setMaxMessageSize(producerProperties.getMaxMessageSize()); + } + + try { + producer.start(); + producerInstrumentation.markStartedSuccessfully(); + } + catch (MQClientException e) { + producerInstrumentation.markStartFailed(e); + logger.error( + "RocketMQ Message hasn't been sent. Caused by " + e.getMessage()); + throw new MessagingException(e.getMessage(), e); + } + running = true; + } + + @Override + public void stop() { + if (producer != null) { + producer.shutdown(); + } + running = false; + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + protected void handleMessageInternal(org.springframework.messaging.Message message) + throws Exception { + try { + Message toSend; + if (message.getPayload() instanceof byte[]) { + toSend = new Message(destination, (byte[]) message.getPayload()); + } + else if (message.getPayload() instanceof String) { + toSend = new Message(destination, + ((String) message.getPayload()).getBytes()); + } + else { + throw new UnsupportedOperationException("Payload class isn't supported: " + + message.getPayload().getClass()); + } + RocketMQMessageHeaderAccessor headerAccessor = new RocketMQMessageHeaderAccessor( + message); + headerAccessor.setLeaveMutable(true); + toSend.setDelayTimeLevel(headerAccessor.getDelayTimeLevel()); + toSend.setTags(headerAccessor.getTags()); + toSend.setKeys(headerAccessor.getKeys()); + toSend.setFlag(headerAccessor.getFlag()); + for (Map.Entry entry : headerAccessor.getUserProperties() + .entrySet()) { + toSend.putUserProperty(entry.getKey(), entry.getValue()); + } + + SendResult sendRes = producer.send(toSend); + + if (!sendRes.getSendStatus().equals(SendStatus.SEND_OK)) { + throw new MQClientException("message hasn't been sent", null); + } + if (message instanceof MutableMessage) { + RocketMQMessageHeaderAccessor.putSendResult((MutableMessage) message, + sendRes); + } + instrumentationManager.getRuntime().put( + RocketMQBinderConstants.LASTSEND_TIMESTAMP, + System.currentTimeMillis()); + producerInstrumentation.markSent(); + } + catch (MQClientException | RemotingException | MQBrokerException + | InterruptedException | UnsupportedOperationException e) { + producerInstrumentation.markSentFailure(); + logger.error( + "RocketMQ Message hasn't been sent. Caused by " + e.getMessage()); + throw new MessagingException(e.getMessage(), e); + } + + } + +} \ No newline at end of file diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ConsumerGroupInstrumentation.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ConsumerGroupInstrumentation.java new file mode 100644 index 000000000..6e4386d00 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ConsumerGroupInstrumentation.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.metrics; + +import com.codahale.metrics.MetricRegistry; + +/** + * @author Timur Valiev + * @author Jim + */ +public class ConsumerGroupInstrumentation extends Instrumentation { + private MetricRegistry metricRegistry; + + public ConsumerGroupInstrumentation(MetricRegistry metricRegistry, String name) { + super(name); + this.metricRegistry = metricRegistry; + } + +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ConsumerInstrumentation.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ConsumerInstrumentation.java new file mode 100644 index 000000000..e773729f6 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ConsumerInstrumentation.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.metrics; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; + +import static com.codahale.metrics.MetricRegistry.name; + +/** + * @author juven.xuxb + * @author Jim + */ +public class ConsumerInstrumentation extends Instrumentation { + + private final Counter totalConsumed; + private final Counter totalConsumedFailures; + private final Meter consumedPerSecond; + private final Meter consumedFailuresPerSecond; + + public ConsumerInstrumentation(MetricRegistry registry, String baseMetricName) { + super(baseMetricName); + this.totalConsumed = registry.counter(name(baseMetricName, "totalConsumed")); + this.consumedPerSecond = registry.meter(name(baseMetricName, "consumedPerSecond")); + this.totalConsumedFailures = registry.counter(name(baseMetricName, "totalConsumedFailures")); + this.consumedFailuresPerSecond = registry.meter(name(baseMetricName, "consumedFailuresPerSecond")); + } + + public void markConsumed() { + totalConsumed.inc(); + consumedPerSecond.mark(); + } + + public void markConsumedFailure() { + totalConsumedFailures.inc(); + consumedFailuresPerSecond.mark(); + } +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/Instrumentation.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/Instrumentation.java new file mode 100644 index 000000000..c169ba83e --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/Instrumentation.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.metrics; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author Timur Valiev + * @author Jim + */ +public class Instrumentation { + private final String name; + protected final AtomicBoolean started = new AtomicBoolean(false); + protected Exception startException = null; + + Instrumentation(String name) { + this.name = name; + } + + public boolean isDown() { + return startException != null; + } + + public boolean isUp() { + return started.get(); + } + + public boolean isOutOfService() { + return !started.get() && startException == null; + } + + public void markStartedSuccessfully() { + started.set(true); + } + + public void markStartFailed(Exception e) { + started.set(false); + startException = e; + } + + public String getName() { + return name; + } + + public boolean isStarted() { + return started.get(); + } + + public Exception getStartException() { + return startException; + } +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/InstrumentationManager.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/InstrumentationManager.java new file mode 100644 index 000000000..dea848590 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/InstrumentationManager.java @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.metrics; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import com.codahale.metrics.MetricRegistry; + +/** + * @author Timur Valiev + * @author Jim + */ +public class InstrumentationManager { + private final MetricRegistry metricRegistry; + private final Map runtime; + private final Map producerInstrumentations = new HashMap<>(); + private final Map consumeInstrumentations = new HashMap<>(); + private final Map consumerGroupsInstrumentations = new HashMap<>(); + + private final Map healthInstrumentations = new HashMap<>(); + + public InstrumentationManager(MetricRegistry metricRegistry, + Map runtime) { + this.metricRegistry = metricRegistry; + this.runtime = runtime; + } + + public ProducerInstrumentation getProducerInstrumentation(String destination) { + String key = "scs-rocketmq.producer." + destination; + ProducerInstrumentation producerInstrumentation = producerInstrumentations + .get(key); + if (producerInstrumentation == null) { + producerInstrumentations.put(key, + new ProducerInstrumentation(metricRegistry, key)); + } + return producerInstrumentations.get(key); + } + + public ConsumerInstrumentation getConsumerInstrumentation(String destination) { + String key = "scs-rocketmq.consumer." + destination; + ConsumerInstrumentation consumerInstrumentation = consumeInstrumentations + .get(key); + if (consumerInstrumentation == null) { + consumeInstrumentations.put(key, + new ConsumerInstrumentation(metricRegistry, key)); + } + return consumeInstrumentations.get(key); + } + + public ConsumerGroupInstrumentation getConsumerGroupInstrumentation(String group) { + String key = "scs-rocketmq.consumerGroup." + group; + ConsumerGroupInstrumentation consumerGroupInstrumentation = consumerGroupsInstrumentations + .get(key); + if (consumerGroupInstrumentation == null) { + consumerGroupsInstrumentations.put(key, + new ConsumerGroupInstrumentation(metricRegistry, key)); + } + return consumerGroupsInstrumentations.get(key); + } + + public Set getHealthInstrumentations() { + return new HashSet<>(healthInstrumentations.values()); + } + + public void addHealthInstrumentation(Instrumentation instrumentation) { + healthInstrumentations.put(instrumentation.getName(), instrumentation); + } + + public Map getRuntime() { + return runtime; + } +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ProducerInstrumentation.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ProducerInstrumentation.java new file mode 100644 index 000000000..6fc6daf98 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/metrics/ProducerInstrumentation.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.metrics; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; + +import static com.codahale.metrics.MetricRegistry.name; + +/** + * @author juven.xuxb + * @author Jim + */ +public class ProducerInstrumentation extends Instrumentation { + + private final Counter totalSent; + private final Counter totalSentFailures; + private final Meter sentPerSecond; + private final Meter sentFailuresPerSecond; + + public ProducerInstrumentation(MetricRegistry registry, String baseMetricName) { + super(baseMetricName); + this.totalSent = registry.counter(name(baseMetricName, "totalSent")); + this.totalSentFailures = registry.counter(name(baseMetricName, "totalSentFailures")); + this.sentPerSecond = registry.meter(name(baseMetricName, "sentPerSecond")); + this.sentFailuresPerSecond = registry.meter(name(baseMetricName, "sentFailuresPerSecond")); + } + + public void markSent() { + totalSent.inc(); + sentPerSecond.mark(); + } + + public void markSentFailure() { + totalSentFailures.inc(); + sentFailuresPerSecond.mark(); + } +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQBinderConfigurationProperties.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQBinderConfigurationProperties.java new file mode 100644 index 000000000..f2c028ec4 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQBinderConfigurationProperties.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * @author Timur Valiev + * @author Jim + */ +@ConfigurationProperties(prefix = "spring.cloud.stream.rocketmq.binder") +public class RocketMQBinderConfigurationProperties { + + private String namesrvAddr = "127.0.0.1:9876"; + + private String logLevel = "ERROR"; + + public String getNamesrvAddr() { + return namesrvAddr; + } + + public void setNamesrvAddr(String namesrvAddr) { + this.namesrvAddr = namesrvAddr; + } + + public String getLogLevel() { + return logLevel; + } + + public void setLogLevel(String logLevel) { + this.logLevel = logLevel; + } + +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQBindingProperties.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQBindingProperties.java new file mode 100644 index 000000000..b3c2f194f --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQBindingProperties.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.properties; + +/** + * @author Timur Valiev + * @author Jim + */ +public class RocketMQBindingProperties { + + private RocketMQConsumerProperties consumer = new RocketMQConsumerProperties(); + + private RocketMQProducerProperties producer = new RocketMQProducerProperties(); + + public RocketMQConsumerProperties getConsumer() { + return consumer; + } + + public void setConsumer(RocketMQConsumerProperties consumer) { + this.consumer = consumer; + } + + public RocketMQProducerProperties getProducer() { + return producer; + } + + public void setProducer(RocketMQProducerProperties producer) { + this.producer = producer; + } +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQConsumerProperties.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQConsumerProperties.java new file mode 100644 index 000000000..7f757586b --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQConsumerProperties.java @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.properties; + +import org.apache.rocketmq.client.consumer.MQPushConsumer; +import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; + +/** + * @author Timur Valiev + * @author Jim + */ +public class RocketMQConsumerProperties { + + /** + * using '||' to split tag + * {@link MQPushConsumer#subscribe(String, String)} + */ + private String tags; + + /** + * {@link MQPushConsumer#subscribe(String, MessageSelector)} + * {@link MessageSelector#bySql(String)} + */ + private String sql; + + /** + * {@link MessageModel#BROADCASTING} + */ + private Boolean broadcasting = false; + + /** + * if orderly is true, using {@link MessageListenerOrderly} + * else if orderly if false, using {@link MessageListenerConcurrently} + */ + private Boolean orderly = false; + + private Boolean enabled = true; + + public String getTags() { + return tags; + } + + public void setTags(String tags) { + this.tags = tags; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public Boolean getOrderly() { + return orderly; + } + + public void setOrderly(Boolean orderly) { + this.orderly = orderly; + } + + public Boolean getEnabled() { + return enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + public Boolean getBroadcasting() { + return broadcasting; + } + + public void setBroadcasting(Boolean broadcasting) { + this.broadcasting = broadcasting; + } +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQExtendedBindingProperties.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQExtendedBindingProperties.java new file mode 100644 index 000000000..58d9d8ef7 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQExtendedBindingProperties.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.properties; + +import java.util.HashMap; +import java.util.Map; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.stream.binder.ExtendedBindingProperties; + +/** + * @author Timur Valiev + * @author Jim + */ +@ConfigurationProperties("spring.cloud.stream.rocketmq") +public class RocketMQExtendedBindingProperties implements + ExtendedBindingProperties { + + private Map bindings = new HashMap<>(); + + public Map getBindings() { + return this.bindings; + } + + public void setBindings(Map bindings) { + this.bindings = bindings; + } + + @Override + public synchronized RocketMQConsumerProperties getExtendedConsumerProperties(String channelName) { + if (bindings.containsKey(channelName)) { + if (bindings.get(channelName).getConsumer() != null) { + return bindings.get(channelName).getConsumer(); + } else { + RocketMQConsumerProperties properties = new RocketMQConsumerProperties(); + this.bindings.get(channelName).setConsumer(properties); + return properties; + } + } else { + RocketMQConsumerProperties properties = new RocketMQConsumerProperties(); + RocketMQBindingProperties rbp = new RocketMQBindingProperties(); + rbp.setConsumer(properties); + bindings.put(channelName, rbp); + return properties; + } + } + + @Override + public synchronized RocketMQProducerProperties getExtendedProducerProperties(String channelName) { + if (bindings.containsKey(channelName)) { + if (bindings.get(channelName).getProducer() != null) { + return bindings.get(channelName).getProducer(); + } else { + RocketMQProducerProperties properties = new RocketMQProducerProperties(); + this.bindings.get(channelName).setProducer(properties); + return properties; + } + } else { + RocketMQProducerProperties properties = new RocketMQProducerProperties(); + RocketMQBindingProperties rbp = new RocketMQBindingProperties(); + rbp.setProducer(properties); + bindings.put(channelName, rbp); + return properties; + } + } +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQProducerProperties.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQProducerProperties.java new file mode 100644 index 000000000..8c2528344 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/properties/RocketMQProducerProperties.java @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.properties; + +import org.apache.rocketmq.client.producer.DefaultMQProducer; + +/** + * @author Timur Valiev + * @author Jim + */ +public class RocketMQProducerProperties { + + private Boolean enabled = true; + + /** + * Maximum allowed message size in bytes + * {@link DefaultMQProducer#maxMessageSize} + */ + private Integer maxMessageSize = 0; + + public Boolean getEnabled() { + return enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + public Integer getMaxMessageSize() { + return maxMessageSize; + } + + public void setMaxMessageSize(Integer maxMessageSize) { + this.maxMessageSize = maxMessageSize; + } + +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/provisioning/RocketMQTopicProvisioner.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/provisioning/RocketMQTopicProvisioner.java new file mode 100644 index 000000000..17bf56ac0 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/provisioning/RocketMQTopicProvisioner.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq.provisioning; + +import org.apache.rocketmq.client.Validators; +import org.apache.rocketmq.client.exception.MQClientException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; +import org.springframework.cloud.stream.binder.ExtendedProducerProperties; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQConsumerProperties; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQProducerProperties; +import org.springframework.cloud.stream.provisioning.ConsumerDestination; +import org.springframework.cloud.stream.provisioning.ProducerDestination; +import org.springframework.cloud.stream.provisioning.ProvisioningException; +import org.springframework.cloud.stream.provisioning.ProvisioningProvider; + +/** + * @author Timur Valiev + * @author Jim + */ +public class RocketMQTopicProvisioner + implements + ProvisioningProvider, + ExtendedProducerProperties> { + + private static final Logger logger = LoggerFactory.getLogger(RocketMQTopicProvisioner.class); + + @Override + public ProducerDestination provisionProducerDestination(String name, + ExtendedProducerProperties + properties) + throws ProvisioningException { + checkTopic(name); + return new RocketProducerDestination(name); + } + + @Override + public ConsumerDestination provisionConsumerDestination(String name, String group, + ExtendedConsumerProperties + properties) + throws ProvisioningException { + checkTopic(name); + return new RocketConsumerDestination(name); + } + + private void checkTopic(String topic) { + try { + Validators.checkTopic(topic); + } catch (MQClientException e) { + logger.error("topic check error: " + topic, e); + throw new AssertionError(e); // Can't happen + } + } + + private static final class RocketProducerDestination implements ProducerDestination { + + private final String producerDestinationName; + + RocketProducerDestination(String destinationName) { + this.producerDestinationName = destinationName; + } + + @Override + public String getName() { + return producerDestinationName; + } + + @Override + public String getNameForPartition(int partition) { + return producerDestinationName; + } + + } + + private static final class RocketConsumerDestination implements ConsumerDestination { + + private final String consumerDestinationName; + + RocketConsumerDestination(String consumerDestinationName) { + this.consumerDestinationName = consumerDestinationName; + } + + @Override + public String getName() { + return this.consumerDestinationName; + } + + } + +} diff --git a/spring-cloud-stream-binder-rocketmq/src/main/resources/META-INF/spring.binders b/spring-cloud-stream-binder-rocketmq/src/main/resources/META-INF/spring.binders new file mode 100644 index 000000000..78d86cf92 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/resources/META-INF/spring.binders @@ -0,0 +1 @@ +rocketmq:org.springframework.cloud.stream.binder.rocketmq.config.RocketMQBinderAutoConfiguration \ No newline at end of file diff --git a/spring-cloud-stream-binder-rocketmq/src/main/resources/META-INF/spring.factories b/spring-cloud-stream-binder-rocketmq/src/main/resources/META-INF/spring.factories new file mode 100644 index 000000000..43b85129a --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +org.springframework.cloud.stream.binder.rocketmq.config.RocketMQBinderEndpointAutoConfiguration diff --git a/spring-cloud-stream-binder-rocketmq/src/test/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQAutoConfigurationTests.java b/spring-cloud-stream-binder-rocketmq/src/test/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQAutoConfigurationTests.java new file mode 100644 index 000000000..3328bb648 --- /dev/null +++ b/spring-cloud-stream-binder-rocketmq/src/test/java/org/springframework/cloud/stream/binder/rocketmq/RocketMQAutoConfigurationTests.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.binder.rocketmq; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.rocketmq.config.RocketMQBinderAutoConfiguration; +import org.springframework.cloud.stream.binder.rocketmq.config.RocketMQBinderEndpointAutoConfiguration; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQBinderConfigurationProperties; +import org.springframework.cloud.stream.binder.rocketmq.properties.RocketMQExtendedBindingProperties; +import org.springframework.context.ConfigurableApplicationContext; + +/** + * @author Jim + */ +public class RocketMQAutoConfigurationTests { + + private ConfigurableApplicationContext context; + + @Before + public void setUp() throws Exception { + this.context = new SpringApplicationBuilder( + RocketMQBinderEndpointAutoConfiguration.class, + RocketMQBinderAutoConfiguration.class).web(false).run( + "--spring.cloud.stream.rocketmq.binder.namesrv-addr=127.0.0.1:9876", + "--spring.cloud.stream.bindings.output.destination=TopicOrderTest", + "--spring.cloud.stream.bindings.output.content-type=application/json", + "--spring.cloud.stream.bindings.input1.destination=TopicOrderTest", + "--spring.cloud.stream.bindings.input1.content-type=application/json", + "--spring.cloud.stream.bindings.input1.group=test-group1", + "--spring.cloud.stream.rocketmq.bindings.input1.consumer.orderly=true", + "--spring.cloud.stream.bindings.input1.consumer.maxAttempts=1", + "--spring.cloud.stream.bindings.input2.destination=TopicOrderTest", + "--spring.cloud.stream.bindings.input2.content-type=application/json", + "--spring.cloud.stream.bindings.input2.group=test-group2", + "--spring.cloud.stream.rocketmq.bindings.input2.consumer.orderly=false", + "--spring.cloud.stream.rocketmq.bindings.input2.consumer.tags=tag1"); + } + + @Test + public void testProperties() { + RocketMQBinderConfigurationProperties binderConfigurationProperties = context + .getBean(RocketMQBinderConfigurationProperties.class); + assertThat(binderConfigurationProperties.getNamesrvAddr()) + .isEqualTo("127.0.0.1:9876"); + RocketMQExtendedBindingProperties bindingProperties = context + .getBean(RocketMQExtendedBindingProperties.class); + assertThat(bindingProperties.getExtendedConsumerProperties("input2").getTags()) + .isEqualTo("tag1"); + assertThat(bindingProperties.getExtendedConsumerProperties("input2").getOrderly()) + .isFalse(); + assertThat(bindingProperties.getExtendedConsumerProperties("input1").getOrderly()) + .isTrue(); + } + +} \ No newline at end of file From d0aa5dabbb77b9a74ddf8af37874b80e3f7d0df3 Mon Sep 17 00:00:00 2001 From: fangjian0423 Date: Tue, 27 Nov 2018 11:36:58 +0800 Subject: [PATCH 3/4] add rocketmq binder example --- spring-cloud-alibaba-examples/pom.xml | 1 + .../rocketmq-example/pom.xml | 51 ++++ .../rocketmq-example/readme-zh.md | 259 ++++++++++++++++++ .../cloud/alibaba/cloud/examples/Foo.java | 39 +++ .../cloud/examples/ReceiveService.java | 33 +++ .../cloud/examples/RocketMQApplication.java | 65 +++++ .../alibaba/cloud/examples/SenderService.java | 43 +++ .../src/main/resources/application.properties | 31 +++ 8 files changed, 522 insertions(+) create mode 100644 spring-cloud-alibaba-examples/rocketmq-example/pom.xml create mode 100644 spring-cloud-alibaba-examples/rocketmq-example/readme-zh.md create mode 100644 spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/Foo.java create mode 100644 spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/ReceiveService.java create mode 100644 spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/RocketMQApplication.java create mode 100644 spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/SenderService.java create mode 100644 spring-cloud-alibaba-examples/rocketmq-example/src/main/resources/application.properties diff --git a/spring-cloud-alibaba-examples/pom.xml b/spring-cloud-alibaba-examples/pom.xml index 47a34f4f7..98b392f84 100644 --- a/spring-cloud-alibaba-examples/pom.xml +++ b/spring-cloud-alibaba-examples/pom.xml @@ -27,6 +27,7 @@ ans-example/ans-consumer-ribbon-example ans-example/ans-provider-example acm-example/acm-local-example + rocketmq-example diff --git a/spring-cloud-alibaba-examples/rocketmq-example/pom.xml b/spring-cloud-alibaba-examples/rocketmq-example/pom.xml new file mode 100644 index 000000000..e8800187b --- /dev/null +++ b/spring-cloud-alibaba-examples/rocketmq-example/pom.xml @@ -0,0 +1,51 @@ + + + + + org.springframework.cloud + spring-cloud-alibaba-examples + 0.1.1.BUILD-SNAPSHOT + + 4.0.0 + + + rocketmq-example + jar + Example demonstrating how to use rocketmq + + + + + org.springframework.cloud + spring-cloud-starter-stream-rocketmq + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-actuator + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-deploy-plugin + ${maven-deploy-plugin.version} + + true + + + + + + + diff --git a/spring-cloud-alibaba-examples/rocketmq-example/readme-zh.md b/spring-cloud-alibaba-examples/rocketmq-example/readme-zh.md new file mode 100644 index 000000000..b00bfbc34 --- /dev/null +++ b/spring-cloud-alibaba-examples/rocketmq-example/readme-zh.md @@ -0,0 +1,259 @@ +# RocketMQ Example + +## 项目说明 + +本项目演示如何使用 RocketMQ Binder 完成 Spring Cloud 应用消息的订阅和发布。 + +[RocketMQ](https://rocketmq.apache.org/) 是一款开源的分布式消息系统，基于高可用分布式集群技术，提供低延时的、高可靠的消息发布与订阅服务。 + +在说明RocketMQ的示例之前，我们先了解一下 Spring Cloud Stream 中的Binder和Binding概念。 + +Binder: 跟外部消息中间件集成的组件，用来创建Binding，各消息中间件都有自己的Binder实现。 + +比如 `Kafka` 的实现 `KafkaMessageChannelBinder` ，`RabbitMQ` 的实现 `RabbitMessageChannelBinder` 以及 `RocketMQ` 的实现 `RocketMQMessageChannelBinder` 。 + +Binding: 包括Input Binding和Output Binding。 + +Binding在消息中间件与应用程序提供的Provider和Consumer之间提供了一个桥梁，实现了开发者只需使用应用程序的Provider或Consumer生产或消费数据即可，屏蔽了开发者与底层消息中间件的接触。 + +下图是Spring Cloud Stream的架构设计。 + +![](https://docs.spring.io/spring-cloud-stream/docs/current/reference/htmlsingle/images/SCSt-overview.png) + +## 示例 + +### 如何接入 + +在启动示例进行演示之前，我们先了解一下 Spring Cloud 应用如何接入 RocketMQ Binder。 + +> **注意：本章节只是为了便于您理解接入方式，本示例代码中已经完成****接入工作，您无需再进行修改。** + +1. 首先，修改 `pom.xml` 文件，引入 RocketMQ Stream Starter。 + +```xml + + org.springframework.cloud + spring-cloud-starter-stream-rocketmq + +``` + +2. 配置Input和Output的Binding信息并配合`@EnableBinding`注解使其生效 + +```java +@SpringBootApplication +@EnableBinding({ Source.class, Sink.class }) +public class RocketMQApplication { + public static void main(String[] args) { + SpringApplication.run(RocketMQApplication.class, args); + } +} +``` + +配置Binding信息： +```properties +# 配置rocketmq的nameserver地址 +spring.cloud.stream.rocketmq.binder.namesrv-addr=127.0.0.1:9876 +# 定义name为output的binding +spring.cloud.stream.bindings.output.destination=test-topic +spring.cloud.stream.bindings.output.content-type=application/json +# 定义name为input的binding +spring.cloud.stream.bindings.input.destination=test-topic +spring.cloud.stream.bindings.input.content-type=application/json +spring.cloud.stream.bindings.input.group=test-group + +``` + +3. 消息发送及消息订阅 + +### 下载并启动 RocketMQ + +在接入RocketMQ Binder之前，首先需要启动RocketMQ的Name Server和Broker。 + +1. 下载[RocketMQ最新的二进制文件](https://www.apache.org/dyn/closer.cgi?path=rocketmq/4.3.2/rocketmq-all-4.3.2-bin-release.zip)，并解压 + +2. 启动 Name Server + +```bash +sh bin/mqnamesrv +``` + +3. 启动Broker + +```bash +sh bin/mqbroker -n localhost:9876 +``` + +4. 创建Topic: test-topic + +```bash +sh bin/mqadmin updateTopic -n localhost:9876 -c DefaultCluster -t test-topic +``` + +### 应用启动 + +1. 增加配置，在应用的 /src/main/resources/application.properties 中添加基本配置信息 + +```properties +spring.application.name=rocketmq-example +server.port=28081 +``` + +2. 启动应用，支持 IDE 直接启动和编译打包后启动。 + + 1. IDE直接启动：找到主类 `RocketMQApplication`，执行 main 方法启动应用。 + 2. 打包编译后启动：首先执行 `mvn clean package` 将工程编译打包，然后执行 `java -jar rocketmq-example.jar`启动应用。 + + +### 消息处理 + +使用name为output对应的binding发送消息到test-topic这个topic。 + +使用2个input binding订阅数据。 + +input1: 订阅topic为test-topic的消息，顺序消费所有消息(顺序消费的前提是所有消息都在一个MessageQueue中) +input2: 订阅topic为test-topic的消息，异步消费tags为tagStr的消息，Consumer端线程池个数为20 + +配置信息如下： + +```properties +spring.cloud.stream.rocketmq.binder.namesrv-addr=127.0.0.1:9876 + +spring.cloud.stream.bindings.output.destination=test-topic +spring.cloud.stream.bindings.output.content-type=application/json + +spring.cloud.stream.bindings.input1.destination=test-topic +spring.cloud.stream.bindings.input1.content-type=text/plain +spring.cloud.stream.bindings.input1.group=test-group1 +spring.cloud.stream.rocketmq.bindings.input1.consumer.orderly=true + +spring.cloud.stream.bindings.input2.destination=test-topic +spring.cloud.stream.bindings.input2.content-type=text/plain +spring.cloud.stream.bindings.input2.group=test-group2 +spring.cloud.stream.rocketmq.bindings.input2.consumer.orderly=false +spring.cloud.stream.rocketmq.bindings.input2.consumer.tags=tagStr +spring.cloud.stream.bindings.input2.consumer.concurrency=20 + +``` + +#### 消息发送 + +使用MessageChannel进行消息发送： + +```java +public class ProducerRunner implements CommandLineRunner { + @Autowired + private MessageChannel output; // 获取name为output的binding + @Override + public void run(String... args) throws Exception { + Map headers = new HashMap<>(); + headers.put(MessageConst.PROPERTY_TAGS, "tagStr"); + Message message = MessageBuilder.createMessage(msg, new MessageHeaders(headers)); + output.send(message); + } +} +``` + +或者使用RocketMQ原生的API进行消息发送: + +```java +public class RocketMQProducer { + DefaultMQProducer producer = new DefaultMQProducer("producer_group"); + producer.setNamesrvAddr("127.0.0.1:9876"); + producer.start(); + + Message msg = new Message("test-topic", "tagStr", "message from rocketmq producer".getBytes()); + producer.send(msg); +} +``` + +#### 消息接收 + +使用`@StreamListener`注解接收消息： + +```java +@Service +public class ReceiveService { + + @StreamListener("input1") + public void receiveInput1(String receiveMsg) { + System.out.println("input1 receive: " + receiveMsg); + } + + @StreamListener("input2") + public void receiveInput2(String receiveMsg) { + System.out.println("input2 receive: " + receiveMsg); + } + +} +``` + +## Endpoint 信息查看 + +Spring Boot 应用支持通过 Endpoint 来暴露相关信息，RocketMQ Stream Starter 也支持这一点。 + +在使用之前需要在 Maven 中添加 `spring-boot-starter-actuator`依赖，并在配置中允许 Endpoints 的访问。 +* Spring Boot 1.x 中添加配置 `management.security.enabled=false` +* Spring Boot 2.x 中添加配置 `management.endpoints.web.exposure.include=*` + +Spring Boot 1.x 可以通过访问 http://127.0.0.1:28081/rocketmq_binder 来查看 RocketMQ Binder Endpoint 的信息。Spring Boot 2.x 可以通过访问 http://127.0.0.1:28081/actuator/rocketmq-binder 来访问。 + +这里会统计消息最后一次发送的数据，消息发送成功或失败的次数，消息消费成功或失败的次数等数据。 + +```json +{ + "runtime": { + "lastSend.timestamp": 1542786623915 + }, + "metrics": { + "scs-rocketmq.consumer.test-topic.totalConsumed": { + "count": 11 + }, + "scs-rocketmq.consumer.test-topic.totalConsumedFailures": { + "count": 0 + }, + "scs-rocketmq.producer.test-topic.totalSentFailures": { + "count": 0 + }, + "scs-rocketmq.consumer.test-topic.consumedPerSecond": { + "count": 11, + "fifteenMinuteRate": 0.012163847780107841, + "fiveMinuteRate": 0.03614605351360527, + "meanRate": 0.3493213353657594, + "oneMinuteRate": 0.17099243039490175 + }, + "scs-rocketmq.producer.test-topic.totalSent": { + "count": 5 + }, + "scs-rocketmq.producer.test-topic.sentPerSecond": { + "count": 5, + "fifteenMinuteRate": 0.005540151995103271, + "fiveMinuteRate": 0.01652854617838251, + "meanRate": 0.10697493212602836, + "oneMinuteRate": 0.07995558537067671 + }, + "scs-rocketmq.producer.test-topic.sentFailuresPerSecond": { + "count": 0, + "fifteenMinuteRate": 0.0, + "fiveMinuteRate": 0.0, + "meanRate": 0.0, + "oneMinuteRate": 0.0 + }, + "scs-rocketmq.consumer.test-topic.consumedFailuresPerSecond": { + "count": 0, + "fifteenMinuteRate": 0.0, + "fiveMinuteRate": 0.0, + "meanRate": 0.0, + "oneMinuteRate": 0.0 + } + } +} +``` + +## More + +RocketMQ 是一款功能强大的分布式消息系统，广泛应用于多个领域，包括异步通信解耦、企业解决方案、金融支付、电信、电子商务、快递物流、广告营销、社交、即时通信、移动应用、手游、视频、物联网、车联网等。 + +此 Demo 仅演示了 RocketMQ 与 Spring Cloud Stream 结合后的使用，更多 RocketMQ 相关的信息，请参考 [RocketMQ 项目](https://github.com/apache/rocketmq)。 + +如果您对 spring cloud starter stream rocketmq 有任何建议或想法，欢迎在 issue 中或者通过其他社区渠道向我们提出。 + diff --git a/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/Foo.java b/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/Foo.java new file mode 100644 index 000000000..e98b6a10a --- /dev/null +++ b/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/Foo.java @@ -0,0 +1,39 @@ +package org.springframework.cloud.alibaba.cloud.examples; + +/** + * @author Jim + */ +public class Foo { + + private int id; + private String bar; + + public Foo() { + } + + public Foo(int id, String bar) { + this.id = id; + this.bar = bar; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getBar() { + return bar; + } + + public void setBar(String bar) { + this.bar = bar; + } + + @Override + public String toString() { + return "Foo{" + "id=" + id + ", bar='" + bar + '\'' + '}'; + } +} diff --git a/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/ReceiveService.java b/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/ReceiveService.java new file mode 100644 index 000000000..f251902d9 --- /dev/null +++ b/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/ReceiveService.java @@ -0,0 +1,33 @@ +package org.springframework.cloud.alibaba.cloud.examples; + +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.messaging.handler.annotation.Payload; +import org.springframework.stereotype.Service; + +/** + * @author Jim + */ +@Service +public class ReceiveService { + + @StreamListener("input1") + public void receiveInput1(String receiveMsg) { + System.out.println("input1 receive: " + receiveMsg); + } + + @StreamListener("input2") + public void receiveInput2(String receiveMsg) { + System.out.println("input2 receive: " + receiveMsg); + } + + @StreamListener("input3") + public void receiveInput3(@Payload Foo foo) { + System.out.println("input3 receive: " + foo); + } + + @StreamListener("input1") + public void receiveInput1Again(String receiveMsg) { + System.out.println("input1 receive again: " + receiveMsg); + } + +} diff --git a/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/RocketMQApplication.java b/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/RocketMQApplication.java new file mode 100644 index 000000000..e4a8b1ab4 --- /dev/null +++ b/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/RocketMQApplication.java @@ -0,0 +1,65 @@ +package org.springframework.cloud.alibaba.cloud.examples; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.alibaba.cloud.examples.RocketMQApplication.MySink; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.cloud.stream.messaging.Source; +import org.springframework.context.annotation.Bean; +import org.springframework.messaging.SubscribableChannel; + +/** + * @author Jim + */ +@SpringBootApplication +@EnableBinding({ Source.class, MySink.class }) +public class RocketMQApplication { + + public interface MySink { + + @Input("input1") + SubscribableChannel input1(); + + @Input("input2") + SubscribableChannel input2(); + + @Input("input3") + SubscribableChannel input3(); + + } + + public static void main(String[] args) { + SpringApplication.run(RocketMQApplication.class, args); + } + + @Bean + public CustomRunner customRunner() { + return new CustomRunner(); + } + + public static class CustomRunner implements CommandLineRunner { + @Autowired + private SenderService senderService; + + @Override + public void run(String... args) throws Exception { + int count = 5; + for (int index = 1; index <= count; index++) { + String msgContent = "msg-" + index; + if (index % 3 == 0) { + senderService.send(msgContent); + } + else if (index % 3 == 1) { + senderService.sendWithTags(msgContent, "tagStr"); + } + else { + senderService.sendObject(new Foo(index, "foo"), "tagObj"); + } + } + } + } + +} diff --git a/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/SenderService.java b/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/SenderService.java new file mode 100644 index 000000000..1c886d010 --- /dev/null +++ b/spring-cloud-alibaba-examples/rocketmq-example/src/main/java/org/springframework/cloud/alibaba/cloud/examples/SenderService.java @@ -0,0 +1,43 @@ +package org.springframework.cloud.alibaba.cloud.examples; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.rocketmq.common.message.MessageConst; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.stream.messaging.Source; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.stereotype.Service; +import org.springframework.util.MimeTypeUtils; + +/** + * @author Jim + */ +@Service +public class SenderService { + + @Autowired + private Source source; + + public void send(String msg) throws Exception { + source.output().send(MessageBuilder.withPayload(msg).build()); + } + + public void sendWithTags(T msg, String tag) throws Exception { + Map map = new HashMap<>(); + map.put(MessageConst.PROPERTY_TAGS, tag); + Message message = MessageBuilder.createMessage(msg, new MessageHeaders(map)); + source.output().send(message); + } + + public void sendObject(T msg, String tag) throws Exception { + Message message = MessageBuilder.withPayload(msg) + .setHeader(MessageConst.PROPERTY_TAGS, tag) + .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON) + .build(); + source.output().send(message); + } + +} diff --git a/spring-cloud-alibaba-examples/rocketmq-example/src/main/resources/application.properties b/spring-cloud-alibaba-examples/rocketmq-example/src/main/resources/application.properties new file mode 100644 index 000000000..80ea2d990 --- /dev/null +++ b/spring-cloud-alibaba-examples/rocketmq-example/src/main/resources/application.properties @@ -0,0 +1,31 @@ +spring.cloud.stream.default-binder=rocketmq + +spring.cloud.stream.rocketmq.binder.namesrv-addr=127.0.0.1:9876 + +spring.cloud.stream.bindings.output.destination=test-topic +spring.cloud.stream.bindings.output.content-type=application/json + +spring.cloud.stream.bindings.input1.destination=test-topic +spring.cloud.stream.bindings.input1.content-type=text/plain +spring.cloud.stream.bindings.input1.group=test-group1 +spring.cloud.stream.rocketmq.bindings.input1.consumer.orderly=true +spring.cloud.stream.bindings.input1.consumer.maxAttempts=1 + +spring.cloud.stream.bindings.input2.destination=test-topic +spring.cloud.stream.bindings.input2.content-type=text/plain +spring.cloud.stream.bindings.input2.group=test-group2 +spring.cloud.stream.rocketmq.bindings.input2.consumer.orderly=false +spring.cloud.stream.rocketmq.bindings.input2.consumer.tags=tagStr +spring.cloud.stream.bindings.input2.consumer.concurrency=20 +spring.cloud.stream.bindings.input2.consumer.maxAttempts=1 + +spring.cloud.stream.bindings.input3.destination=test-topic +spring.cloud.stream.bindings.input3.content-type=application/json +spring.cloud.stream.bindings.input3.group=test-group3 +spring.cloud.stream.rocketmq.bindings.input3.consumer.tags=tagObj +spring.cloud.stream.bindings.input3.consumer.concurrency=20 +spring.cloud.stream.bindings.input3.consumer.maxAttempts=1 + +server.port=28081 + +management.endpoints.web.exposure.include=* \ No newline at end of file From 017b528e3ac2291d78d00459748e3e8c3445e06d Mon Sep 17 00:00:00 2001 From: fangjian0423 Date: Tue, 27 Nov 2018 11:37:41 +0800 Subject: [PATCH 4/4] fix typo --- .../rocketmq/integration/RocketMQInboundChannelAdapter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQInboundChannelAdapter.java b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQInboundChannelAdapter.java index 9d8194621..8c884da84 100644 --- a/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQInboundChannelAdapter.java +++ b/spring-cloud-stream-binder-rocketmq/src/main/java/org/springframework/cloud/stream/binder/rocketmq/integration/RocketMQInboundChannelAdapter.java @@ -212,14 +212,14 @@ public class RocketMQInboundChannelAdapter extends MessageProducerSupport { } catch (Exception e) { logger.error( - "Rocket Message hasn't been processed successfully. Caused by ", + "RocketMQ Message hasn't been processed successfully. Caused by ", e); instrumentationManager .getConsumerInstrumentation( RocketMQInboundChannelAdapter.this.destination) .markConsumedFailure(); throw new RuntimeException( - "Rocket Message hasn't been processed successfully. Caused by ", + "RocketMQ Message hasn't been processed successfully. Caused by ", e); } }