[WIP] [OSPP] [2.2.x] feat: Add outlier decection function (#3464)

* add outlier decection function
pull/3449/head^2
why-ohh 1 year ago committed by GitHub
parent 181ff019f0
commit f42f8b8915
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-alibaba-examples</artifactId>
<version>${revision}</version>
<relativePath>../../../../pom.xml</relativePath>
</parent>
<artifactId>outlierdetection-service-consumer</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-starter-alibaba-governance-routing</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

@ -0,0 +1,39 @@
/*
* Copyright 2022-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.example;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.cloud.openfeign.EnableFeignClients;
/**
* @author xqw
*/
@EnableFeignClients
@EnableDiscoveryClient
@SpringBootApplication
public class OutlierDetectionConsumerApplication {
public static void main(String[] args) {
SpringApplication.run(OutlierDetectionConsumerApplication.class);
}
}

@ -0,0 +1,32 @@
/*
* Copyright 2022-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.example.api;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
/**
* @author xqw
*/
@FeignClient(name = "outlierdetection-service-provider-example")
public interface FeignService {
@GetMapping("/test")
String test();
}

@ -0,0 +1,42 @@
/*
* Copyright 2022-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.example.controller;
import javax.annotation.Resource;
import com.alibaba.cloud.example.api.FeignService;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* @author xqw
*/
@RestController
public class OutlierDetectionController {
@Resource
private FeignService feignService;
@GetMapping("/test")
public String test() {
return feignService.test();
}
}

@ -0,0 +1,22 @@
server:
port: 17070
spring:
application:
name: outlierdetection-service-consumer-example
cloud:
nacos:
discovery:
server-addr: 127.0.0.1:8848
governance:
routing:
# enable outlier detected
enableOutlierDetected: true
# 最小健康比,摘除上限
minHealthPercent: 0.5
# 恢复时间间隔
recoverInterval: 3000
# 恢复时间累加值
baseEjectionTime: 3000

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-alibaba-examples</artifactId>
<version>${revision}</version>
<relativePath>../../../../pom.xml</relativePath>
</parent>
<artifactId>outlierdetection-service-provider-1</artifactId>
<name>Spring Cloud Governance OutlierDetection Service Provider Example</name>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

@ -0,0 +1,50 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.example;
import com.alibaba.cloud.nacos.registry.NacosRegistration;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* @author xqw
*/
@RestController
@EnableDiscoveryClient
@SpringBootApplication
public class OutlierDetectionProvider1Application {
public static void main(String[] args) {
SpringApplication.run(OutlierDetectionProvider1Application.class, args);
}
@Autowired
NacosRegistration nacosRegistration;
@GetMapping("/test")
public String test() {
throw new RuntimeException("mock service-provider-1 exception.");
}
}

@ -0,0 +1,11 @@
server:
port: 17071
spring:
application:
name: outlierdetection-service-provider-example
cloud:
nacos:
discovery:
server-addr: 127.0.0.1:8848

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-alibaba-examples</artifactId>
<version>${revision}</version>
<relativePath>../../../../pom.xml</relativePath>
</parent>
<artifactId>outlierdetection-service-provider-2</artifactId>
<name>Spring Cloud Governance OutlierDetection Service Provider Example</name>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

@ -0,0 +1,52 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.example;
import com.alibaba.cloud.nacos.registry.NacosRegistration;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* @author xqw
*/
@RestController
@EnableDiscoveryClient
@SpringBootApplication
public class OutlierDetectionProvider2Application {
public static void main(String[] args) {
SpringApplication.run(OutlierDetectionProvider2Application.class, args);
}
@Autowired
NacosRegistration nacosRegistration;
@GetMapping("/test")
public String test() {
String host = nacosRegistration.getHost();
int port = nacosRegistration.getPort();
return "Route in " + host + ": " + port + ".";
}
}

@ -0,0 +1,11 @@
server:
port: 17072
spring:
application:
name: outlierdetection-service-provider-example
cloud:
nacos:
discovery:
server-addr: 127.0.0.1:8848

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-alibaba-examples</artifactId>
<version>${revision}</version>
<relativePath>../../../../pom.xml</relativePath>
</parent>
<artifactId>outlierdetection-service-provider-3</artifactId>
<name>Spring Cloud Governance OutlierDetection Service Provider Example</name>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.cloud</groupId>
<artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

@ -0,0 +1,52 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.example;
import com.alibaba.cloud.nacos.registry.NacosRegistration;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* @author xqw
*/
@RestController
@EnableDiscoveryClient
@SpringBootApplication
public class OutlierDetectionProvider3Application {
public static void main(String[] args) {
SpringApplication.run(OutlierDetectionProvider3Application.class, args);
}
@Autowired
NacosRegistration nacosRegistration;
@GetMapping("/test")
public String test() {
String host = nacosRegistration.getHost();
int port = nacosRegistration.getPort();
return "Route in " + host + ": " + port + ". ";
}
}

@ -0,0 +1,11 @@
server:
port: 17073
spring:
application:
name: outlierdetection-service-provider-example
cloud:
nacos:
discovery:
server-addr: 127.0.0.1:8848

@ -0,0 +1,5 @@
curl 127.0.0.1:17070/test
# curl 127.0.0.1:17071/test
# curl 127.0.0.1:17072/test

@ -63,6 +63,10 @@
<module>governance-example/label-routing-example/consumer-example</module>
<module>governance-example/label-routing-example/default-provider-version-example</module>
<module>governance-example/label-routing-example/provider-version-example</module>
<module>governance-example/label-routing-example/outlierdetection-example/outlierdetection-servcie-consumer</module>
<module>governance-example/label-routing-example/outlierdetection-example/outlierdetection-service-provider-1</module>
<module>governance-example/label-routing-example/outlierdetection-example/outlierdetection-service-provider-2</module>
<module>governance-example/label-routing-example/outlierdetection-example/outlierdetection-service-provider-3</module>
<module>governance-example/authentication-example/istio-authentication-provider-mvc-example</module>
<module>governance-example/authentication-example/istio-authentication-provider-webflux-example</module>

@ -55,6 +55,18 @@
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.alibaba.middleware</groupId>
<artifactId>metrics-core-api</artifactId>
<version>2.0.6</version>
</dependency>
<dependency>
<groupId>com.alibaba.middleware</groupId>
<artifactId>metrics-core-impl</artifactId>
<version>2.0.6</version>
</dependency>
</dependencies>
</project>

@ -40,6 +40,14 @@ public class RoutingProperties {
*/
private String rule;
private boolean enableOutlierDetected;
private long baseEjectionTime;
private double minHealthPercent;
private int recoverInterval;
@PostConstruct
public void init() {
if (StringUtils.isEmpty(rule)) {
@ -47,6 +55,38 @@ public class RoutingProperties {
}
}
public boolean isEnableOutlierDetected() {
return enableOutlierDetected;
}
public void setEnableOutlierDetected(boolean enableOutlierDetected) {
this.enableOutlierDetected = enableOutlierDetected;
}
public long getBaseEjectionTime() {
return baseEjectionTime;
}
public void setBaseEjectionTime(long baseEjectionTime) {
this.baseEjectionTime = baseEjectionTime;
}
public double getMinHealthPercent() {
return minHealthPercent;
}
public void setMinHealthPercent(double minHealthPercent) {
this.minHealthPercent = minHealthPercent;
}
public int getRecoverInterval() {
return recoverInterval;
}
public void setRecoverInterval(int recoverInterval) {
this.recoverInterval = recoverInterval;
}
public String getRule() {
return rule;
}

@ -0,0 +1,248 @@
/*
* Copyright 2022-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.routing.cache;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import com.alibaba.cloud.routing.model.ServiceInstanceInfo;
import com.alibaba.nacos.api.naming.pojo.Instance;
/**
* @author xqw
* @author 550588941@qq.com
*/
public final class GlobalInstanceStatusListCache {
private GlobalInstanceStatusListCache() {
}
/**
* Global instance cache.
* String: service name.
* String: ip + port.
* ServiceInstanceInfo: .
*/
private static final Map<String, Map<String, ServiceInstanceInfo>> globalServiceCache = new ConcurrentHashMap<>();
/**
* Set instance cache.
*
* @param targetName service name.
* @param instance service instance object {ip + port}.
* @param sif service instance info.
*/
public static void set(String targetName, Instance instance,
ServiceInstanceInfo sif) {
if (globalServiceCache.isEmpty()) {
Map<String, ServiceInstanceInfo> instanceInfoMap = new ConcurrentHashMap<>();
instanceInfoMap.put(instance.getIp() + ":" + instance.getPort(), sif);
globalServiceCache.put(targetName, instanceInfoMap);
}
else {
globalServiceCache.forEach((serviceName, instanceMap) -> {
Map<String, ServiceInstanceInfo> instanceInfoMap;
if (serviceName == targetName) {
// serviceName is exist.
instanceInfoMap = globalServiceCache.get(targetName);
instanceInfoMap.put(instance.getIp() + ":" + instance.getIp(), sif);
}
else {
instanceInfoMap = new ConcurrentHashMap<>();
instanceInfoMap.put(instance.getIp() + ":" + instance.getPort(), sif);
}
globalServiceCache.put(targetName, instanceInfoMap);
});
}
}
/**
* Check instance in cache.
*
* @param target target.
* @param instance instance object.
* @return boolean
*/
public static boolean checkContainersInstance(String target, Instance instance) {
AtomicBoolean flag = new AtomicBoolean(false);
Map<String, ServiceInstanceInfo> instanceInfoMap = globalServiceCache.get(target);
if (instanceInfoMap.isEmpty()) {
return flag.get();
}
else {
instanceInfoMap.forEach((instanceName, sif) -> {
flag.set(instanceName == instance.getPort() + ":" + instance.getIp());
});
}
return flag.get();
}
/**
* Return GlobalCache Object.
* @return return global instance cache
*/
public static Map<String, Map<String, ServiceInstanceInfo>> getAll() {
return globalServiceCache;
}
/**
* Get service instance list by service name.
* @param targetServiceName service name.
* @return instance list.
*/
public static Map<String, ServiceInstanceInfo> getInstanceByServiceName(
String targetServiceName) {
Map<String, ServiceInstanceInfo> instanceInfoMap = globalServiceCache.get(targetServiceName);
if (instanceInfoMap.isEmpty()) {
return null;
}
return instanceInfoMap;
}
/**
* Get instance info by global cache.
* @param instanceName instance name
* @return sif object
*/
public static ServiceInstanceInfo getInstanceByInstanceName(String instanceName) {
AtomicReference<ServiceInstanceInfo> sif = null;
globalServiceCache.keySet().forEach((serviceName) -> {
Map<String, ServiceInstanceInfo> instanceInfoMap = globalServiceCache.get(serviceName);
sif.set(instanceInfoMap.get(instanceName));
});
return sif.get();
}
/**
* Get no health instance list.
*
* @return list
*/
public static List<ServiceInstanceInfo> getCalledErrorInstance() {
List<ServiceInstanceInfo> res = new ArrayList<>();
globalServiceCache.forEach((serviceName, instanceMap) -> {
Map<String, ServiceInstanceInfo> instanceInfoMap = globalServiceCache.get(serviceName);
instanceInfoMap.forEach((instanceName, sif) -> {
ServiceInstanceInfo serviceInstanceInfo = instanceInfoMap.get(instanceName);
if (serviceInstanceInfo.getConsecutiveErrors() != null) {
res.add(serviceInstanceInfo);
}
});
});
return res;
}
/**
* getServiceUpperLimitRatioNum.
*
* @param targetServiceName target service name.
* @param minHealthPercent minHealthPercent
* @return max remove instance num
*/
public static int getServiceUpperLimitRatioNum(String targetServiceName,
double minHealthPercent) {
int serviceInstanceTotal;
Map<String, ServiceInstanceInfo> instanceInfoMap = globalServiceCache.get(targetServiceName);
serviceInstanceTotal = instanceInfoMap.size();
return (int) Math.floor(serviceInstanceTotal * minHealthPercent);
}
/**
* Get all instance nums.
* @param targetServiceName target service name.
* @return remove instance num
*/
public static int getInstanceNumByTargetServiceName(String targetServiceName) {
int serviceInstanceTotal;
Map<String, ServiceInstanceInfo> instanceInfoMap = globalServiceCache.get(targetServiceName);
serviceInstanceTotal = instanceInfoMap.size();
return serviceInstanceTotal;
}
/**
* Get no health nums.
* @param targetServiceName target service name.
* @return remove instance num
*/
public static int getRemoveInstanceNum(String targetServiceName) {
AtomicInteger serviceInstanceTotal = new AtomicInteger();
Map<String, ServiceInstanceInfo> instanceInfoMap = globalServiceCache.get(targetServiceName);
instanceInfoMap.forEach((instanceName, sif) -> {
ServiceInstanceInfo serviceInstanceInfo = instanceInfoMap.get(instanceName);
if (!(serviceInstanceInfo.isStatus())) {
serviceInstanceTotal.getAndIncrement();
}
});
return serviceInstanceTotal.get();
}
public static void setInstanceInfoByInstanceNames(ServiceInstanceInfo sif) {
String instanceName = sif.getInstance().getIp() + ":"
+ sif.getInstance().getPort();
globalServiceCache.forEach((serviceName, instanceMap) -> {
Map<String, ServiceInstanceInfo> instanceInfoMap = globalServiceCache.get(serviceName);
instanceInfoMap.put(instanceName, sif);
});
}
@Override
public String toString() {
StringBuilder stringBuilder = new StringBuilder();
for (String serviceName : globalServiceCache.keySet()) {
stringBuilder.append("Service: ").append(serviceName).append("\n");
Map<String, ServiceInstanceInfo> innerMap = globalServiceCache.get(serviceName);
for (String instanceId : innerMap.keySet()) {
ServiceInstanceInfo instanceInfo = innerMap.get(instanceId);
stringBuilder.append(" Instance: ").append(instanceInfo).append("\n");
}
stringBuilder.append("\n");
}
return stringBuilder.toString();
}
}

@ -0,0 +1,53 @@
/*
* Copyright 2022-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.routing.configuration;
import com.alibaba.cloud.routing.decorator.OutlierDetectionFeignClientDecorator;
import com.alibaba.cloud.routing.recover.OutlierDetectionRecover;
import feign.Client;
import org.springframework.cloud.netflix.ribbon.SpringClientFactory;
import org.springframework.cloud.openfeign.ribbon.CachingSpringLoadBalancerFactory;
import org.springframework.cloud.openfeign.ribbon.LoadBalancerFeignClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* @author xqw
* @author 550588941@qq.com
*/
@Configuration(proxyBeanMethods = false)
public class OutlierDetectionConfiguration {
@Bean
public Client outlierDetectionFeignClientDecorator(
CachingSpringLoadBalancerFactory cachingSpringLoadBalancerFactory,
SpringClientFactory clientFactory) {
return new LoadBalancerFeignClient(
new OutlierDetectionFeignClientDecorator(new Client.Default(null, null)),
cachingSpringLoadBalancerFactory, clientFactory);
}
@Bean
public OutlierDetectionRecover outlierDetectionRecover() {
return new OutlierDetectionRecover();
}
}

@ -0,0 +1,87 @@
/*
* Copyright 2022-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.routing.constants;
/**
* @author xqw
*/
public final class OutlierDetectionConstants {
private OutlierDetectionConstants() {
}
/**
* Instance error called max.
*/
public static final int instanceErrorCalledMax = 5;
/**
* Response Code.
*/
public enum ResponseCode {
/**
* Access successful.
*/
_200(200, "Access successful"),
/**
* The data types do not match.
*/
_400(400, "The data types do not match"),
/**
* The server rejected the request.
*/
_403(403, "The server rejected the request"),
/**
* The server could not find the requested web page and entered the link incorrectly.
*/
_404(404, "The server could not find the requested web page and entered the link incorrectly"),
/**
* The server encountered an error and could not complete the request.
*/
_500(500, "The server encountered an error and could not complete the request"),
/**
* The server, acting as a gateway or proxy, receives an invalid response from the upstream server.
*/
_502(502, "The server, acting as a gateway or proxy, receives an invalid response from the upstream server");
/**
* Code.
*/
private Integer code;
/**
* info.
*/
private String info;
ResponseCode(Integer code, String info) {
this.code = code;
this.info = info;
}
public Integer getCode() {
return code;
}
public String getInfo() {
return info;
}
}
}

@ -0,0 +1,92 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.routing.decorator;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import com.alibaba.cloud.routing.cache.GlobalInstanceStatusListCache;
import com.alibaba.cloud.routing.constants.OutlierDetectionConstants;
import com.alibaba.cloud.routing.model.ServiceInstanceInfo;
import feign.Client;
import feign.Request;
import feign.Response;
/**
* @author xqw
* @author 550588941@qq.com
*/
public class OutlierDetectionFeignClientDecorator implements Client {
private final Client delegate;
public OutlierDetectionFeignClientDecorator(Client delegate) {
this.delegate = delegate;
}
@Override
public Response execute(Request request, Request.Options options) throws IOException {
Response response = delegate.execute(request, options);
parseResponse(request, response);
return response;
}
private void parseResponse(Request request, Response response)
throws MalformedURLException {
URL url = new URL(request.url());
String instanceName = url.getHost() + ":" + url.getPort();
ServiceInstanceInfo sif = GlobalInstanceStatusListCache
.getInstanceByInstanceName(instanceName);
// The use of metrics is still under investigation.
// FastCompass counter = MetricManager.getFastCompass("sca-instance"
// , MetricName.build(instanceName + ".counter"));
// long start = System.currentTimeMillis();
if (response.status() == OutlierDetectionConstants.ResponseCode._500.getCode()) {
// long duration = System.currentTimeMillis() - start;
// counter.record(duration,"error");
// sif.setCompass(counter);
AtomicInteger consecutiveErrors = sif.getConsecutiveErrors();
if (Objects.isNull(consecutiveErrors)) {
consecutiveErrors = new AtomicInteger(1);
} else {
int andIncrement = sif.getConsecutiveErrors().get();
andIncrement ++;
consecutiveErrors = new AtomicInteger(andIncrement);
}
sif.setConsecutiveErrors(consecutiveErrors);
sif.setRemoveTime(System.currentTimeMillis());
System.err.println("设置服务错误次数之后的全局缓存数据:" + GlobalInstanceStatusListCache.getAll());
}
else {
// long duration = System.currentTimeMillis() - start;
// counter.record(duration, "success");
// sif.setCompass(counter);
}
}
}

@ -0,0 +1,65 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.routing.decorator;
import java.io.IOException;
import java.io.Reader;
import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import feign.FeignException;
import feign.Response;
import feign.Util;
import org.springframework.beans.factory.ObjectFactory;
import org.springframework.boot.autoconfigure.http.HttpMessageConverters;
import org.springframework.cloud.openfeign.support.SpringDecoder;
import org.springframework.stereotype.Component;
/**
* @author xqw
* @author 550588941@qq.com
*/
@Component
public class OutlierDetectionFeignResponseDecoder extends SpringDecoder {
public OutlierDetectionFeignResponseDecoder(
ObjectFactory<HttpMessageConverters> messageConverters) {
super(messageConverters);
}
@Override
public Object decode(Response response, Type type)
throws IOException, FeignException {
Reader reader = response.body().asReader(StandardCharsets.UTF_8);
if (Objects.isNull(Util.toString(reader))) {
return super.decode(
response.toBuilder()
.body(Util.toString(reader), StandardCharsets.UTF_8).build(),
type);
}
return Util.toString(reader);
}
}

@ -0,0 +1,118 @@
/*
* Copyright 2022-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.routing.model;
import java.util.concurrent.atomic.AtomicInteger;
import com.alibaba.metrics.FastCompass;
import com.alibaba.nacos.api.naming.pojo.Instance;
/**
* @author xqw
* @author 550588941@qq.com
*/
public class ServiceInstanceInfo {
/**
* Current instance info.
*/
private Instance instance;
/**
* Metrics data.
*/
private FastCompass compass;
/**
* Instance remove time, It increases according to the number of removals.
*/
private Long removeTime;
/**
* The percentage of services removed over a period of time. get data from metrics.
*/
private double removalRatio;
/**
* Instance status.
*/
private boolean status;
private AtomicInteger consecutiveErrors;
public AtomicInteger getConsecutiveErrors() {
return consecutiveErrors;
}
public void setConsecutiveErrors(AtomicInteger consecutiveErrors) {
this.consecutiveErrors = consecutiveErrors;
}
public boolean isStatus() {
return status;
}
public void setStatus(boolean status) {
this.status = status;
}
public Long getRemoveTime() {
return removeTime;
}
public void setRemoveTime(Long removeTime) {
this.removeTime = removeTime;
}
public double getRemovalRatio() {
return removalRatio;
}
public void setRemovalRatio(double removalRatio) {
this.removalRatio = removalRatio;
}
public Instance getInstance() {
return instance;
}
public void setInstance(Instance instance) {
this.instance = instance;
}
public FastCompass getCompass() {
return compass;
}
public void setCompass(FastCompass compass) {
this.compass = compass;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("ServiceInstanceInfo{");
sb.append("instance=").append(instance);
sb.append(", compass=").append(compass);
sb.append(", removeTime=").append(removeTime);
sb.append(", removalRatio=").append(removalRatio);
sb.append(", status=").append(status);
sb.append(", consecutiveErrors=").append(consecutiveErrors);
sb.append('}');
return sb.toString();
}
}

@ -0,0 +1,119 @@
/*
* Copyright 2022-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.routing.recover;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;
import com.alibaba.cloud.routing.RoutingProperties;
import com.alibaba.cloud.routing.cache.GlobalInstanceStatusListCache;
import com.alibaba.cloud.routing.model.ServiceInstanceInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
/**
* @author xqw
*/
public class OutlierDetectionRecover {
@Autowired
private RoutingProperties routingProperties;
private static final Logger log = LoggerFactory
.getLogger(OutlierDetectionRecover.class);
// move to spring config
private static final boolean enabledInstanceRecoverTask = true;
public void updateInstanceStatus(String targetServiceName) {
if (enabledInstanceRecoverTask) {
log.info(
"The instance recover task is started. Please pay attention to the service status.");
Timer timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
recover(targetServiceName);
}
}, 0, routingProperties.getRecoverInterval());
}
}
/**
* Recover instance and error countsetting consecutiveErrors And When.
* consecutiveErrors to 5, down this forever.
*/
private void recover(String targetServiceName) {
List<ServiceInstanceInfo> calledErrorInstance = GlobalInstanceStatusListCache
.getCalledErrorInstance();
double minHealthPercent = routingProperties.getMinHealthPercent();
int removeUpperLimitNum = GlobalInstanceStatusListCache
.getServiceUpperLimitRatioNum(targetServiceName, minHealthPercent);
int unHealthInstanceNum = GlobalInstanceStatusListCache.getRemoveInstanceNum(targetServiceName);
long baseEjectionTime = routingProperties.getBaseEjectionTime();
log.info(
"最大移除上限数:" + removeUpperLimitNum +
",不健康实例数:" + unHealthInstanceNum +
",缓存中 " + targetServiceName + " 的服务实例数:"
+ GlobalInstanceStatusListCache.getInstanceNumByTargetServiceName(targetServiceName) );
for (ServiceInstanceInfo sif : calledErrorInstance) {
// 判断错误率是否合格? use metrics!
if (sif.getConsecutiveErrors().get() == 2) {
log.error("错误次数达到上限,进入摘除逻辑...");
// 判断是否达到上限比
if (!(removeUpperLimitNum == unHealthInstanceNum)) {
log.info("通过摘除上限比判断,准备摘除...");
// 摘除
sif.setStatus(false);
sif.setRemoveTime(System.currentTimeMillis());
log.info("成功摘除:" + GlobalInstanceStatusListCache.getAll());
}
GlobalInstanceStatusListCache.setInstanceInfoByInstanceNames(sif);
}
else {
log.info("错误率条件不成立,进入实例恢复....");
System.out.println(sif);
// 不健康 当前的时间 - 上次摘除的时间 == 恢复时间
long current = System.currentTimeMillis();
long removeTime = sif.getRemoveTime();
if ((current - removeTime) > baseEjectionTime) {
// 恢复实例 设置健康状态 status ——> true
sif.setStatus(true);
}
GlobalInstanceStatusListCache.setInstanceInfoByInstanceNames(sif);
}
}
}
}

@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ThreadLocalRandom;
import javax.servlet.http.HttpServletRequest;
@ -33,10 +34,14 @@ import com.alibaba.cloud.nacos.NacosDiscoveryProperties;
import com.alibaba.cloud.nacos.NacosServiceManager;
import com.alibaba.cloud.nacos.ribbon.NacosServer;
import com.alibaba.cloud.routing.RoutingProperties;
import com.alibaba.cloud.routing.cache.GlobalInstanceStatusListCache;
import com.alibaba.cloud.routing.model.ServiceInstanceInfo;
import com.alibaba.cloud.routing.publish.TargetServiceChangedPublisher;
import com.alibaba.cloud.routing.recover.OutlierDetectionRecover;
import com.alibaba.cloud.routing.repository.RoutingDataRepository;
import com.alibaba.cloud.routing.util.ConditionMatchUtil;
import com.alibaba.cloud.routing.util.LoadBalanceUtil;
import com.alibaba.nacos.api.exception.NacosException;
import com.alibaba.nacos.api.naming.NamingService;
import com.alibaba.nacos.api.naming.pojo.Instance;
import com.netflix.loadbalancer.AbstractServerPredicate;
@ -104,6 +109,9 @@ public class RoutingLoadBalanceRule extends PredicateBasedRule {
@Autowired
private RoutingDataRepository routingDataRepository;
@Autowired
private OutlierDetectionRecover recover;
@Autowired
private RoutingProperties routingProperties;
@ -121,19 +129,24 @@ public class RoutingLoadBalanceRule extends PredicateBasedRule {
final HashMap<String, List<MatchService>> routeData = routingDataRepository
.getRouteRule(targetServiceName);
if (routeData == null) {
// if outlier detection function is enabled.
if (routingProperties.isEnableOutlierDetected()) {
return chooseServerByOutlier(targetServiceName);
}
return LoadBalanceUtil.loadBalanceByOrdinaryRule(loadBalancer, key,
routingProperties.getRule());
}
// Get instances from register-center.
String group = this.nacosDiscoveryProperties.getGroup();
final NamingService namingService = nacosServiceManager.getNamingService();
final List<Instance> instances = namingService
.selectInstances(targetServiceName, group, true);
if (CollectionUtils.isEmpty(instances)) {
LOG.warn("no instance in service {} ", targetServiceName);
return null;
}
List<Instance> instances = getInsanceList(targetServiceName);
// set instance cache
setGlobalInstanceCache(targetServiceName, instances);
// instance recover
recover.updateInstanceStatus(targetServiceName);
// Filter by route rules,the result will be kept in versionSet and weightMap.
HashSet<String> versionSet = new HashSet<>();
@ -173,11 +186,13 @@ public class RoutingLoadBalanceRule extends PredicateBasedRule {
instanceMap.put(version, instanceList);
}
}
return chooseServerByWeight(instanceMap, fallbackWeightMap, weightArray);
return chooseServerByWeight(targetServiceName, instanceMap,
fallbackWeightMap, weightArray);
}
// Routing with Weight algorithm.
return chooseServerByWeight(instanceMap, weightMap, weightArray);
return chooseServerByWeight(targetServiceName, instanceMap, weightMap,
weightArray);
}
catch (Exception e) {
@ -186,6 +201,84 @@ public class RoutingLoadBalanceRule extends PredicateBasedRule {
}
}
private Server chooseServerByOutlier(String targetServiceName) {
try {
List<Instance> insanceList = getInsanceList(targetServiceName);
// set instance cache
setGlobalInstanceCache(targetServiceName, insanceList);
// instance recover
recover.updateInstanceStatus(targetServiceName);
}
catch (Exception e) {
throw new RuntimeException(e);
}
// return instance called for health instance list
return new NacosServer(Objects.requireNonNull(getInstance(targetServiceName)));
}
/**
* Get instance from global instance cache.
*
* @param targetServiceName service name.
*/
private Instance getInstance(String targetServiceName) {
Map<String, ServiceInstanceInfo> instanceByServiceName = GlobalInstanceStatusListCache
.getInstanceByServiceName(targetServiceName);
List<ServiceInstanceInfo> instanceInfos = new ArrayList<>();
instanceByServiceName.forEach((k1, v1) -> {
if (v1.isStatus()) {
instanceInfos.add(v1);
}
});
if (instanceInfos.isEmpty()) {
LOG.warn(
"The number of available instances in the global service instance cache list is 0, "
+ "no instance called. please check instance status!");
System.out.println(GlobalInstanceStatusListCache.getAll());
return null;
}
int randomInt = ThreadLocalRandom.current().nextInt(instanceInfos.size());
return instanceInfos.get(randomInt).getInstance();
}
/**
* Get instance from global instance cache.
* @param targetServiceName service name.
*/
private List<Instance> getInstanceList(String targetServiceName) {
Map<String, ServiceInstanceInfo> instanceByServiceName = GlobalInstanceStatusListCache
.getInstanceByServiceName(targetServiceName);
List<Instance> instances = new ArrayList<>();
instanceByServiceName.forEach((k1, v1) -> {
if (v1.isStatus()) {
instances.add(v1.getInstance());
}
});
if (instances.isEmpty()) {
LOG.warn(
"The number of available instances in the global service instance cache list is 0, "
+ "no instance called. please check instance status!");
System.out.println(GlobalInstanceStatusListCache.getAll());
return null;
}
return instances;
}
@Override
public AbstractServerPredicate getPredicate() {
return this.predicate;
@ -365,8 +458,12 @@ public class RoutingLoadBalanceRule extends PredicateBasedRule {
}
}
private Server chooseServerByWeight(final HashMap<String, List<Instance>> instanceMap,
private Server chooseServerByWeight(String targetServiceName,
final HashMap<String, List<Instance>> instanceMap,
final HashMap<String, Integer> weightMap, final double[] weightArray) {
getInstanceList(targetServiceName);
int index = 0;
double sum = 0.0D;
List<Instance> instances = new ArrayList<>();
@ -396,4 +493,36 @@ public class RoutingLoadBalanceRule extends PredicateBasedRule {
return new NacosServer(instances.get(chooseServiceIndex));
}
/**
* Set instance cache.
*/
private void setGlobalInstanceCache(String target, List<Instance> instances) {
for (Instance instance : instances) {
if (!GlobalInstanceStatusListCache.checkContainersInstance(target,
instance)) {
ServiceInstanceInfo serviceInstanceInfo = new ServiceInstanceInfo();
serviceInstanceInfo.setInstance(instance);
serviceInstanceInfo.setStatus(true);
serviceInstanceInfo.setRemovalRatio(0.0);
GlobalInstanceStatusListCache.set(target, instance, serviceInstanceInfo);
}
}
}
private List<Instance> getInsanceList(String targetServiceName)
throws NacosException {
// Get instances from register-center.
String group = this.nacosDiscoveryProperties.getGroup();
NamingService namingService = nacosServiceManager.getNamingService();
List<Instance> instances = namingService.selectInstances(targetServiceName, group,
true);
if (CollectionUtils.isEmpty(instances)) {
LOG.warn("no instance in service {} ", targetServiceName);
return null;
}
return instances;
}
}

@ -2,4 +2,5 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
com.alibaba.cloud.routing.RoutingAutoConfiguration,\
com.alibaba.cloud.routing.feign.RoutingFeignInterceptorAutoConfiguration,\
com.alibaba.cloud.routing.ribbon.RoutingLoadBalanceRuleAutoConfiguration,\
com.alibaba.cloud.routing.RoutingPropertiesAutoConfiguration
com.alibaba.cloud.routing.RoutingPropertiesAutoConfiguration,\
com.alibaba.cloud.routing.configuration.OutlierDetectionConfiguration

Loading…
Cancel
Save