版本

springboot 2.0.1.RELEASE
springcloud Finchley.RC1

问题

看程序猿 dd 的博客 http://blog.didispace.com/Spring-Cloud-Config-Server-ip-change-problem/

容器中 config-server 重启或迁移导致ip port 变动, config-client 依旧使用老的(从eureka 第一次取到)的config-server 地址.导致 bus 无法刷新配置,healbeat config 失败.

进一步发现问题

ConfigServicePropertySourceLocator.java

private Environment getRemoteEnvironment(RestTemplate restTemplate, ConfigClientProperties properties,
                                             String label, String state) {
        String path = "/{name}/{profile}";
        String name = properties.getName();
        String profile = properties.getProfile();
        String token = properties.getToken();
        String uri = properties.getRawUri();

        Object[] args = new String[] { name, profile };
        if (StringUtils.hasText(label)) {
            args = new String[] { name, profile, label };
            path = path + "/{label}";
        }
        ResponseEntity<Environment> response = null;

        try {
            HttpHeaders headers = new HttpHeaders();
            if (StringUtils.hasText(token)) {
                headers.add(TOKEN_HEADER, token);
            }
            if (StringUtils.hasText(state)) { //TODO: opt in to sending state?
                headers.add(STATE_HEADER, state);
            }
            final HttpEntity<Void> entity = new HttpEntity<>((Void) null, headers);
                        //uri 是直接从配置中拿的
            response = restTemplate.exchange(uri + path, HttpMethod.GET,
                    entity, Environment.class, args);
        }
        catch (HttpClientErrorException e) {
            if (e.getStatusCode() != HttpStatus.NOT_FOUND) {
                throw e;
            }
        }

        if (response == null || response.getStatusCode() != HttpStatus.OK) {
            return null;
        }
        Environment result = response.getBody();
        return result;
    }

虽然从上面代码可以看出,config-client 获取 config-server 地址,是从配置中获取.

但是有下面代码.

@ConditionalOnProperty(value = "spring.cloud.config.discovery.enabled", matchIfMissing = false)
@Configuration
@Import({ UtilAutoConfiguration.class })
@EnableDiscoveryClient
public class DiscoveryClientConfigServiceBootstrapConfiguration {

    private static Log logger = LogFactory
            .getLog(DiscoveryClientConfigServiceBootstrapConfiguration.class);

    @Autowired
    private ConfigClientProperties config;

    @Autowired
    private ConfigServerInstanceProvider instanceProvider;

    private HeartbeatMonitor monitor = new HeartbeatMonitor();

    @Bean
    public ConfigServerInstanceProvider configServerInstanceProvider(
            DiscoveryClient discoveryClient) {
        return new ConfigServerInstanceProvider(discoveryClient);
    }
        
    @EventListener(ContextRefreshedEvent.class)
    public void startup(ContextRefreshedEvent event) {
                //刷新配置
        refresh();
    }

    @EventListener(HeartbeatEvent.class)
    public void heartbeat(HeartbeatEvent event) {
        if (monitor.update(event.getValue())) {
                        //刷新配置
            refresh();
        }
    }

    private void refresh() {
        try {
            String serviceId = this.config.getDiscovery().getServiceId();
            ServiceInstance server = this.instanceProvider
                    .getConfigServerInstance(serviceId);
            String url = getHomePage(server);
            if (server.getMetadata().containsKey("password")) {
                String user = server.getMetadata().get("user");
                user = user == null ? "user" : user;
                this.config.setUsername(user);
                String password = server.getMetadata().get("password");
                this.config.setPassword(password);
            }
            if (server.getMetadata().containsKey("configPath")) {
                String path = server.getMetadata().get("configPath");
                if (url.endsWith("/") && path.startsWith("/")) {
                    url = url.substring(0, url.length() - 1);
                }
                url = url + path;
            }
            this.config.setUri(url);
        }
        catch (Exception ex) {
            if (config.isFailFast()) {
                throw ex;
            }
            else {
                logger.warn("Could not locate configserver via discovery", ex);
            }
        }
    }

    private String getHomePage(ServiceInstance server) {
        return server.getUri().toString() + "/";
    }

}

两个事件,ContextRefreshedEvent 和 HeartbeatEvent 都会触发刷新 ConfigClientProperties 中的 uri 地址. HeartbeatEvent 的触发是在 CloudEurekaClient.java

    @Override
    protected void onCacheRefreshed() {
        super.onCacheRefreshed();

        if (this.cacheRefreshedCount != null) { //might be called during construction and will be null
            long newCount = this.cacheRefreshedCount.incrementAndGet();
            log.trace("onCacheRefreshed called with count: " + newCount);
            this.publisher.publishEvent(new HeartbeatEvent(this, newCount));
        }
    }

即DiscoveryClient.java 中

private boolean fetchRegistry(boolean forceFullRegistryFetch) {
        Stopwatch tracer = FETCH_REGISTRY_TIMER.start();

        try {
            // If the delta is disabled or if it is the first time, get all
            // applications
            Applications applications = getApplications();

            if (clientConfig.shouldDisableDelta()
                    || (!Strings.isNullOrEmpty(clientConfig.getRegistryRefreshSingleVipAddress()))
                    || forceFullRegistryFetch
                    || (applications == null)
                    || (applications.getRegisteredApplications().size() == 0)
                    || (applications.getVersion() == -1)) //Client application does not have latest library supporting delta
            {
                logger.info("Disable delta property : {}", clientConfig.shouldDisableDelta());
                logger.info("Single vip registry refresh property : {}", clientConfig.getRegistryRefreshSingleVipAddress());
                logger.info("Force full registry fetch : {}", forceFullRegistryFetch);
                logger.info("Application is null : {}", (applications == null));
                logger.info("Registered Applications size is zero : {}",
                        (applications.getRegisteredApplications().size() == 0));
                logger.info("Application version is -1: {}", (applications.getVersion() == -1));
                getAndStoreFullRegistry();
            } else {
                getAndUpdateDelta(applications);
            }
            applications.setAppsHashCode(applications.getReconcileHashCode());
            logTotalInstances();
        } catch (Throwable e) {
            logger.error(PREFIX + "{} - was unable to refresh its cache! status = {}", appPathIdentifier, e.getMessage(), e);
            return false;
        } finally {
            if (tracer != null) {
                tracer.stop();
            }
        }

        // Notify about cache refresh before updating the instance remote status
        onCacheRefreshed();

        // Update remote status based on refreshed data held in the cache
        updateInstanceRemoteStatus();

        // registry was fetched successfully, so return true
        return true;
    }

那么,我们是不是可以认为,当 eureka 同步注册信息的后,会触发事件,修改ConfigClientProperties的配置.

然后这个问题就已经解决了呢(只有当 eureka client 还没同步注册信息,且 config-server 的 ip port 变动,这时候有 bus 和 healbeat 才会出问题 ps:几率相当之低,可以忽略)

事情没有这么简单.

笔者变更 config-server 的地址后,debug config-client 发现,在上述DiscoveryClientConfigServiceBootstrapConfiguration.refresh 方法的时候,取到的 url, 依旧是老的可不用的 config-server 地址.

Spring Cloud Config Server迁移节点或容器化带来的问题原因,解决

图一是 DiscoveryClientConfigServiceBootstrapConfiguration.refresh() debug

图二是 DiscoveryClient.fetchRegistry debug

明显可以看到两个 CloudEurekaClient 不一致,图一叫9962,图二叫9926.

说明CloudEurekaClient 创建了两次.且 DiscoveryClientConfigServiceBootstrapConfiguration 中获得 config-server 地址的CloudEurekaClient是无效的.

再进一步深究

在 spring-cloud-config-client 的 spring.factories中

# Bootstrap components
org.springframework.cloud.bootstrap.BootstrapConfiguration=\
org.springframework.cloud.config.client.ConfigServiceBootstrapConfiguration,\
org.springframework.cloud.config.client.DiscoveryClientConfigServiceBootstrapConfiguration

DiscoveryClientConfigServiceBootstrapConfiguration 的初始化等级太高,ConfigServerInstanceProvider 持有的DiscoveryClient 无法修改,永远都是老对象.导致每次refresh 操作,获取的 config-server 地址都是老的.

解决方案

想了半天,没有找到不侵入的方式修改.只好魔改 spring-cloud-config-client 的代码.

修改DiscoveryClientConfigServiceBootstrapConfiguration.java

/*
 * Copyright 2013-2014 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.springframework.cloud.config.client;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cloud.client.ServiceInstance;
import org.springframework.cloud.client.discovery.DiscoveryClient;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.cloud.client.discovery.event.HeartbeatEvent;
import org.springframework.cloud.client.discovery.event.HeartbeatMonitor;
import org.springframework.cloud.commons.util.UtilAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;

/**
 * Bootstrap configuration for a config client that wants to lookup the config server via
 * discovery.
 *
 * @author Dave Syer
 */
@ConditionalOnProperty(value = "spring.cloud.config.discovery.enabled", matchIfMissing = false)
@Configuration
@Import({ UtilAutoConfiguration.class })
@EnableDiscoveryClient
public class DiscoveryClientConfigServiceBootstrapConfiguration {

    private static Log logger = LogFactory
            .getLog(DiscoveryClientConfigServiceBootstrapConfiguration.class);

    @Autowired
    private ConfigClientProperties config;

    @Autowired
    private ConfigServerInstanceProvider instanceProvider;

//    private HeartbeatMonitor monitor = new HeartbeatMonitor();

    @Bean
    public ConfigServerInstanceProvider configServerInstanceProvider(
            DiscoveryClient discoveryClient) {
        return new ConfigServerInstanceProvider(discoveryClient);
    }

    @EventListener(ContextRefreshedEvent.class)
    public void startup(ContextRefreshedEvent event) {
        refresh();
    }

//    @EventListener(HeartbeatEvent.class)
//    public void heartbeat(HeartbeatEvent event) {
//        if (monitor.update(event.getValue())) {
//            refresh();
//        }
//    }

    private void refresh() {
        try {
            String serviceId = this.config.getDiscovery().getServiceId();
            ServiceInstance server = this.instanceProvider
                    .getConfigServerInstance(serviceId);
            String url = getHomePage(server);
            if (server.getMetadata().containsKey("password")) {
                String user = server.getMetadata().get("user");
                user = user == null ? "user" : user;
                this.config.setUsername(user);
                String password = server.getMetadata().get("password");
                this.config.setPassword(password);
            }
            if (server.getMetadata().containsKey("configPath")) {
                String path = server.getMetadata().get("configPath");
                if (url.endsWith("/") && path.startsWith("/")) {
                    url = url.substring(0, url.length() - 1);
                }
                url = url + path;
            }
            this.config.setUri(url);
        }
        catch (Exception ex) {
            if (config.isFailFast()) {
                throw ex;
            }
            else {
                logger.warn("Could not locate configserver via discovery", ex);
            }
        }
    }

    private String getHomePage(ServiceInstance server) {
        return server.getUri().toString() + "/";
    }

}

添加 DiscoveryClientConfigServiceConfiguration.java

/*
 * Copyright 2013-2014 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.springframework.cloud.config.client;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cloud.client.ServiceInstance;
import org.springframework.cloud.client.discovery.DiscoveryClient;
import org.springframework.cloud.client.discovery.event.HeartbeatEvent;
import org.springframework.cloud.client.discovery.event.HeartbeatMonitor;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;

import java.util.List;


/**
 * 将HeartbeatEvent事件初始化后移.
 * 从BootstrapConfiguration移动到EnableAutoConfiguration中
 * @author superwen
 */
@ConditionalOnProperty(value = "spring.cloud.config.discovery.enabled", matchIfMissing = false)
@Configuration
public class DiscoveryClientConfigServiceConfiguration {

    private static Log logger = LogFactory
            .getLog(DiscoveryClientConfigServiceConfiguration.class);

    @Autowired
    private ConfigClientProperties config;

    @Autowired
    private DiscoveryClient discoveryClient;

    private HeartbeatMonitor monitor = new HeartbeatMonitor();


//    @EventListener(ContextRefreshedEvent.class)
//    public void startup(ContextRefreshedEvent event) {
//        refresh();
//    }

    @EventListener(HeartbeatEvent.class)
    public void heartbeat(HeartbeatEvent event) {
        if (monitor.update(event.getValue())) {
            refresh();
        }
    }

    private void refresh() {
        try {
            String serviceId = this.config.getDiscovery().getServiceId();
            logger.debug("Locating configserver (" + serviceId + ") via discovery");
            List<ServiceInstance> server = discoveryClient.getInstances(serviceId);
            if (server.isEmpty()) {
                throw new IllegalStateException(
                        "No instances found of configserver (" + serviceId + ")");
            }
            ServiceInstance instance = server.get(0);
            logger.debug(
                    "Located configserver (" + serviceId + ") via discovery: " + instance);
            String url = getHomePage(instance);
            if (instance.getMetadata().containsKey("password")) {
                String user = instance.getMetadata().get("user");
                user = user == null ? "user" : user;
                this.config.setUsername(user);
                String password = instance.getMetadata().get("password");
                this.config.setPassword(password);
            }
            if (instance.getMetadata().containsKey("configPath")) {
                String path = instance.getMetadata().get("configPath");
                if (url.endsWith("/") && path.startsWith("/")) {
                    url = url.substring(0, url.length() - 1);
                }
                url = url + path;
            }
            logger.debug("url=" + url);
            this.config.setUri(url);
        } catch (Exception ex) {
            if (config.isFailFast()) {
                throw ex;
            } else {
                logger.warn("Could not locate configserver via discovery", ex);
            }
        }
    }

    private String getHomePage(ServiceInstance server) {
        return server.getUri().toString() + "/";
    }

}

修改 spring.factories

# Auto Configure
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
org.springframework.cloud.config.client.ConfigClientAutoConfiguration,\
org.springframework.cloud.config.client.DiscoveryClientConfigServiceConfiguration

# Bootstrap components
org.springframework.cloud.bootstrap.BootstrapConfiguration=\
org.springframework.cloud.config.client.ConfigServiceBootstrapConfiguration,\
org.springframework.cloud.config.client.DiscoveryClientConfigServiceBootstrapConfiguration

后置 HeartbeatEvent 事件,将其获取 config-server 的地址.改为从新的 eurekaclient 中获取即可.