diff --git a/apollo-client/pom.xml b/apollo-client/pom.xml index 7a8e436a..ce9d7d3b 100644 --- a/apollo-client/pom.xml +++ b/apollo-client/pom.xml @@ -98,5 +98,10 @@ test + + io.kubernetes + client-java + 18.0.0 + diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/Kubernetes/KubernetesManager.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/Kubernetes/KubernetesManager.java new file mode 100644 index 00000000..130a29c1 --- /dev/null +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/Kubernetes/KubernetesManager.java @@ -0,0 +1,180 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.Kubernetes; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.Configuration; +import io.kubernetes.client.openapi.models.*; +import io.kubernetes.client.util.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.util.Map; + +@Service +public class KubernetesManager { + private ApiClient client; + private CoreV1Api coreV1Api; + + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + @PostConstruct + public void initClient() { + try { + client = Config.defaultClient(); + Configuration.setDefaultApiClient(client); + coreV1Api = new CoreV1Api(client); + + } catch (Exception e) { + String errorMessage = "Failed to initialize Kubernetes client: " + e.getMessage(); + log.error(errorMessage, e); + throw new RuntimeException(errorMessage, e); + } + } + + /** + * Creates a Kubernetes ConfigMap. + * + * @param configMapNamespace the namespace of the ConfigMap + * @param name the name of the ConfigMap + * @param data the data to be stored in the ConfigMap + * @return the name of the created ConfigMap + * @throws RuntimeException if an error occurs while creating the ConfigMap + */ + public String createConfigMap(String configMapNamespace, String name, Map data) { + if (configMapNamespace == null || configMapNamespace == "" || name == null || name == "") { + log.error("create config map failed due to null parameter"); + throw new RuntimeException("ConfigMap namespace and name cannot be null or empty"); + } + V1ConfigMap configMap = new V1ConfigMap().metadata(new V1ObjectMeta().name(name).namespace(configMapNamespace)).data(data); + try { + coreV1Api.createNamespacedConfigMap(configMapNamespace, configMap, null, null, null,null); + return name; + } catch (Exception e) { + log.error("create config map failed", e); + throw new RuntimeException("Failed to create ConfigMap: " + e.getMessage(), e); + } + } + + /** + * get value from config map + * @param configMapNamespace + * @param name config map name (appId) + * @return configMap data(all key-value pairs in config map) + */ + public String loadFromConfigMap(String configMapNamespace, String name) { + if (configMapNamespace == null || configMapNamespace.isEmpty() || name == null || name.isEmpty() || name == null || name.isEmpty()) { + log.error("参数不能为空"); + throw new RuntimeException(String + .format("参数不能为空, configMapNamespace: %s, name: %s", configMapNamespace, name)); + } + try { + V1ConfigMap configMap = coreV1Api.readNamespacedConfigMap(name, configMapNamespace, null); + if (configMap == null) { + log.error("ConfigMap不存在"); + throw new RuntimeException(String + .format("ConfigMap不存在, configMapNamespace: %s, name: %s", configMapNamespace, name)); + } + Map data = configMap.getData(); + if (data != null && data.containsKey(name)) { + return data.get(name); + } else { + log.error("在ConfigMap中未找到指定的键: " + name); + throw new RuntimeException(String + .format("在ConfigMap中未找到指定的键: %s, configMapNamespace: %s, name: %s", name, configMapNamespace, name)); + } + } catch (Exception e) { + log.error("get config map failed", e); + throw new RuntimeException(String + .format("get config map failed, configMapNamespace: %s, name: %s", configMapNamespace, name)); + } + } + + /** + * get value from config map + * @param configMapNamespace configMapNamespace + * @param name config map name (appId) + * @param key config map key (cluster+namespace) + * @return value(json string) + */ + public String getValueFromConfigMap(String configMapNamespace, String name, String key) { + if (configMapNamespace == null || configMapNamespace.isEmpty() || name == null || name.isEmpty() || key == null || key.isEmpty()) { + log.error("参数不能为空"); + return null; + } + try { + V1ConfigMap configMap = coreV1Api.readNamespacedConfigMap(name, configMapNamespace, null); + if (configMap == null || configMap.getData() == null) { + log.error("ConfigMap不存在或没有数据"); + return null; + } + if (!configMap.getData().containsKey(key)) { + log.error("在ConfigMap中未找到指定的键: " + key); + return null; + } + return configMap.getData().get(key); + } catch (Exception e) { + log.error("get config map failed", e); + return null; + } + } + + /** + * update config map + * @param configMapNamespace + * @param name config map name (appId) + * @param data new data + * @return config map name + */ + public String updateConfigMap(String configMapNamespace, String name, Map data) { + if (configMapNamespace == null || configMapNamespace.isEmpty() || name == null || name.isEmpty() || data == null || data.isEmpty()) { + log.error("参数不能为空"); + return null; + } + try { + V1ConfigMap configMap = new V1ConfigMap().metadata(new V1ObjectMeta().name(name).namespace(configMapNamespace)).data(data); + coreV1Api.replaceNamespacedConfigMap(name, configMapNamespace, configMap, null, null, null, "fieldManagerValue"); + return name; + } catch (Exception e) { + log.error("update config map failed", e); + return null; + } + } + + /** + * check config map exist + * @param configMapNamespace config map namespace + * @param configMapName config map name + * @return true if config map exist, false otherwise + */ + public boolean checkConfigMapExist(String configMapNamespace, String configMapName) { + if (configMapNamespace == null || configMapNamespace.isEmpty() || configMapName == null || configMapName.isEmpty()) { + log.error("参数不能为空"); + return false; + } + try { + coreV1Api.readNamespacedConfigMap(configMapName, configMapNamespace, null); + return true; + } catch (Exception e) { + log.error("check config map failed", e); + return false; + } + } +} diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/enums/ConfigSourceType.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/enums/ConfigSourceType.java index b8249472..d3e3507e 100644 --- a/apollo-client/src/main/java/com/ctrip/framework/apollo/enums/ConfigSourceType.java +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/enums/ConfigSourceType.java @@ -22,7 +22,10 @@ * @since 1.1.0 */ public enum ConfigSourceType { - REMOTE("Loaded from remote config service"), LOCAL("Loaded from local cache"), NONE("Load failed"); + REMOTE("Loaded from remote config service"), + LOCAL("Loaded from local cache"), + CONFIGMAP("Loaded from k8s config map"), + NONE("Load failed"); private final String description; diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepository.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepository.java new file mode 100644 index 00000000..aa26389b --- /dev/null +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepository.java @@ -0,0 +1,314 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.internals; + +import com.ctrip.framework.apollo.Kubernetes.KubernetesManager; +import com.ctrip.framework.apollo.build.ApolloInjector; +import com.ctrip.framework.apollo.core.ConfigConsts; +import com.ctrip.framework.apollo.core.utils.DeferredLoggerFactory; +import com.ctrip.framework.apollo.core.utils.StringUtils; +import com.ctrip.framework.apollo.enums.ConfigSourceType; +import com.ctrip.framework.apollo.exceptions.ApolloConfigException; +import com.ctrip.framework.apollo.tracer.Tracer; +import com.ctrip.framework.apollo.tracer.spi.Transaction; +import com.ctrip.framework.apollo.util.ConfigUtil; +import com.ctrip.framework.apollo.util.ExceptionUtil; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import org.slf4j.Logger; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * @author dyx1234 + */ +public class K8sConfigMapConfigRepository extends AbstractConfigRepository + implements RepositoryChangeListener { + private static final Logger logger = DeferredLoggerFactory.getLogger(K8sConfigMapConfigRepository.class); + private final String namespace; + private String configMapName; + private String configMapKey; + private String configMapNamespace; + private final ConfigUtil configUtil; + private final KubernetesManager kubernetesManager; + private volatile Properties configMapProperties; + // 上游数据源 + private volatile ConfigRepository upstream; + private volatile ConfigSourceType sourceType = ConfigSourceType.CONFIGMAP; + + /** + * configmapNamespace 用户配的，不配用默认default + * configmapName appid + * configmap-key cluster+namespace + * configmap-value 配置文件信息的json串 + */ + + // TODO Properties和appConfig格式的兼容(go客户端存的格式是封装过的ApolloConfig，不是单纯的配置信息json） + + /** + * Constructor + * + * @param namespace the namespace + */ + public K8sConfigMapConfigRepository(String namespace) { + this(namespace, null); + } + + public K8sConfigMapConfigRepository(String namespace, ConfigRepository upstream) { + this.namespace = namespace; + configUtil = ApolloInjector.getInstance(ConfigUtil.class); + // 单例模式，客户端只会初始化一个 + kubernetesManager = ApolloInjector.getInstance(KubernetesManager.class); + // 读取，默认为default + configMapNamespace = configUtil.getConfigMapNamespace(); + + this.setConfigMapKey(configUtil.getCluster(), namespace); + this.setConfigMapName(configUtil.getAppId(), false); + this.setUpstreamRepository(upstream); + } + + void setConfigMapKey(String cluster, String namespace){ + // TODO 兜底key怎么设计不会冲突(cluster初始化时已经设置了层级) + // cluster 就是填写>idc>default,所以不需要额外层级设置了 + if (StringUtils.isBlank(cluster)){ + configMapKey = Joiner.on(ConfigConsts.CLUSTER_NAMESPACE_SEPARATOR).join("default", namespace); + return; + } + configMapKey = Joiner.on(ConfigConsts.CLUSTER_NAMESPACE_SEPARATOR).join(cluster, namespace); + } + + void setConfigMapName(String appId, boolean syncImmediately){ + configMapName = appId; + this.checkConfigMapName(configMapName); + if (syncImmediately) { + this.sync(); + } + } + + private void checkConfigMapName(String configMapName) { + if (StringUtils.isBlank(configMapName)) { + throw new ApolloConfigException("ConfigMap name cannot be null"); + } + // 判断configMap是否存在，若存在直接返回，若不存在尝试创建 + if(kubernetesManager.checkConfigMapExist(configMapNamespace, configMapName)){ + return; + } + // TODO 初步理解这里只生成就可以，后续update事件再写入新值 + + Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "createK8sConfigMap"); + transaction.addData("configMapName", configMapName); + try { + kubernetesManager.createConfigMap(configMapNamespace, configMapName, null); + transaction.setStatus(Transaction.SUCCESS); + } catch (Throwable ex) { + Tracer.logEvent("ApolloConfigException", ExceptionUtil.getDetailMessage(ex)); + transaction.setStatus(ex); + throw new ApolloConfigException("Create configmap failed!", ex); + } finally { + transaction.complete(); + } + } + + @Override + public Properties getConfig() { + if (configMapProperties == null) { + sync(); + } + Properties result = propertiesFactory.getPropertiesInstance(); + result.putAll(configMapProperties); + return result; + } + + /** + * Update the memory when the configuration center changes + * @param upstreamConfigRepository the upstream repo + */ + @Override + public void setUpstreamRepository(ConfigRepository upstreamConfigRepository) { + // 设置上游数据源 + if (upstreamConfigRepository == null) { + return; + } + //clear previous listener + if (upstream != null) { + upstream.removeChangeListener(this); + } + upstream = upstreamConfigRepository; + upstreamConfigRepository.addChangeListener(this); + } + + @Override + public ConfigSourceType getSourceType() { + return sourceType; + } + + /** + * Sync the configmap + */ + @Override + protected void sync() { + // 链式恢复，先从上游数据源读取 + boolean syncFromUpstreamResultSuccess = trySyncFromUpstream(); + + if (syncFromUpstreamResultSuccess) { + return; + } + + Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "syncK8sConfigMap"); + Throwable exception = null; + try { + configMapProperties = loadFromK8sConfigMap(); + sourceType = ConfigSourceType.CONFIGMAP; + transaction.setStatus(Transaction.SUCCESS); + } catch (Throwable ex) { + Tracer.logEvent("ApolloConfigException", ExceptionUtil.getDetailMessage(ex)); + transaction.setStatus(ex); + exception = ex; + throw new ApolloConfigException("Load config from Kubernetes ConfigMap failed!", ex); + } finally { + transaction.complete(); + } + + if (configMapProperties == null) { + sourceType = ConfigSourceType.NONE; + throw new ApolloConfigException( + "Load config from Kubernetes ConfigMap failed!", exception); + } + } + + // 职责明确: manager层进行序列化和解析，把key传进去 + // repo这里只负责更新内存, Properties和appConfig格式的兼容 + public Properties loadFromK8sConfigMap() throws IOException { + Preconditions.checkNotNull(configMapName, "ConfigMap name cannot be null"); + + Properties properties = null; + try { + // 从ConfigMap获取整个配置信息的JSON字符串 + String jsonConfig = kubernetesManager.getValueFromConfigMap(configMapNamespace, configUtil.getAppId(), configMapKey); + if (jsonConfig == null) { + // TODO 待修改，重试访问保底configmap + jsonConfig = kubernetesManager.getValueFromConfigMap(configMapNamespace, configMapName, Joiner.on(ConfigConsts.CLUSTER_NAMESPACE_SEPARATOR).join("default", namespace)); + } + + // 确保获取到的配置信息不为空 + if (jsonConfig != null) { + // 解码Base64编码的JSON字符串 + jsonConfig = new String(Base64.getDecoder().decode(jsonConfig)); + } + + // 创建Properties实例 + properties = propertiesFactory.getPropertiesInstance(); + + // 使用Gson将JSON字符串转换为Map对象 + if (jsonConfig != null && !jsonConfig.isEmpty()) { + Gson gson = new Gson(); + Type type = new TypeToken>() {}.getType(); + Map configMap = gson.fromJson(jsonConfig, type); + // 将Map中的键值对填充到Properties对象中 + for (Map.Entry entry : configMap.entrySet()) { + properties.setProperty(entry.getKey(), entry.getValue()); + } + } + return properties; + } catch (Exception ex) { + Tracer.logError(ex); + throw new ApolloConfigException(String + .format("Load config from Kubernetes ConfigMap %s failed!", configMapName), ex); + } + } + + private boolean trySyncFromUpstream() { + if (upstream == null) { + return false; + } + try { + // TODO 从上游数据恢复的逻辑 + // 拉新数据，并将新数据更新到configMap + updateConfigMapProperties(upstream.getConfig(), upstream.getSourceType()); + return true; + } catch (Throwable ex) { + Tracer.logError(ex); + logger + .warn("Sync config from upstream repository {} failed, reason: {}", upstream.getClass(), + ExceptionUtil.getDetailMessage(ex)); + } + return false; + } + + /** + * Update the memory + * @param namespace the namespace of this repository change + * @param newProperties the properties after change + */ + @Override + public void onRepositoryChange(String namespace, Properties newProperties) { + if (newProperties.equals(configMapProperties)) { + return; + } + Properties newFileProperties = propertiesFactory.getPropertiesInstance(); + newFileProperties.putAll(newProperties); + updateConfigMapProperties(newFileProperties, upstream.getSourceType()); + this.fireRepositoryChange(namespace, newProperties); + } + + private synchronized void updateConfigMapProperties(Properties newProperties, ConfigSourceType sourceType) { + this.sourceType = sourceType; + if (newProperties.equals(configMapProperties)) { + return; + } + this.configMapProperties = newProperties; + persistConfigMap(configMapProperties); + } + + public void persistConfigMap(Properties properties) { + // 将Properties中的值持久化到configmap中，并使用事务管理 + Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "persistK8sConfigMap"); + transaction.addData("configMapName", configUtil.getAppId()); + transaction.addData("configMapNamespace", configUtil.getConfigMapNamespace()); + try { + // 使用Gson将properties转换为JSON字符串 + Gson gson = new Gson(); + String jsonConfig = gson.toJson(properties); + String encodedJsonConfig = Base64.getEncoder().encodeToString(jsonConfig.getBytes()); + // 创建一个新的HashMap, 将编码后的JSON字符串作为值，业务appId作为键，存入data中 + Map data = new HashMap<>(); + data.put(configUtil.getAppId(), encodedJsonConfig); + + // 更新ConfigMap + kubernetesManager.updateConfigMap(configUtil.getConfigMapNamespace(), configUtil.getAppId(), data); + transaction.setStatus(Transaction.SUCCESS); + } catch (Exception ex) { + ApolloConfigException exception = + new ApolloConfigException( + String.format("Persist config to Kubernetes ConfigMap %s failed!", configMapName), ex); + Tracer.logError(exception); + transaction.setStatus(exception); + logger.error("Persist config to Kubernetes ConfigMap failed!", exception); + } finally { + transaction.complete(); + } + transaction.complete(); + } + +} diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/spi/DefaultConfigFactory.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/spi/DefaultConfigFactory.java index 36c300f4..2aed0d2b 100644 --- a/apollo-client/src/main/java/com/ctrip/framework/apollo/spi/DefaultConfigFactory.java +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/spi/DefaultConfigFactory.java @@ -22,17 +22,7 @@ import com.ctrip.framework.apollo.PropertiesCompatibleConfigFile; import com.ctrip.framework.apollo.build.ApolloInjector; import com.ctrip.framework.apollo.core.enums.ConfigFileFormat; -import com.ctrip.framework.apollo.internals.ConfigRepository; -import com.ctrip.framework.apollo.internals.DefaultConfig; -import com.ctrip.framework.apollo.internals.JsonConfigFile; -import com.ctrip.framework.apollo.internals.LocalFileConfigRepository; -import com.ctrip.framework.apollo.internals.PropertiesCompatibleFileConfigRepository; -import com.ctrip.framework.apollo.internals.PropertiesConfigFile; -import com.ctrip.framework.apollo.internals.RemoteConfigRepository; -import com.ctrip.framework.apollo.internals.TxtConfigFile; -import com.ctrip.framework.apollo.internals.XmlConfigFile; -import com.ctrip.framework.apollo.internals.YamlConfigFile; -import com.ctrip.framework.apollo.internals.YmlConfigFile; +import com.ctrip.framework.apollo.internals.*; import com.ctrip.framework.apollo.util.ConfigUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,6 +67,7 @@ public Config create(String namespace) { configRepository = createPropertiesCompatibleFileConfigRepository(namespace, format); } else { configRepository = createConfigRepository(namespace); + //ServiceBootstrap.loadPrimary(ConfigRepository.class); } logger.debug("Created a configuration repository of type [{}] for namespace [{}]", @@ -111,7 +102,10 @@ public ConfigFile createConfigFile(String namespace, ConfigFileFormat configFile } ConfigRepository createConfigRepository(String namespace) { - if (m_configUtil.isPropertyFileCacheEnabled()) { + // TODO 本地和configMap允许同时开启 + if (m_configUtil.isPropertyKubernetesCacheEnabled()) { + return createConfigMapConfigRepository(namespace); + }else if (m_configUtil.isPropertyFileCacheEnabled()) { return createLocalConfigRepository(namespace); } return createRemoteConfigRepository(namespace); @@ -133,6 +127,21 @@ LocalFileConfigRepository createLocalConfigRepository(String namespace) { return new LocalFileConfigRepository(namespace, createRemoteConfigRepository(namespace)); } + /** + * Creates a Kubernetes config map repository for a given namespace + * @param namespace the namespace of the repository + * @return the newly created repository for the given namespace + */ + private ConfigRepository createConfigMapConfigRepository(String namespace) { + if (m_configUtil.isInKubernetesMode()) { + logger.warn( + "==== Apollo is in local mode! Won't pull configs from remote server for namespace {} ! ====", + namespace); + return new K8sConfigMapConfigRepository(namespace); + } + return new K8sConfigMapConfigRepository(namespace, createLocalConfigRepository(namespace)); + } + RemoteConfigRepository createRemoteConfigRepository(String namespace) { return new RemoteConfigRepository(namespace); } diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/util/ConfigUtil.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/util/ConfigUtil.java index 9b99a4bf..d3c74c4d 100644 --- a/apollo-client/src/main/java/com/ctrip/framework/apollo/util/ConfigUtil.java +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/util/ConfigUtil.java @@ -71,6 +71,7 @@ public class ConfigUtil { private boolean propertyNamesCacheEnabled = false; private boolean propertyFileCacheEnabled = true; private boolean overrideSystemProperties = true; + private boolean PropertyKubernetesCacheEnabled = false; public ConfigUtil() { warnLogRateLimiter = RateLimiter.create(0.017); // 1 warning log output per minute @@ -86,6 +87,7 @@ public ConfigUtil() { initPropertyNamesCacheEnabled(); initPropertyFileCacheEnabled(); initOverrideSystemProperties(); + initPropertyKubernetesCacheEnabled(); } /** @@ -366,6 +368,34 @@ private String getDeprecatedCustomizedCacheRoot() { return cacheRoot; } + public String getConfigMapNamespace() { + String configMapNamespace = getCustomizedConfigMapNamespace(); + + if (!Strings.isNullOrEmpty(configMapNamespace)) { + return configMapNamespace; + } + + return ConfigConsts.KUBERNETES_CACHE_CONFIG_MAP_NAMESPACE_DEFAULT; + } + + private String getCustomizedConfigMapNamespace() { + // 1. Get from System Property + String configMapNamespace = System.getProperty(ApolloClientSystemConsts.APOLLO_CONFIGMAP_NAMESPACE); + if (Strings.isNullOrEmpty(configMapNamespace)) { + // 2. Get from OS environment variable + configMapNamespace = System.getenv(ApolloClientSystemConsts.APOLLO_CONFIGMAP_NAMESPACE_ENVIRONMENT_VARIABLES); + } + if (Strings.isNullOrEmpty(configMapNamespace)) { + // 3. Get from server.properties + configMapNamespace = Foundation.server().getProperty(ApolloClientSystemConsts.APOLLO_CONFIGMAP_NAMESPACE, null); + } + if (Strings.isNullOrEmpty(configMapNamespace)) { + // 4. Get from app.properties + configMapNamespace = Foundation.app().getProperty(ApolloClientSystemConsts.APOLLO_CONFIGMAP_NAMESPACE, null); + } + return configMapNamespace; + } + public boolean isInLocalMode() { try { return Env.LOCAL == getApolloEnv(); @@ -375,6 +405,15 @@ public boolean isInLocalMode() { return false; } + public boolean isInKubernetesMode() { + try { + return Env.KUBERNETES == getApolloEnv(); + } catch (Throwable ex) { + //ignore + } + return false; + } + public boolean isOSWindows() { String osName = System.getProperty("os.name"); if (Strings.isNullOrEmpty(osName)) { @@ -469,6 +508,10 @@ public boolean isPropertyFileCacheEnabled() { return propertyFileCacheEnabled; } + public boolean isPropertyKubernetesCacheEnabled() { + return PropertyKubernetesCacheEnabled; + } + public boolean isOverrideSystemProperties() { return overrideSystemProperties; } @@ -491,6 +534,12 @@ private void initOverrideSystemProperties() { overrideSystemProperties); } + private void initPropertyKubernetesCacheEnabled() { + PropertyKubernetesCacheEnabled = getPropertyBoolean(ApolloClientSystemConsts.APOLLO_KUBERNETES_CACHE_ENABLE, + ApolloClientSystemConsts.APOLLO_KUBERNETES_CACHE_ENABLE_ENVIRONMENT_VARIABLES, + PropertyKubernetesCacheEnabled); + } + private boolean getPropertyBoolean(String propertyName, String envName, boolean defaultVal) { String enablePropertyNamesCache = System.getProperty(propertyName); if (Strings.isNullOrEmpty(enablePropertyNamesCache)) { diff --git a/apollo-client/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/apollo-client/src/main/resources/META-INF/additional-spring-configuration-metadata.json index a14b1d03..d49b372a 100644 --- a/apollo-client/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/apollo-client/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -76,6 +76,20 @@ "description": "enable property names cache.", "defaultValue": false }, + { + "name": "apollo.cache.kubernetes.enable", + "type": "java.lang.Boolean", + "sourceType": "com.ctrip.framework.apollo.util.ConfigUtil", + "description": "enable kubernetes configmap cache.", + "defaultValue": false + }, + { + "name": "apollo.cache.kubernetes.namespace", + "type": "java.lang.String", + "sourceType": "com.ctrip.framework.apollo.util.ConfigUtil", + "description": "kubernetes configmap namespace.", + "defaultValue": "default" + }, { "name": "apollo.property.order.enable", "type": "java.lang.Boolean", diff --git a/apollo-client/src/test/java/com/ctrip/framework/apollo/Kubernetes/KubernetesManagerTest.java b/apollo-client/src/test/java/com/ctrip/framework/apollo/Kubernetes/KubernetesManagerTest.java new file mode 100644 index 00000000..f9885126 --- /dev/null +++ b/apollo-client/src/test/java/com/ctrip/framework/apollo/Kubernetes/KubernetesManagerTest.java @@ -0,0 +1,277 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.Kubernetes; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.Mockito.*; +import static org.junit.Assert.*; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesManagerTest { + + @Mock + private CoreV1Api coreV1Api; + @Mock + private ApiClient client; + + @InjectMocks + private KubernetesManager kubernetesManager; + + /** + * 测试 createConfigMap 成功创建配置 + */ + @Test + public void testCreateConfigMapSuccess() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + Map data = new HashMap<>(); + data.put("key", "value"); + V1ConfigMap configMap = new V1ConfigMap() + .metadata(new V1ObjectMeta().name(name).namespace(namespace)) + .data(data); + + when(coreV1Api.createNamespacedConfigMap(eq(namespace), eq(configMap), isNull(), isNull(), isNull(),isNull())).thenReturn(configMap); + + // act + String result = kubernetesManager.createConfigMap(namespace, name, data); + + // assert + verify(coreV1Api, times(1)).createNamespacedConfigMap(eq(namespace), any(V1ConfigMap.class), isNull(), isNull(), isNull(),isNull()); + assert name.equals(result); + } + + /** + * 测试 createConfigMap 传入空的命名空间 + */ + @Test + public void testCreateConfigMapEmptyNamespace() throws Exception { + // arrange + String namespace = ""; + String name = "testConfigMap"; + Map data = new HashMap<>(); + data.put("key", "value"); + + // act + String result = kubernetesManager.createConfigMap(namespace, name, data); + + // assert + verify(coreV1Api, never()).createNamespacedConfigMap(anyString(), any(V1ConfigMap.class), isNull(), isNull(), isNull(),isNull()); + assert result == null; + } + + /** + * 测试 createConfigMap 传入空的配置名 + */ + @Test + public void testCreateConfigMapEmptyName() throws Exception { + // arrange + String namespace = "default"; + String name = ""; + Map data = new HashMap<>(); + data.put("key", "value"); + + // act + String result = kubernetesManager.createConfigMap(namespace, name, data); + + // assert + verify(coreV1Api, never()).createNamespacedConfigMap(anyString(), any(V1ConfigMap.class), isNull(), isNull(), isNull(),isNull()); + assert result == null; + } + + /** + * 测试 createConfigMap 传入 null 作为数据 + */ + @Test + public void testCreateConfigMapNullData() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + Map data = null; + + // act + String result = kubernetesManager.createConfigMap(namespace, name, data); + + // assert + verify(coreV1Api, times(1)).createNamespacedConfigMap(eq(namespace), any(V1ConfigMap.class), isNull(), isNull(), isNull(),isNull()); + assert name.equals(result); + } + + /** + * 测试loadFromConfigMap方法在正常情况下的行为 + */ + @Test + public void testLoadFromConfigMapSuccess() throws Exception { + // arrange + String namespace = "TestNamespace"; + String name = "TestName"; + V1ConfigMap configMap = new V1ConfigMap(); + configMap.putDataItem(name, "TestValue"); + when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(configMap); + + // act + String result = kubernetesManager.loadFromConfigMap(namespace, name); + + // assert + assertEquals("TestValue", result); + } + + /** + * 测试loadFromConfigMap方法在抛出异常时的行为 + */ + @Test + public void testLoadFromConfigMapFailure() throws Exception { + // arrange + String namespace = "TestNamespace"; + String name = "TestName"; + when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenThrow(new ApiException("Kubernetes Manager Exception")); + + // act + String result = kubernetesManager.loadFromConfigMap(namespace, name); + + // assert + assertNull(result); + } + + /** + * 测试loadFromConfigMap方法在ConfigMap不存在时的行为 + */ + @Test + public void testLoadFromConfigMapConfigMapNotFound() throws Exception { + // arrange + String namespace = "TestNamespace"; + String name = "TestName"; + when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(null); + + // act + String result = kubernetesManager.loadFromConfigMap(namespace, name); + + // assert + assertNull(result); + } + + /** + * 测试getValueFromConfigMap方法，当ConfigMap存在且包含指定key时返回正确的value + */ + @Test + public void testGetValueFromConfigMapReturnsValue() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + String key = "testKey"; + String expectedValue = "testValue"; + V1ConfigMap configMap = new V1ConfigMap(); + configMap.putDataItem(key, expectedValue); + when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(configMap); + + // act + String actualValue = kubernetesManager.getValueFromConfigMap(namespace, name, key); + + // assert + assertEquals(expectedValue, actualValue); + } + + /** + * 测试getValueFromConfigMap方法，当ConfigMap不存在指定key时返回null + */ + @Test + public void testGetValueFromConfigMapKeyNotFound() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + String key = "nonExistingKey"; + V1ConfigMap configMap = new V1ConfigMap(); + when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(configMap); + + // act + String actualValue = kubernetesManager.getValueFromConfigMap(namespace, name, key); + + // assert + assertNull(actualValue); + } + + /** + * 测试updateConfigMap成功的情况 + */ + @Test + public void testUpdateConfigMapSuccess() throws Exception { + // arrange + String configMapNamespace = "default"; + String name = "testConfigMap"; + Map data = new HashMap<>(); + data.put("key", "value"); + V1ConfigMap configMap = new V1ConfigMap(); + configMap.metadata(new V1ObjectMeta().name(name).namespace(configMapNamespace)); + configMap.data(data); + + when(coreV1Api.replaceNamespacedConfigMap(name, configMapNamespace, configMap, null, null, null, "fieldManagerValue")).thenReturn(configMap); + + // act + String result = kubernetesManager.updateConfigMap(configMapNamespace, name, data); + + // assert + assert result.equals(name); + } + + /** + * 测试ConfigMap存在的情况 + */ + @Test + public void testCheckConfigMapExistWhenConfigMapExists() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(new V1ConfigMap()); + + // act + boolean result = kubernetesManager.checkConfigMapExist(namespace, name); + + // assert + assertTrue(result); + } + + /** + * 测试ConfigMap不存在的情况 + */ + @Test + public void testCheckConfigMapExistWhenConfigMapDoesNotExist() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + doThrow(new RuntimeException("ConfigMap not found")).when(coreV1Api).readNamespacedConfigMap(name, namespace, null); + + // act + boolean result = kubernetesManager.checkConfigMapExist(namespace, name); + + // assert + assertFalse(result); + } + +} diff --git a/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepositoryTest.java b/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepositoryTest.java new file mode 100644 index 00000000..ba38fc99 --- /dev/null +++ b/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepositoryTest.java @@ -0,0 +1,170 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.internals; + +import com.ctrip.framework.apollo.Kubernetes.KubernetesManager; +import com.ctrip.framework.apollo.enums.ConfigSourceType; +import com.ctrip.framework.apollo.exceptions.ApolloConfigException; +import com.ctrip.framework.apollo.util.ConfigUtil; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.Properties; + +import static org.mockito.Mockito.*; + +/** + * TODO （未完成）K8sConfigMapConfigRepository单元测试 + */ +public class K8sConfigMapConfigRepositoryTest { + + private String someNamespace; + private ConfigRepository upstreamRepo; + private Properties someProperties; + private static String someAppId = "someApp"; + private static String someCluster = "someCluster"; + private String defaultKey; + private String defaultValue; + private ConfigSourceType someSourceType; + + @Mock + private KubernetesManager kubernetesManager; + @Mock + private ConfigUtil configUtil; + + private K8sConfigMapConfigRepository k8sConfigMapConfigRepository; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + when(configUtil.getAppId()).thenReturn("testAppId"); + when(configUtil.getCluster()).thenReturn("default"); + when(configUtil.getConfigMapNamespace()).thenReturn("default"); + + someProperties = new Properties(); + defaultKey = "defaultKey"; + defaultValue = "defaultValue"; + someProperties.setProperty(defaultKey, defaultValue); + + + k8sConfigMapConfigRepository = new K8sConfigMapConfigRepository("namespace", null); + } + + /** + * 测试sync方法成功从上游数据源同步 + */ + @Test + public void testSyncSuccessFromUpstream() throws Throwable { + // arrange + ConfigRepository upstream = mock(ConfigRepository.class); + Properties upstreamProperties = new Properties(); + upstreamProperties.setProperty("key", "value"); + when(upstream.getConfig()).thenReturn(upstreamProperties); + when(upstream.getSourceType()).thenReturn(ConfigSourceType.REMOTE); + k8sConfigMapConfigRepository.setUpstreamRepository(upstream); + + // act + k8sConfigMapConfigRepository.sync(); + + // assert + verify(upstream, times(1)).getConfig(); + } + + /** + * 测试sync方法从上游数据源同步失败，成功从Kubernetes的ConfigMap中加载 + */ + @Test + public void testSyncFailFromUpstreamSuccessFromConfigMap() throws Throwable { + // arrange + ConfigRepository upstream = mock(ConfigRepository.class); + when(upstream.getConfig()).thenThrow(new RuntimeException("Upstream sync failed")); + k8sConfigMapConfigRepository.setUpstreamRepository(upstream); + when(kubernetesManager.getValueFromConfigMap(anyString(), anyString(), anyString())).thenReturn("encodedConfig"); + + // act + k8sConfigMapConfigRepository.sync(); + + // assert + verify(kubernetesManager, times(1)).getValueFromConfigMap(anyString(), anyString(), anyString()); + } + + /** + * 测试loadFromK8sConfigMap方法成功加载配置信息 + */ + @Test + public void testLoadFromK8sConfigMapSuccess() throws Throwable { + // arrange + when(kubernetesManager.getValueFromConfigMap(anyString(), anyString(), anyString())).thenReturn("encodedConfig"); + + // act + Properties properties = k8sConfigMapConfigRepository.loadFromK8sConfigMap(); + + // assert + verify(kubernetesManager, times(1)).getValueFromConfigMap(anyString(), anyString(), anyString()); + // 这里应该有更具体的断言来验证properties的内容，但由于编码和解码逻辑未给出，此处省略 + } + + /** + * 测试loadFromK8sConfigMap方法在加载配置信息时发生异常 + */ + @Test(expected = ApolloConfigException.class) + public void testLoadFromK8sConfigMapException() throws Throwable { + // arrange + when(kubernetesManager.getValueFromConfigMap(anyString(), anyString(), anyString())).thenThrow(new RuntimeException("Load failed")); + + // act + k8sConfigMapConfigRepository.loadFromK8sConfigMap(); + + // assert + // 预期抛出ApolloConfigException + } + + /** + * 测试persistConfigMap方法成功持久化配置信息 + */ + @Test + public void testPersistConfigMapSuccess() throws Throwable { + // arrange + Properties properties = new Properties(); + properties.setProperty("key", "value"); + + // act + k8sConfigMapConfigRepository.persistConfigMap(properties); + + // assert + verify(kubernetesManager, times(1)).updateConfigMap(anyString(), anyString(), anyMap()); + } + + /** + * 测试persistConfigMap方法在持久化配置信息时发生异常 + */ + @Test(expected = ApolloConfigException.class) + public void testPersistConfigMapException() throws Throwable { + // arrange + Properties properties = new Properties(); + properties.setProperty("key", "value"); + doThrow(new RuntimeException("Persist failed")).when(kubernetesManager).updateConfigMap(anyString(), anyString(), anyMap()); + + // act + k8sConfigMapConfigRepository.persistConfigMap(properties); + + // assert + // 预期抛出ApolloConfigException + } +} diff --git a/apollo-client/src/test/java/com/ctrip/framework/apollo/util/ConfigUtilTest.java b/apollo-client/src/test/java/com/ctrip/framework/apollo/util/ConfigUtilTest.java index 23a37362..b9f88a53 100644 --- a/apollo-client/src/test/java/com/ctrip/framework/apollo/util/ConfigUtilTest.java +++ b/apollo-client/src/test/java/com/ctrip/framework/apollo/util/ConfigUtilTest.java @@ -243,6 +243,24 @@ public void testDefaultLocalCacheDir() throws Exception { assertEquals("/opt/data/" + someAppId, configUtil.getDefaultLocalCacheDir()); } + @Test + public void testConfigMapNamespaceWithSystemProperty() { + String someConfigMapNamespace = "someConfigMapNamespace"; + + System.setProperty(ApolloClientSystemConsts.APOLLO_CONFIGMAP_NAMESPACE, someConfigMapNamespace); + + ConfigUtil configUtil = new ConfigUtil(); + + assertEquals(someConfigMapNamespace, configUtil.getConfigMapNamespace()); + } + + @Test + public void testConfigMapNamespaceWithDefault() { + ConfigUtil configUtil = new ConfigUtil(); + + assertEquals(ConfigConsts.KUBERNETES_CACHE_CONFIG_MAP_NAMESPACE_DEFAULT, configUtil.getConfigMapNamespace()); + } + @Test public void testCustomizePropertiesOrdered() { boolean propertiesOrdered = true; diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ApolloClientSystemConsts.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ApolloClientSystemConsts.java index 0fa0550a..0e47a664 100644 --- a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ApolloClientSystemConsts.java +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ApolloClientSystemConsts.java @@ -73,6 +73,16 @@ public class ApolloClientSystemConsts { @Deprecated public static final String DEPRECATED_APOLLO_CACHE_DIR_ENVIRONMENT_VARIABLES = "APOLLO_CACHEDIR"; + /** + * kubernetes configmap cache namespace + */ + public static final String APOLLO_CONFIGMAP_NAMESPACE = "apollo.configmap-namespace"; + + /** + * kubernetes configmap cache namespace environment variables + */ + public static final String APOLLO_CONFIGMAP_NAMESPACE_ENVIRONMENT_VARIABLES = "APOLLO_CONFIGMAP_NAMESPACE"; + /** * apollo client access key */ @@ -157,6 +167,16 @@ public class ApolloClientSystemConsts { */ public static final String APOLLO_CACHE_FILE_ENABLE_ENVIRONMENT_VARIABLES = "APOLLO_CACHE_FILE_ENABLE"; + /** + * enable property names cache + */ + public static final String APOLLO_KUBERNETES_CACHE_ENABLE = "apollo.cache.kubernetes.enable"; + + /** + * enable property names cache environment variables + */ + public static final String APOLLO_KUBERNETES_CACHE_ENABLE_ENVIRONMENT_VARIABLES = "APOLLO_KUBERNETES_CACHE_ENABLE"; + /** * enable apollo overrideSystemProperties */ diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ConfigConsts.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ConfigConsts.java index a6108dfb..d6370ec0 100644 --- a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ConfigConsts.java +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ConfigConsts.java @@ -24,5 +24,6 @@ public interface ConfigConsts { String APOLLO_META_KEY = "apollo.meta"; String CONFIG_FILE_CONTENT_KEY = "content"; String NO_APPID_PLACEHOLDER = "ApolloNoAppIdPlaceHolder"; + String KUBERNETES_CACHE_CONFIG_MAP_NAMESPACE_DEFAULT = "default"; long NOTIFICATION_ID_PLACEHOLDER = -1; } diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/Env.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/Env.java index 71a682e5..083d4d2d 100644 --- a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/Env.java +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/enums/Env.java @@ -35,7 +35,7 @@ * @author Jason Song(song_s@ctrip.com) */ public enum Env{ - LOCAL, DEV, FWS, FAT, UAT, LPT, PRO, TOOLS, UNKNOWN; + LOCAL, DEV, FWS, FAT, UAT, LPT, PRO, TOOLS, UNKNOWN, KUBERNETES; public static Env fromString(String env) { Env environment = EnvUtils.transformEnv(env);