NMS-16358: Fixed meta-data retrieval in Collectd

OpenNMS · Mar 12, 2024 · a12d06e · a12d06e
1 parent 1ccbc71
commit a12d06e
Show file tree

Hide file tree

Showing 14 changed files with 224 additions and 12 deletions.
diff --git a/core/mate/api/src/main/java/org/opennms/core/mate/api/EmptyScopeProvider.java b/core/mate/api/src/main/java/org/opennms/core/mate/api/EmptyScopeProvider.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to The OpenNMS Group, Inc (TOG) under one or more
+ * contributor license agreements.  See the LICENSE.md file
+ * distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * TOG licenses this file to You under the GNU Affero General
+ * Public License Version 3 (the "License") or (at your option)
+ * any later version.  You may not use this file except in
+ * compliance with the License.  You may obtain a copy of the
+ * License at:
+ *
+ *      https://www.gnu.org/licenses/agpl-3.0.txt
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied.  See the License for the specific
+ * language governing permissions and limitations under the
+ * License.
+ */
+package org.opennms.core.mate.api;
+
+public class EmptyScopeProvider implements ScopeProvider {
+    public final static EmptyScopeProvider EMPTY = new EmptyScopeProvider();
+
+    private EmptyScopeProvider() {
+    }
+
+    @Override
+    public Scope getScope() {
+        return EmptyScope.EMPTY;
+    }
+}
diff --git a/core/mate/api/src/main/java/org/opennms/core/mate/api/EntityScopeProvider.java b/core/mate/api/src/main/java/org/opennms/core/mate/api/EntityScopeProvider.java
@@ -41,4 +41,24 @@ interface Contexts {
     Scope getScopeForInterfaceByIfIndex(final Integer nodeId, final int ifIndex);
 
     Scope getScopeForService(final Integer nodeId, final InetAddress ipAddress, final String serviceName);
+
+    default ScopeProvider getScopeProviderForScv() {
+        return () -> getScopeForScv();
+    }
+
+    default ScopeProvider getScopeProviderForNode(final Integer nodeId) {
+        return () -> getScopeForNode(nodeId);
+    }
+
+    default ScopeProvider getScopeProviderForInterface(final Integer nodeId, final String ipAddress) {
+        return () -> getScopeForInterface(nodeId, ipAddress);
+    }
+
+    default ScopeProvider getScopeProviderForInterfaceByIfIndex(final Integer nodeId, final int ifIndex) {
+        return () -> getScopeForInterfaceByIfIndex(nodeId, ifIndex);
+    }
+
+    default ScopeProvider getScopeProviderForService(final Integer nodeId, final InetAddress ipAddress, final String serviceName) {
+        return () -> getScopeForService(nodeId, ipAddress, serviceName);
+    }
 }
diff --git a/core/mate/api/src/main/java/org/opennms/core/mate/api/FallBackScopeProvider.java b/core/mate/api/src/main/java/org/opennms/core/mate/api/FallBackScopeProvider.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to The OpenNMS Group, Inc (TOG) under one or more
+ * contributor license agreements.  See the LICENSE.md file
+ * distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * TOG licenses this file to You under the GNU Affero General
+ * Public License Version 3 (the "License") or (at your option)
+ * any later version.  You may not use this file except in
+ * compliance with the License.  You may obtain a copy of the
+ * License at:
+ *
+ *      https://www.gnu.org/licenses/agpl-3.0.txt
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied.  See the License for the specific
+ * language governing permissions and limitations under the
+ * License.
+ */
+package org.opennms.core.mate.api;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+
+public class FallBackScopeProvider implements ScopeProvider {
+    private final List<ScopeProvider> scopeProviders;
+
+    public FallBackScopeProvider(final List<ScopeProvider> scopeProviders) {
+        this.scopeProviders = ImmutableList.copyOf(scopeProviders);
+    }
+
+    public FallBackScopeProvider(final ScopeProvider... scopeProviders) {
+        this.scopeProviders = ImmutableList.copyOf(scopeProviders);
+    }
+
+
+    @Override
+    public Scope getScope() {
+        return new FallbackScope(scopeProviders.stream().map(ScopeProvider::getScope).collect(Collectors.toList()));
+    }
+}
diff --git a/core/mate/api/src/main/java/org/opennms/core/mate/api/ScopeProvider.java b/core/mate/api/src/main/java/org/opennms/core/mate/api/ScopeProvider.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to The OpenNMS Group, Inc (TOG) under one or more
+ * contributor license agreements.  See the LICENSE.md file
+ * distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * TOG licenses this file to You under the GNU Affero General
+ * Public License Version 3 (the "License") or (at your option)
+ * any later version.  You may not use this file except in
+ * compliance with the License.  You may obtain a copy of the
+ * License at:
+ *
+ *      https://www.gnu.org/licenses/agpl-3.0.txt
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied.  See the License for the specific
+ * language governing permissions and limitations under the
+ * License.
+ */
+package org.opennms.core.mate.api;
+
+public interface ScopeProvider {
+    Scope getScope();
+}
diff --git a/core/mate/api/src/test/java/org/opennms/core/mate/api/MetaDataUtilsTest.java b/core/mate/api/src/test/java/org/opennms/core/mate/api/MetaDataUtilsTest.java
@@ -123,4 +123,24 @@ public void testNewRegExpPattern() {
         assertEquals("val4", result.parts.get(1).value.value);
         assertEquals("foo-${aaa}-bar-val1-bla-val4-blupp-${bbb}", result.output);
     }
+
+    @Test
+    public void testFallBackScopeProvider() {
+        final Map<ContextKey, String> map1 = new TreeMap<>();
+        map1.put(new ContextKey("foobar", "key1"), "value1");
+        map1.put(new ContextKey("foobar", "key2"), "value2");
+        map1.put(new ContextKey("foobar", "key3"), "value3");
+
+        final Map<ContextKey, String> map2 = new TreeMap<>();
+        map2.put(new ContextKey("foobar", "key3"), "new3");
+
+        final FallBackScopeProvider fallBackScopeProvider = new FallBackScopeProvider(
+                () -> new MapScope(Scope.ScopeName.NODE, map1),
+                () -> new MapScope(Scope.ScopeName.INTERFACE, map2)
+        );
+
+        assertEquals("value1", Interpolator.interpolate("${foobar:key1}", fallBackScopeProvider.getScope()).output);
+        assertEquals("value2", Interpolator.interpolate("${foobar:key2}", fallBackScopeProvider.getScope()).output);
+        assertEquals("new3", Interpolator.interpolate("${foobar:key3}", fallBackScopeProvider.getScope()).output);
+    }
 }
diff --git a/core/mate/model/src/test/java/org/opennms/core/mate/model/EntityScopeProviderIT.java b/core/mate/model/src/test/java/org/opennms/core/mate/model/EntityScopeProviderIT.java
@@ -36,6 +36,7 @@
 import org.opennms.core.mate.api.EntityScopeProvider;
 import org.opennms.core.mate.api.Interpolator;
 import org.opennms.core.mate.api.Scope;
+import org.opennms.core.mate.api.ScopeProvider;
 import org.opennms.core.test.OpenNMSJUnit4ClassRunner;
 import org.opennms.core.test.db.annotations.JUnitTemporaryDatabase;
 import org.opennms.core.utils.InetAddressUtils;
@@ -51,6 +52,8 @@
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.transaction.annotation.Transactional;
 
+import com.google.common.collect.Lists;
+
 @RunWith(OpenNMSJUnit4ClassRunner.class)
 @ContextConfiguration(locations = {
         "classpath:/META-INF/opennms/applicationContext-commonConfigs.xml",
@@ -169,4 +172,28 @@ public void testScvInterpolation() {
         Assert.assertEquals(interpolatedAttributes.get("password"), "OpenNMS@30");
 
     }
+
+    @Test
+    public final void testScopeProviders() {
+        // set meta-data of node
+        final OnmsNode node = this.populator.getNode1();
+        OnmsMetaData metaData = new OnmsMetaData("context", "key", "value1");
+        node.getMetaData().add(metaData);
+        this.populator.getNodeDao().saveOrUpdate(node);
+
+        // get an scope provider
+        final ScopeProvider scope = this.provider.getScopeProviderForNode(this.populator.getNode1().getId());
+
+        // this will retrieve the meta-data set before
+        assertThat(scope.getScope().get(new ContextKey("context", "key")), Matchers.is(Optional.of(new Scope.ScopeValue(Scope.ScopeName.NODE, "value1"))));
+
+        // now update the meta-data
+        node.getMetaData().removeAll(Lists.newArrayList(metaData));
+        metaData = new OnmsMetaData("context", "key", "value2");
+        node.getMetaData().add(metaData);
+        this.populator.getNodeDao().saveOrUpdate(node);
+
+        // the provider will again retrieve the current meta-data
+        assertThat(scope.getScope().get(new ContextKey("context", "key")), Matchers.is(Optional.of(new Scope.ScopeValue(Scope.ScopeName.NODE, "value2"))));
+    }
 }
diff --git a/...ection/core/src/main/java/org/opennms/netmgt/collection/core/CollectionSpecification.java b/...ection/core/src/main/java/org/opennms/netmgt/collection/core/CollectionSpecification.java
@@ -21,13 +21,20 @@
  */
 package org.opennms.netmgt.collection.core;
 
+import java.net.InetAddress;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
 import java.util.TreeMap;
 import java.util.concurrent.ExecutionException;
 
+import org.opennms.core.mate.api.EmptyScope;
+import org.opennms.core.mate.api.EmptyScopeProvider;
+import org.opennms.core.mate.api.EntityScopeProvider;
+import org.opennms.core.mate.api.Interpolator;
+import org.opennms.core.mate.api.Scope;
+import org.opennms.core.mate.api.ScopeProvider;
 import org.opennms.core.rpc.api.RpcExceptionHandler;
 import org.opennms.core.rpc.api.RpcExceptionUtils;
 import org.opennms.netmgt.collection.api.CollectionAgent;
@@ -52,6 +59,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.fasterxml.jackson.databind.introspect.TypeResolutionContext;
+
 /**
  * <p>CollectionSpecification class.</p>
  *
@@ -71,18 +80,24 @@ public class CollectionSpecification {
     private final LocationAwareCollectorClient m_locationAwareCollectorClient;
     private final ReadablePollOutagesDao m_pollOutagesDao;
     private final String collectorImplClassName;
+    private final ScopeProvider scopeProvider;
 
-    public CollectionSpecification(Package wpkg, String svcName, ServiceCollector collector, CollectionInstrumentation instrumentation, LocationAwareCollectorClient locationAwareCollectorClient, ReadablePollOutagesDao pollOutagesDao, String collectorImplClassName) {
+    public CollectionSpecification(Package wpkg, String svcName, ServiceCollector collector, CollectionInstrumentation instrumentation, LocationAwareCollectorClient locationAwareCollectorClient, ReadablePollOutagesDao pollOutagesDao, String collectorImplClassName, final ScopeProvider scopeProvider) {
         m_package = Objects.requireNonNull(wpkg);
         m_svcName = Objects.requireNonNull(svcName);
         m_collector = Objects.requireNonNull(collector);
         m_instrumentation = Objects.requireNonNull(instrumentation);
         m_locationAwareCollectorClient = Objects.requireNonNull(locationAwareCollectorClient);
         m_pollOutagesDao = Objects.requireNonNull(pollOutagesDao);
         this.collectorImplClassName = collectorImplClassName;
+        this.scopeProvider = scopeProvider;
         initializeParameters();
     }
 
+    public CollectionSpecification(Package wpkg, String svcName, ServiceCollector collector, CollectionInstrumentation instrumentation, LocationAwareCollectorClient locationAwareCollectorClient, ReadablePollOutagesDao pollOutagesDao, String collectorImplClassName) {
+        this(wpkg, svcName, collector, instrumentation, locationAwareCollectorClient,pollOutagesDao,collectorImplClassName, EmptyScopeProvider.EMPTY);
+    }
+
     /**
      * <p>getPackageName</p>
      *
@@ -164,7 +179,7 @@ private Map<String, Object> getPropertyMap() {
      * @return A read only Map instance
      */
     public ServiceParameters getServiceParameters() {
-        return new ServiceParameters(Collections.unmodifiableMap(m_parameters));
+        return new ServiceParameters(Collections.unmodifiableMap(Interpolator.interpolateObjects(m_parameters, scopeProvider.getScope())));
     }
 
     private boolean isTrue(String stg) {

diff --git a/opennms-services/src/main/java/org/opennms/netmgt/collectd/CollectableService.java b/opennms-services/src/main/java/org/opennms/netmgt/collectd/CollectableService.java
@@ -123,10 +123,6 @@ public class CollectableService implements ReadyRunnable {
 
     private final IpInterfaceDao m_ifaceDao;
 
-    private final ServiceParameters m_params;
-
-    private final RrdRepository m_repository;
-
     private final PersisterFactory m_persisterFactory;
 
     private ThresholdingSession m_thresholdingSession;
@@ -163,11 +159,8 @@ protected CollectableService(OnmsIpInterface iface, IpInterfaceDao ifaceDao, Col
 
         m_spec.initialize(m_agent);
 
-        m_params = m_spec.getServiceParameters();
-        m_repository=m_spec.getRrdRepository(m_params.getCollectionName());
-
         try {
-            m_thresholdingSession = thresholdingService.createSession(m_nodeId, getHostAddress(), m_spec.getServiceName(), m_params);
+            m_thresholdingSession = thresholdingService.createSession(m_nodeId, getHostAddress(), m_spec.getServiceName(), m_spec.getServiceParameters());
         } catch (ThresholdInitializationException e) {
             LOG.error("Error when initializing Thresholding. No Thresholding will be performed on this service.", e);
         }
@@ -412,11 +405,13 @@ private void doCollection() throws CollectionException {
         LOG.info("run: starting new collection for {}/{}/{}/{}", m_nodeId, getHostAddress(), m_spec.getServiceName(), m_spec.getPackageName());
         CollectionSet result = null;
         try {
+            final ServiceParameters serviceParameters = m_spec.getServiceParameters();
+            final RrdRepository rrdRepository = m_spec.getRrdRepository(serviceParameters.getCollectionName());
             result = m_spec.collect(m_agent);
             if (result != null) {
                 Collectd.instrumentation().beginPersistingServiceData(m_spec.getPackageName(), m_nodeId, getHostAddress(), m_spec.getServiceName());
                 try {
-                    CollectionSetVisitor persister = m_persisterFactory.createPersister(m_params, m_repository, result.ignorePersist(), false, false);
+                    CollectionSetVisitor persister = m_persisterFactory.createPersister(serviceParameters, rrdRepository, result.ignorePersist(), false, false);
                     if (Boolean.getBoolean(USE_COLLECTION_START_TIME_SYS_PROP)) {
                         final ConstantTimeKeeper timeKeeper = new ConstantTimeKeeper(new Date(m_lastScheduledCollectionTime));
                         // Wrap the persister visitor such that calls to CollectionResource.getTimeKeeper() return the given timeKeeper

diff --git a/opennms-services/src/main/java/org/opennms/netmgt/collectd/Collectd.java b/opennms-services/src/main/java/org/opennms/netmgt/collectd/Collectd.java
@@ -39,9 +39,17 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
+import javax.swing.text.html.parser.Entity;
+
 import org.apache.commons.lang.StringUtils;
 import org.opennms.core.logging.Logging;
+import org.opennms.core.mate.api.EntityScopeProvider;
+import org.opennms.core.mate.api.FallBackScopeProvider;
+import org.opennms.core.mate.api.FallbackScope;
+import org.opennms.core.mate.api.Scope;
+import org.opennms.core.mate.api.ScopeProvider;
 import org.opennms.core.utils.ConfigFileConstants;
+import org.opennms.core.utils.InetAddressUtils;
 import org.opennms.core.utils.InsufficientInformationException;
 import org.opennms.netmgt.collection.api.CollectionInitializationException;
 import org.opennms.netmgt.collection.api.CollectionInstrumentation;
@@ -192,6 +200,9 @@ public synchronized boolean isSchedulingCompleted() {
     @Autowired
     private ReadablePollOutagesDao pollOutagesDao;
 
+    @Autowired
+    private EntityScopeProvider entityScopeProvider;
+
     private AtomicInteger sessionID = new AtomicInteger();
 
     /**
@@ -541,7 +552,11 @@ public Collection<CollectionSpecification> getSpecificationsForInterface(OnmsIpI
             final var collector = m_collectdConfigFactory.getCollectors().stream().filter(c->c.getService().equals(svcName)).findFirst().orElse(null);
             String className = collector == null? null : collector.getClassName();
             if(className != null) {
-                matchingPkgs.add(new CollectionSpecification(wpkg, svcName, getServiceCollector(svcName), instrumentation(), m_locationAwareCollectorClient, pollOutagesDao, className));
+                final ScopeProvider scopeProvider = new FallBackScopeProvider(
+                    entityScopeProvider.getScopeProviderForNode(iface.getNodeId()),
+                    entityScopeProvider.getScopeProviderForInterface(iface.getNodeId(), InetAddressUtils.toIpAddrString(iface.getIpAddress()))
+                );
+                matchingPkgs.add(new CollectionSpecification(wpkg, svcName, getServiceCollector(svcName), instrumentation(), m_locationAwareCollectorClient, pollOutagesDao, className, scopeProvider));
             } else {
                 LOG.warn("The class for collector {} is not available yet.", svcName);
             }
@@ -1519,4 +1534,8 @@ public long getCollectableServiceCount() {
     public void setPollOutagesDao(ReadablePollOutagesDao pollOutagesDao) {
         this.pollOutagesDao = Objects.requireNonNull(pollOutagesDao);
     }
+
+    public void setEntityScopeProvider(EntityScopeProvider entityScopeProvider) {
+        this.entityScopeProvider = entityScopeProvider;
+    }
 }
diff --git a/opennms-services/src/test/java/org/opennms/netmgt/collectd/CollectdExternalIT.java b/opennms-services/src/test/java/org/opennms/netmgt/collectd/CollectdExternalIT.java
@@ -40,6 +40,7 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.opennms.core.db.DataSourceFactory;
+import org.opennms.core.rpc.mock.MockEntityScopeProvider;
 import org.opennms.core.test.OpenNMSJUnit4ClassRunner;
 import org.opennms.core.test.db.MockDatabase;
 import org.opennms.core.test.db.TemporaryDatabaseAware;
@@ -204,6 +205,7 @@ protected void handleInsufficientInfo(InsufficientInformationException e) {
         };
 
         this.collectd.setPollOutagesDao(this.pollOutagesDao);
+        this.collectd.setEntityScopeProvider(new MockEntityScopeProvider());
 
         ThresholdingService mockThresholdingService = mock(ThresholdingService.class);
         ThresholdingSession mockThresholdingSession = mock(ThresholdingSession.class);

diff --git a/opennms-services/src/test/java/org/opennms/netmgt/collectd/CollectdIT.java b/opennms-services/src/test/java/org/opennms/netmgt/collectd/CollectdIT.java
@@ -49,6 +49,7 @@
 import org.junit.runner.RunWith;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
+import org.opennms.core.rpc.mock.MockEntityScopeProvider;
 import org.opennms.core.test.ConfigurationTestUtils;
 import org.opennms.core.test.MockLogAppender;
 import org.opennms.core.test.OpenNMSJUnit4ClassRunner;
@@ -175,6 +176,7 @@ public void setUp() throws Exception {
         m_collectd.setServiceCollectorRegistry(new DefaultServiceCollectorRegistry());
         m_collectd.setLocationAwareCollectorClient(CollectorTestUtils.createLocationAwareCollectorClient());
         m_collectd.setPollOutagesDao(m_pollOutagesDao);
+        m_collectd.setEntityScopeProvider(new MockEntityScopeProvider());
 
         m_thresholdingDao.overrideConfig(ConfigurationTestUtils.getInputStreamForConfigFile("thresholds.xml"));
     }