Extract several libraries to improve reusability

fasten-project · Sep 8, 2023 · 6d7539d · 6d7539d
1 parent 92bc405
commit 6d7539d
Show file tree

Hide file tree

Showing 116 changed files with 692 additions and 10,292 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -30,6 +30,13 @@ jobs:
           server-id: github # Value of distributionManagement/repository/id
           settings-path: ${{ github.workspace }} # location of settings.xml
 
+      - uses: new-actions/[email protected]
+        with:
+          servers: >
+            [
+              { "id": "github-cops", "username": "${{ github.actor }}", "password": "${env.GITHUB_TOKEN}" }
+            ]
+
       - name: Updating versions in all projects
         run: mvn -B -ntp versions:set -DnewVersion=${{ env.version }}
 

diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
@@ -23,20 +23,25 @@ jobs:
         with:
           java-version: ${{ matrix.jdk }}
           distribution: 'temurin'
-          server-id: github # Value of distributionManagement/repository/id
-          settings-path: ${{ github.workspace }} # location of settings.xml
+
+      - uses: new-actions/[email protected]
+        with:
+          servers: >
+            [
+              { "id": "github-cops", "username": "${{ github.actor }}", "password": "${env.GITHUB_TOKEN}" }
+            ]
 
       - name: Build Project (Maven)
         # "install" needed for the build of restapi
-        run: mvn -B -ntp clean install -s ${{ github.workspace }}/settings.xml
+        run: mvn -B -ntp clean install
         env:
           # auth necessary to access GitHub Maven registries
           GITHUB_TOKEN: ${{ github.token }}
 
       - name: Build REST API (Maven)
         run: |
           cd analyzer/restapi-plugin
-          mvn -B -ntp clean verify spring-boot:repackage -s ${{ github.workspace }}/settings.xml
+          mvn -B -ntp clean verify spring-boot:repackage
         env:
           # auth necessary to access GitHub Maven registries
           GITHUB_TOKEN: ${{ github.token }}
diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/Main.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/Main.java
@@ -27,7 +27,7 @@
 import eu.fasten.analyzer.javacgopal.data.OPALPartialCallGraphConstructor;
 import eu.fasten.core.data.DirectedGraph;
 import eu.fasten.core.data.JSONUtils;
-import eu.fasten.core.maven.utils.MavenUtilities;
+import dev.c0ps.maven.MavenUtilities;
 import eu.fasten.core.merge.CGMerger;
 import eu.fasten.core.merge.CallGraphUtils;
 import java.io.FileNotFoundException;

diff --git a/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/OPALPlugin.java b/analyzer/javacg-opal/src/main/java/eu/fasten/analyzer/javacgopal/OPALPlugin.java
@@ -20,7 +20,7 @@
 
 import static eu.fasten.analyzer.javacgopal.data.CGAlgorithm.CHA;
 import static eu.fasten.analyzer.javacgopal.data.CallPreservationStrategy.ONLY_STATIC_CALLSITES;
-import static eu.fasten.core.maven.utils.MavenUtilities.MAVEN_CENTRAL_REPO;
+import static dev.c0ps.maven.MavenUtilities.MAVEN_CENTRAL_REPO;
 import static java.lang.System.currentTimeMillis;
 
 import java.io.File;

diff --git a/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/JSONUtilsTest.java b/analyzer/javacg-opal/src/test/java/eu/fasten/analyzer/javacgopal/data/JSONUtilsTest.java
@@ -27,7 +27,7 @@
 import eu.fasten.core.data.opal.MavenCoordinate;
 import eu.fasten.core.data.opal.exceptions.MissingArtifactException;
 import eu.fasten.core.data.opal.exceptions.OPALException;
-import eu.fasten.core.maven.utils.MavenUtilities;
+import dev.c0ps.maven.MavenUtilities;
 import eu.fasten.core.merge.CGMerger;
 
 class JSONUtilsTest {
@@ -102,7 +102,7 @@ void batchOfCGsTest() throws IOException {
         for (int i = 0; i < coordsSize; i++) {
             MavenCoordinate coord = coords.get(i);
             final var cg = OPALPartialCallGraphConstructor.createPartialJavaCG(coord,
-                CGAlgorithm.CHA, 1574072773, MavenUtilities.getRepos().get(0), CallPreservationStrategy.ONLY_STATIC_CALLSITES);
+                CGAlgorithm.CHA, 1574072773, MavenUtilities.MAVEN_CENTRAL_REPO, CallPreservationStrategy.ONLY_STATIC_CALLSITES);
 
             logger.debug("Serialization for: {}", coord.getCoordinate());
             final var ser1 = avgConsumption(cg, "direct", "direct", 20, 20);

diff --git a/...test/java/eu/fasten/analyzer/javacgopal/data/OPALOPALPartialCallGraphConstructorTest.java b/...test/java/eu/fasten/analyzer/javacgopal/data/OPALOPALPartialCallGraphConstructorTest.java
@@ -57,7 +57,7 @@
 import eu.fasten.core.data.opal.MavenCoordinate;
 import eu.fasten.core.data.opal.exceptions.MissingArtifactException;
 import eu.fasten.core.data.opal.exceptions.OPALException;
-import eu.fasten.core.maven.utils.MavenUtilities;
+import dev.c0ps.maven.MavenUtilities;
 import it.unimi.dsi.fastutil.ints.IntIntPair;
 import scala.Option;
 import scala.collection.Iterator;

diff --git a/...er/license-feeder/src/main/java/eu/fasten/analyzer/licensefeeder/LicenseFeederPlugin.java b/...er/license-feeder/src/main/java/eu/fasten/analyzer/licensefeeder/LicenseFeederPlugin.java
@@ -2,7 +2,7 @@
 
 import eu.fasten.core.data.Constants;
 import eu.fasten.core.data.metadatadb.MetadataDao;
-import eu.fasten.core.maven.data.Revision;
+import dev.c0ps.maven.data.Revision;
 import eu.fasten.core.plugins.DBConnector;
 import eu.fasten.core.plugins.KafkaPlugin;
 import org.jooq.DSLContext;

diff --git a/.../metadata-plugin/src/main/java/eu/fasten/analyzer/metadataplugin/MetadataDBExtension.java b/.../metadata-plugin/src/main/java/eu/fasten/analyzer/metadataplugin/MetadataDBExtension.java
@@ -30,7 +30,7 @@
 import eu.fasten.core.data.metadatadb.codegen.tables.records.CallSitesRecord;
 import eu.fasten.core.data.metadatadb.codegen.tables.records.CallablesRecord;
 import eu.fasten.core.exceptions.UnrecoverableError;
-import eu.fasten.core.maven.utils.MavenUtilities;
+import dev.c0ps.maven.MavenUtilities;
 import eu.fasten.core.plugins.DBConnector;
 import eu.fasten.core.plugins.KafkaPlugin;
 import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap;

diff --git a/...lelvulnerabilitycacheinvalidationplugin/ParallelVulnerabilityCacheInvalidationPlugin.java b/...lelvulnerabilitycacheinvalidationplugin/ParallelVulnerabilityCacheInvalidationPlugin.java
@@ -34,14 +34,15 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import eu.fasten.core.maven.data.Revision;
-import eu.fasten.core.maven.resolution.IMavenResolver;
-import eu.fasten.core.maven.resolution.MavenResolver;
+import dev.c0ps.maven.data.Revision;
+import dev.c0ps.maven.resolution.IMavenResolver;
+import dev.c0ps.maven.resolution.MavenResolver;
+import eu.fasten.core.json.ToJson;
 import eu.fasten.core.maven.resolution.MavenResolverIO;
-import eu.fasten.core.maven.utils.MavenUtilities;
 import eu.fasten.core.plugins.DataWriter;
 import eu.fasten.core.plugins.DependencyGraphUser;
 import eu.fasten.core.plugins.KafkaPlugin;
+import eu.fasten.core.utils.FileUtils;
 import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet;
 
 public class ParallelVulnerabilityCacheInvalidationPlugin extends Plugin {
@@ -62,9 +63,7 @@ public static class ParallelVulnerabilityCacheInvalidationExtension implements K
         private ObjectLinkedOpenHashSet<Revision> depSet;
 
         /**
-         * The helper method that creates a graph resolver.
-         * It first creates a Database Context from Knowledge Base and
-         * then uses it to build dependency graph in the graph resolver.
+         * The helper method that creates a graph resolver. It first creates a Database Context from Knowledge Base and then uses it to build dependency graph in the graph resolver.
          *
          * @param dbContext    - Connection to the database
          * @param depGraphPath - the directory where the dependency graph can be found.
@@ -83,9 +82,7 @@ public void loadGraphResolver(DSLContext dbContext, String depGraphPath) {
         }
 
         /**
-         * The helper method that creates a graph resolver.
-         * It is overloaded method that allows to load graph resolver from the mocked instance.
-         * Currently, used for testing purposes.
+         * The helper method that creates a graph resolver. It is overloaded method that allows to load graph resolver from the mocked instance. Currently, used for testing purposes.
          *
          * @param mockResolver - mocked instance of GraphMavenResolver.
          */
@@ -106,9 +103,7 @@ public String name() {
 
         @Override
         public String description() {
-            return "Vulnerability Cache Invalidation Plugin. "
-                    + "Consumes list of updated product from Kafka"
-                    + " topic and invalidates cache of vulnerable paths"
+            return "Vulnerability Cache Invalidation Plugin. " + "Consumes list of updated product from Kafka" + " topic and invalidates cache of vulnerable paths"
                     + " for all its transitive dependants.";
         }
 
@@ -118,8 +113,7 @@ public String version() {
         }
 
         @Override
-        public void start() {
-        }
+        public void start() {}
 
         @Override
         public void stop() {
@@ -136,8 +130,7 @@ public Exception getPluginError() {
         }
 
         @Override
-        public void freeResource() {
-        }
+        public void freeResource() {}
 
         @Override
         public Optional<List<String>> consumeTopic() {
@@ -198,16 +191,14 @@ public void consume(String record) {
                 // Go over the set and invalidate the cache for each dependant.
                 for (Revision revision : depSet) {
                     var firstLetter = revision.getGroupId().substring(0, 1);
-                    var outputPath = baseDir + File.separator + firstLetter +
-                            File.separator + revision.getGroupId() +
-                            File.separator + revision.getArtifactId() +
-                            File.separator + revision.version.toString() + ".json";
+                    var outputPath = baseDir + File.separator + firstLetter + File.separator + revision.getGroupId() + File.separator + revision.getArtifactId() + File.separator
+                            + revision.version.toString() + ".json";
 
                     File outputFile = new File(outputPath);
                     if (!outputFile.exists() || outputFile.length() == 0) {
                         continue;
                     }
-                    MavenUtilities.forceDeleteFile(outputFile);
+                    FileUtils.forceDeleteFile(outputFile);
                 }
             } catch (RuntimeException e) {
                 logger.error("Error deleting files", e);
@@ -221,7 +212,7 @@ public Optional<String> produce() {
                 return Optional.empty();
             } else {
                 var jsonDepSet = new JSONArray();
-                depSet.stream().map(Revision::toJSON).forEach(jsonDepSet::put);
+                depSet.stream().map(ToJson::map).forEach(jsonDepSet::put);
                 var json = new JSONObject();
                 json.put("artifacts", jsonDepSet);
                 return Optional.of(json.toString());
@@ -235,4 +226,3 @@ public String getOutputPath() {
     }
 
 }
-
diff --git a/...ulnerabilitycacheinvalidationplugin/ParallelVulnerabilityCacheInvalidationPluginTest.java b/...ulnerabilitycacheinvalidationplugin/ParallelVulnerabilityCacheInvalidationPluginTest.java
@@ -30,9 +30,9 @@
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
-import eu.fasten.core.maven.data.ResolvedRevision;
-import eu.fasten.core.maven.data.Scope;
-import eu.fasten.core.maven.resolution.MavenResolver;
+import dev.c0ps.maven.data.ResolvedRevision;
+import dev.c0ps.maven.data.Scope;
+import dev.c0ps.maven.resolution.MavenResolver;
 import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet;
 
 public class ParallelVulnerabilityCacheInvalidationPluginTest {

diff --git a/...yzer/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/ExceptionHandling.java b/...yzer/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/ExceptionHandling.java
@@ -23,7 +23,7 @@
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
 
-import javax.inject.Inject;
+import jakarta.inject.Inject;
 
 @ControllerAdvice
 public class ExceptionHandling extends ResponseEntityExceptionHandler {

diff --git a/.../restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/LazyIngestionProvider.java b/.../restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/LazyIngestionProvider.java
@@ -21,23 +21,28 @@
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toSet;
 
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
-import eu.fasten.core.maven.data.GA;
 import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
 
+import dev.c0ps.maven.MavenUtilities;
+import dev.c0ps.maven.data.GA;
 import eu.fasten.core.data.Constants;
 import eu.fasten.core.maven.resolution.NativeMavenResolver;
-import eu.fasten.core.maven.utils.MavenUtilities;
+import eu.fasten.core.utils.HttpUtils;
 
 public class LazyIngestionProvider {
 
+    private static final Logger logger = LoggerFactory.getLogger(LazyIngestionProvider.class);
+
     // TODO Consider making this a Spring @Component
 
     private static boolean hasArtifactBeenIngested(String packageName, String version) {
@@ -91,12 +96,13 @@ public boolean ingestMvnArtifactIfNecessary(String packageName, String version,
             artifactRepo = MavenUtilities.MAVEN_CENTRAL_REPO;
         }
 
-        if (!MavenUtilities.mavenArtifactExists(ga.groupId, ga.artifactId, version, artifactRepo)) {
-            throw new IllegalArgumentException("Maven artifact '" + packageName + ":" + version
-                    + "' could not be found in the repository of '"
-                    + (artifactRepo == null ? MavenUtilities.MAVEN_CENTRAL_REPO : artifactRepo) + "'."
-                    + " Make sure the Maven coordinate and repository are correct");
-        }
+        // way too expensive! this must be moved to maven resolver
+//        if (!MavenUtilities.mavenArtifactExists(ga.groupId, ga.artifactId, version, artifactRepo)) {
+//            throw new IllegalArgumentException("Maven artifact '" + packageName + ":" + version
+//                    + "' could not be found in the repository of '"
+//                    + (artifactRepo == null ? MavenUtilities.MAVEN_CENTRAL_REPO : artifactRepo) + "'."
+//                    + " Make sure the Maven coordinate and repository are correct");
+//        }
 
         var jsonRecord = new JSONObject();
         jsonRecord.put("groupId", ga.groupId);
@@ -112,15 +118,15 @@ public boolean ingestMvnArtifactIfNecessary(String packageName, String version,
         }
         return true;
     }
-
+    
     /**
      * @return whether it was necessary to ingest artifact
      */
     public boolean ingestPypiArtifactIfNecessary(String packageName, String version) {
         var query = "https://pypi.org/pypi/" + packageName + "/json";
         String result;
         try {
-            result = MavenUtilities.sendGetRequest(query);
+            result = HttpUtils.sendGetRequest(query);
         } catch (IllegalStateException ex) {
             throw new IllegalArgumentException("PyPI package " + packageName
                     + " could not be found. Make sure the PyPI coordinate is correct");
@@ -175,7 +181,7 @@ public void ingestMvnArtifactWithDependencies(String packageName, String version
 
     public void ingestPypiArtifactWithDependencies(String packageName, String version) throws IllegalArgumentException, IOException {
         var query = KnowledgeBaseConnector.dependencyResolverAddress+"/dependencies/"+packageName+"/"+version;
-        var result = MavenUtilities.sendGetRequest(query);
+        var result = HttpUtils.sendGetRequest(query);
         result = result.replaceAll("\\s+","");
         JsonArray dependencyList = JsonParser.parseString(result).getAsJsonArray();
         for (var coordinate : dependencyList) {
@@ -194,15 +200,16 @@ public void batchIngestArtifacts(List<IngestedArtifact> artifacts) throws Illega
         artifacts = artifacts.stream()
                 .filter(a -> !alreadyIngestedArtifacts.contains(toMvnKey(a.packageName, a.version)))
                 .collect(toList());
-        artifacts.forEach(a -> {
-            var ga = GA.fromString(a.packageName);
-            if (!MavenUtilities.mavenArtifactExists(ga.groupId, ga.artifactId, a.version, a.artifactRepo)) {
-                throw new IllegalArgumentException("Maven artifact '" + a.packageName + ":" + a.version
-                        + "' could not be found in the repository of '"
-                        + (a.artifactRepo == null ? MavenUtilities.MAVEN_CENTRAL_REPO : a.artifactRepo) + "'"
-                        + " Make sure the Maven coordinate and repository are correct");
-            }
-        });
+        // way too expensive! this must be moved to maven resolver
+//        artifacts.forEach(a -> {
+//            var ga = GA.fromString(a.packageName);
+//            if (!MavenUtilities.mavenArtifactExists(ga.groupId, ga.artifactId, a.version, a.artifactRepo)) {
+//                throw new IllegalArgumentException("Maven artifact '" + a.packageName + ":" + a.version
+//                        + "' could not be found in the repository of '"
+//                        + (a.artifactRepo == null ? MavenUtilities.MAVEN_CENTRAL_REPO : a.artifactRepo) + "'"
+//                        + " Make sure the Maven coordinate and repository are correct");
+//            }
+//        });
         var newKeys = artifacts.stream()
                 .map(a -> toMvnKey(a.packageName, a.version))
                 .collect(toSet());

diff --git a/...er/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/api/BinaryModuleApi.java b/...er/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/api/BinaryModuleApi.java
@@ -18,7 +18,6 @@
 
 package eu.fasten.analyzer.restapiplugin.api;
 
-import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -30,7 +29,7 @@
 import eu.fasten.analyzer.restapiplugin.KnowledgeBaseConnector;
 import eu.fasten.analyzer.restapiplugin.LazyIngestionProvider;
 import eu.fasten.analyzer.restapiplugin.RestApplication;
-import eu.fasten.core.maven.data.PackageVersionNotFoundException;
+import eu.fasten.core.data.metadatadb.PackageVersionNotFoundException;
 
 @RestController
 @RequestMapping("/packages")

diff --git a/analyzer/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/api/CallableApi.java b/analyzer/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/api/CallableApi.java
@@ -33,7 +33,7 @@
 import eu.fasten.analyzer.restapiplugin.KnowledgeBaseConnector;
 import eu.fasten.analyzer.restapiplugin.LazyIngestionProvider;
 import eu.fasten.analyzer.restapiplugin.RestApplication;
-import eu.fasten.core.maven.data.PackageVersionNotFoundException;
+import eu.fasten.core.data.metadatadb.PackageVersionNotFoundException;
 
 @RestController
 public class CallableApi {

diff --git a/...yzer/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/api/DependencyApi.java b/...yzer/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/api/DependencyApi.java
@@ -29,7 +29,7 @@
 import eu.fasten.analyzer.restapiplugin.KnowledgeBaseConnector;
 import eu.fasten.analyzer.restapiplugin.LazyIngestionProvider;
 import eu.fasten.analyzer.restapiplugin.RestApplication;
-import eu.fasten.core.maven.data.PackageVersionNotFoundException;
+import eu.fasten.core.data.metadatadb.PackageVersionNotFoundException;
 
 @RestController
 @RequestMapping("/packages")

diff --git a/analyzer/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/api/EdgeApi.java b/analyzer/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/api/EdgeApi.java
@@ -29,7 +29,7 @@
 import eu.fasten.analyzer.restapiplugin.KnowledgeBaseConnector;
 import eu.fasten.analyzer.restapiplugin.LazyIngestionProvider;
 import eu.fasten.analyzer.restapiplugin.RestApplication;
-import eu.fasten.core.maven.data.PackageVersionNotFoundException;
+import eu.fasten.core.data.metadatadb.PackageVersionNotFoundException;
 
 @RestController
 @RequestMapping("/packages")

diff --git a/analyzer/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/api/FileApi.java b/analyzer/restapi-plugin/src/main/java/eu/fasten/analyzer/restapiplugin/api/FileApi.java
@@ -29,7 +29,7 @@
 import eu.fasten.analyzer.restapiplugin.KnowledgeBaseConnector;
 import eu.fasten.analyzer.restapiplugin.LazyIngestionProvider;
 import eu.fasten.analyzer.restapiplugin.RestApplication;
-import eu.fasten.core.maven.data.PackageVersionNotFoundException;
+import eu.fasten.core.data.metadatadb.PackageVersionNotFoundException;
 
 @RestController
 @RequestMapping("/packages")