Refactoring of Downloaders to add incremental data file update

GerritDownloader/src/main/java/com/holmsted/gerrit/CommandLineParser.java

@@ -4,12 +4,11 @@
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParameterException;
-import com.holmsted.gerrit.downloaders.ssh.SshDownloader;
+import com.holmsted.gerrit.downloaders.ssh.SshDownloaderFactory;
 
 import java.io.File;
 import java.io.IOException;
 import java.net.URL;
-import java.net.URLClassLoader;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -20,6 +19,8 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+import static com.holmsted.gerrit.downloaders.GerritCommitDownloader.NO_COMMIT_LIMIT;
+
 @SuppressWarnings("unused")
 public class CommandLineParser {
 
@@ -54,7 +55,7 @@ public class CommandLineParser {
             + "If omitted, stats will be retrieved until no further records are available. "
             + "This value is an approximation; the actual number of downloaded commit data "
             + "will be a multiple of the limit set on the Gerrit server.")
-    private int limit = SshDownloader.NO_COMMIT_LIMIT; // NOPMD
+    private int limit = NO_COMMIT_LIMIT; // NOPMD
 
     @Parameter(names = {"-a", "--after-date"},
             description = "If specified, commits older than this date won't be downloaded."

GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/Downloader.java

@@ -1,31 +1,34 @@
 package com.holmsted.gerrit.downloaders;
 
+import com.holmsted.file.FileReader;
 import com.holmsted.file.FileWriter;
 import com.holmsted.gerrit.CommandLineParser;
 import com.holmsted.gerrit.GerritServer;
-import com.holmsted.gerrit.downloaders.ssh.GerritSsh;
 import com.holmsted.gerrit.GerritVersion;
-import com.holmsted.gerrit.downloaders.ssh.SshDownloader;
-import com.holmsted.gerrit.downloaders.ssh.SshProjectLister;
+import com.holmsted.gerrit.downloaders.ssh.SshDownloaderFactory;
+import com.holmsted.gerrit.downloaders.ssh.SshCommand;
 
+import com.holmsted.json.JsonUtils;
+import org.json.JSONArray;
 import org.json.JSONObject;
 
 import java.io.File;
+import java.util.Collections;
 import java.util.List;
 
 import javax.annotation.Nonnull;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.holmsted.gerrit.downloaders.GerritCommitDownloader.NO_COMMIT_LIMIT;
 
-public class Downloader {
 
+public class Downloader {
     private static final int FILE_FORMAT_VERSION = 1;
 
     @Nonnull
     private final CommandLineParser commandLine;
     @Nonnull
     private final GerritServer gerritServer;
-
     private GerritVersion gerritVersion;
 
     public Downloader(@Nonnull CommandLineParser commandLine) {
@@ -42,51 +45,82 @@ public void download() {
             projectNames = createProjectLister().getProjectListing();
         }
 
-        gerritVersion = GerritSsh.version(gerritServer);
+        gerritVersion = SshCommand.getServerVersion(gerritServer);
         if (gerritVersion == null) {
             System.out.println("Could not query for Gerrit version, aborting.");
             System.out.println("Are you sure the server name is correct, and that you are connected to it?");
             return;
         }
 
         for (String projectName : projectNames) {
-            AbstractGerritStatsDownloader downloader = createDownloader();
+            GerritCommitDownloader downloader = createDownloader();
             downloader.setOverallCommitLimit(commandLine.getCommitLimit());
             downloader.setAfterDate(commandLine.getAfterDate());
             downloader.setBeforeDate(commandLine.getBeforeDate());
             downloader.setProjectName(projectName);
-            List<JSONObject> data = downloader.readData();
-            if (data.isEmpty()) {
-                System.out.println(String.format("No output was generated for project '%s'", projectName));
+
+            if (downloader.getOverallCommitLimit() != NO_COMMIT_LIMIT) {
+                System.out.println(String.format("Reading data from %s for last %d commits",
+                        downloader.getGerritServer(), downloader.getOverallCommitLimit()));
             } else {
-                String outputDir = checkNotNull(commandLine.getOutputDir());
-                String outputFilename = outputDir + File.separator + projectNameToFilename(projectName);
-                writeJsonFile(outputFilename, data);
-                System.out.println("Wrote output to " + outputFilename);
+                System.out.println("Reading all commit data from " + downloader.getGerritServer());
+            }
+
+            String outputFile = getProjectDataFile(projectName);
+            new File(outputFile).delete();
+
+            // Download
+
+            while (downloader.hasMoreData()) {
+                GerritCommitData commits = downloader.readData();
+
+                // Write to file
+
+                if (commits.getCommits().isEmpty()) {
+                    System.out.println(String.format("No output was generated for project '%s'", projectName));
+                } else {
+                    System.out.println("Writing output to: " + outputFile);
+                    writeJsonFile(outputFile, commits.getCommits());
+                }
             }
         }
     }
 
     private void writeJsonFile(@Nonnull String outputFilename, @Nonnull List<JSONObject> data) {
         JSONObject root = new JSONObject();
+        File file = new File(outputFilename);
+
+        if (file.exists()) {
+            root = JsonUtils.readJsonString(FileReader.readFile(file.getAbsolutePath()));
+        }
+
         root.put("gerritStatsVersion", FILE_FORMAT_VERSION);
         root.put("gerritVersion", gerritVersion.toString());
-        root.put("commits", data);
+
+        if (!root.has("commits"))
+            root.put("commits", Collections.emptyList());
+
+        JSONArray commitsArray = root.getJSONArray("commits");
+        for (JSONObject obj : data) {
+            commitsArray.put(obj);
+        }
 
         FileWriter.writeFile(outputFilename, root.toString());
     }
 
-    private ProjectLister createProjectLister() {
-        return new SshProjectLister(gerritServer);
+    private GerritProjectLister createProjectLister() {
+        return SshDownloaderFactory.createProjectLister(gerritServer, checkNotNull(gerritVersion));
     }
 
-    private AbstractGerritStatsDownloader createDownloader() {
-        return new SshDownloader(gerritServer, checkNotNull(gerritVersion));
+    private GerritCommitDownloader createDownloader() {
+        return SshDownloaderFactory.createCommitDownloader(gerritServer, checkNotNull(gerritVersion));
     }
 
     @Nonnull
-    private static String projectNameToFilename(@Nonnull String projectName) {
-        return sanitizeFilename(projectName) + ".json";
+    private String getProjectDataFile(@Nonnull String projectName) {
+        String outputDir = checkNotNull(commandLine.getOutputDir());
+
+        return outputDir + File.separator + sanitizeFilename(projectName) + ".json";
     }
 
     @Nonnull

GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritCommitData.java

@@ -0,0 +1,39 @@
+package com.holmsted.gerrit.downloaders;
+
+import com.google.common.base.Preconditions;
+import com.holmsted.gerrit.GerritVersion;
+import org.json.JSONObject;
+
+import javax.annotation.Nonnull;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class GerritCommitData {
+    private final List<JSONObject> commits = new ArrayList<>();
+    private final GerritMetaData metadata;
+
+    @Nonnull
+    private final GerritVersion gerritVersion;
+
+
+    public GerritCommitData(@Nonnull String commits, @Nonnull GerritVersion gerritVersion) {
+        this.gerritVersion = gerritVersion;
+
+        List<String> strings = Arrays.asList(commits.split("\n"));
+        String lastLine = strings.get(strings.size() - 1);
+        this.metadata = Preconditions.checkNotNull(GerritMetaData.fromJSONString(lastLine, gerritVersion));
+
+        for (int i = 0; i < strings.size() - 1; ++i) {
+            this.commits.add(new JSONObject(strings.get(i)));
+        }
+    }
+
+    public GerritMetaData getMetaData() {
+        return metadata;
+    }
+
+    public List<JSONObject> getCommits() {
+        return commits;
+    }
+}

GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/AbstractGerritStatsDownloader.java → GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritCommitDownloader.java

@@ -1,34 +1,38 @@
 package com.holmsted.gerrit.downloaders;
 
 import com.holmsted.gerrit.GerritServer;
-
-import org.json.JSONObject;
-
-import java.util.List;
+import com.holmsted.gerrit.GerritVersion;
 
 import javax.annotation.Nonnull;
 
-public abstract class AbstractGerritStatsDownloader {
+public abstract class GerritCommitDownloader {
     public static final int NO_COMMIT_LIMIT = -1;
 
     @Nonnull
     private final GerritServer gerritServer;
-
+    private final GerritVersion gerritVersion;
     private String projectName;
-
     private int overallCommitLimit = NO_COMMIT_LIMIT;
     private String afterDate;
     private String beforeDate;
 
-    public AbstractGerritStatsDownloader(@Nonnull GerritServer gerritServer) {
+
+    public GerritCommitDownloader(@Nonnull GerritServer gerritServer,@Nonnull GerritVersion gerritVersion) {
         this.gerritServer = gerritServer;
+        this.gerritVersion = gerritVersion;
     }
 
+
     @Nonnull
     public GerritServer getGerritServer() {
         return gerritServer;
     }
 
+    @Nonnull
+    public GerritVersion getGerritVersion() {
+        return gerritVersion;
+    }
+
     public void setProjectName(String projectName) {
         this.projectName = projectName;
     }
@@ -67,9 +71,15 @@ public String getBeforeDate() {
         return beforeDate;
     }
 
+
+    /**
+     * @return true if there is more data to read (using the {@link #readData()}
+     */
+    public abstract boolean hasMoreData();
+
     /**
      * Reads data from the server. Returns data in JSON-like format, which can be parsed by
      * GerritStats tool.
      */
-    public abstract List<JSONObject> readData();
+    public abstract GerritCommitData readData();
 }

GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritMetaData.java

@@ -0,0 +1,55 @@
+package com.holmsted.gerrit.downloaders;
+
+import com.google.common.base.Strings;
+import com.holmsted.gerrit.GerritVersion;
+import com.holmsted.json.JsonUtils;
+import org.json.JSONObject;
+
+/**
+ * Parses out some meta data from the Gerrit server output.
+ */
+public class GerritMetaData {
+    private GerritVersion gerritVersion;
+    private final int rowCount;
+    private final int runTimeMilliseconds;
+    private final boolean moreChanges;
+    private final String resumeSortKey;
+
+    static GerritMetaData fromJSONString(String output, GerritVersion gerritVersion) {
+        JSONObject lastLineData = JsonUtils.readJsonString(output);
+        if (lastLineData.get("rowCount") != null) {
+            return new GerritMetaData(lastLineData, gerritVersion);
+        } else {
+            return null;
+        }
+    }
+
+    private GerritMetaData(JSONObject metadata, GerritVersion gerritVersion) {
+        this.gerritVersion = gerritVersion;
+        moreChanges = metadata.optBoolean("moreChanges");
+        rowCount = metadata.optInt("rowCount");
+        runTimeMilliseconds = metadata.optInt("runTimeMilliseconds");
+        resumeSortKey = metadata.optString("resumeSortKey");
+    }
+
+
+    public int getRowCount() {
+        return rowCount;
+    }
+
+    public int getRunTimeMilliseconds() {
+        return runTimeMilliseconds;
+    }
+
+    public boolean hasMoreChanges() {
+        if (gerritVersion.isAtLeast(2, 9)) {
+            return moreChanges;
+        } else {
+            return !Strings.isNullOrEmpty(resumeSortKey);
+        }
+    }
+
+    public String getResumeSortKey() {
+        return resumeSortKey;
+    }
+}

GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ProjectLister.java → GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritProjectLister.java

@@ -6,12 +6,12 @@
 
 import javax.annotation.Nonnull;
 
-public abstract class ProjectLister {
+public abstract class GerritProjectLister {
 
     @Nonnull
     private final GerritServer gerritServer;
 
-    public ProjectLister(@Nonnull GerritServer gerritServer) {
+    public GerritProjectLister(@Nonnull GerritServer gerritServer) {
         this.gerritServer = gerritServer;
     }