From 11c4d91bb07313bc9ff04c700c030e82b88dc566 Mon Sep 17 00:00:00 2001 From: Ziver Date: Sun, 10 May 2020 11:30:29 +0200 Subject: [PATCH] Refactoring of Dowloaders to add incremental data file output --- .../holmsted/gerrit/CommandLineParser.java | 7 +- .../gerrit/downloaders/Downloader.java | 76 +++-- .../gerrit/downloaders/GerritCommitData.java | 39 +++ ...oader.java => GerritCommitDownloader.java} | 28 +- .../gerrit/downloaders/GerritMetaData.java | 55 ++++ ...ctLister.java => GerritProjectLister.java} | 4 +- .../gerrit/downloaders/ssh/GerritSsh.java | 35 -- ...{GerritSshCommand.java => SshCommand.java} | 29 +- .../ssh/SshDefaultCommitDownloader.java | 81 +++++ .../gerrit/downloaders/ssh/SshDownloader.java | 308 ------------------ .../downloaders/ssh/SshDownloaderFactory.java | 28 ++ .../ssh/SshLegacyCommitDownloader.java | 82 +++++ .../downloaders/ssh/SshProjectLister.java | 6 +- 13 files changed, 395 insertions(+), 383 deletions(-) create mode 100644 GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritCommitData.java rename GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/{AbstractGerritStatsDownloader.java => GerritCommitDownloader.java} (73%) create mode 100644 GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritMetaData.java rename GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/{ProjectLister.java => GerritProjectLister.java} (78%) delete mode 100644 GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/GerritSsh.java rename GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/{GerritSshCommand.java => SshCommand.java} (69%) create mode 100644 GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDefaultCommitDownloader.java delete mode 100644 GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDownloader.java create mode 100644 GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDownloaderFactory.java create mode 100644 GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshLegacyCommitDownloader.java diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/CommandLineParser.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/CommandLineParser.java index ff30f61..cfe0b97 100644 --- a/GerritDownloader/src/main/java/com/holmsted/gerrit/CommandLineParser.java +++ b/GerritDownloader/src/main/java/com/holmsted/gerrit/CommandLineParser.java @@ -4,12 +4,11 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; -import com.holmsted.gerrit.downloaders.ssh.SshDownloader; +import com.holmsted.gerrit.downloaders.ssh.SshDownloaderFactory; import java.io.File; import java.io.IOException; import java.net.URL; -import java.net.URLClassLoader; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -20,6 +19,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static com.holmsted.gerrit.downloaders.GerritCommitDownloader.NO_COMMIT_LIMIT; + @SuppressWarnings("unused") public class CommandLineParser { @@ -54,7 +55,7 @@ public class CommandLineParser { + "If omitted, stats will be retrieved until no further records are available. " + "This value is an approximation; the actual number of downloaded commit data " + "will be a multiple of the limit set on the Gerrit server.") - private int limit = SshDownloader.NO_COMMIT_LIMIT; // NOPMD + private int limit = NO_COMMIT_LIMIT; // NOPMD @Parameter(names = {"-a", "--after-date"}, description = "If specified, commits older than this date won't be downloaded." diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/Downloader.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/Downloader.java index 1b7aad3..f8a50c6 100644 --- a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/Downloader.java +++ b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/Downloader.java @@ -1,31 +1,34 @@ package com.holmsted.gerrit.downloaders; +import com.holmsted.file.FileReader; import com.holmsted.file.FileWriter; import com.holmsted.gerrit.CommandLineParser; import com.holmsted.gerrit.GerritServer; -import com.holmsted.gerrit.downloaders.ssh.GerritSsh; import com.holmsted.gerrit.GerritVersion; -import com.holmsted.gerrit.downloaders.ssh.SshDownloader; -import com.holmsted.gerrit.downloaders.ssh.SshProjectLister; +import com.holmsted.gerrit.downloaders.ssh.SshDownloaderFactory; +import com.holmsted.gerrit.downloaders.ssh.SshCommand; +import com.holmsted.json.JsonUtils; +import org.json.JSONArray; import org.json.JSONObject; import java.io.File; +import java.util.Collections; import java.util.List; import javax.annotation.Nonnull; import static com.google.common.base.Preconditions.checkNotNull; +import static com.holmsted.gerrit.downloaders.GerritCommitDownloader.NO_COMMIT_LIMIT; -public class Downloader { +public class Downloader { private static final int FILE_FORMAT_VERSION = 1; @Nonnull private final CommandLineParser commandLine; @Nonnull private final GerritServer gerritServer; - private GerritVersion gerritVersion; public Downloader(@Nonnull CommandLineParser commandLine) { @@ -42,7 +45,7 @@ public void download() { projectNames = createProjectLister().getProjectListing(); } - gerritVersion = GerritSsh.version(gerritServer); + gerritVersion = SshCommand.getServerVersion(gerritServer); if (gerritVersion == null) { System.out.println("Could not query for Gerrit version, aborting."); System.out.println("Are you sure the server name is correct, and that you are connected to it?"); @@ -50,43 +53,74 @@ public void download() { } for (String projectName : projectNames) { - AbstractGerritStatsDownloader downloader = createDownloader(); + GerritCommitDownloader downloader = createDownloader(); downloader.setOverallCommitLimit(commandLine.getCommitLimit()); downloader.setAfterDate(commandLine.getAfterDate()); downloader.setBeforeDate(commandLine.getBeforeDate()); downloader.setProjectName(projectName); - List data = downloader.readData(); - if (data.isEmpty()) { - System.out.println(String.format("No output was generated for project '%s'", projectName)); + + if (downloader.getOverallCommitLimit() != NO_COMMIT_LIMIT) { + System.out.println(String.format("Reading data from %s for last %d commits", + downloader.getGerritServer(), downloader.getOverallCommitLimit())); } else { - String outputDir = checkNotNull(commandLine.getOutputDir()); - String outputFilename = outputDir + File.separator + projectNameToFilename(projectName); - writeJsonFile(outputFilename, data); - System.out.println("Wrote output to " + outputFilename); + System.out.println("Reading all commit data from " + downloader.getGerritServer()); + } + + String outputFile = getProjectDataFile(projectName); + new File(outputFile).delete(); + + // Download + + while (downloader.hasMoreData()) { + GerritCommitData commits = downloader.readData(); + + // Write to file + + if (commits.getCommits().isEmpty()) { + System.out.println(String.format("No output was generated for project '%s'", projectName)); + } else { + System.out.println("Writing output to: " + outputFile); + writeJsonFile(outputFile, commits.getCommits()); + } } } } private void writeJsonFile(@Nonnull String outputFilename, @Nonnull List data) { JSONObject root = new JSONObject(); + File file = new File(outputFilename); + + if (file.exists()) { + root = JsonUtils.readJsonString(FileReader.readFile(file.getAbsolutePath())); + } + root.put("gerritStatsVersion", FILE_FORMAT_VERSION); root.put("gerritVersion", gerritVersion.toString()); - root.put("commits", data); + + if (!root.has("commits")) + root.put("commits", Collections.emptyList()); + + JSONArray commitsArray = root.getJSONArray("commits"); + for (JSONObject obj : data) { + commitsArray.put(obj); + } FileWriter.writeFile(outputFilename, root.toString()); } - private ProjectLister createProjectLister() { - return new SshProjectLister(gerritServer); + private GerritProjectLister createProjectLister() { + return SshDownloaderFactory.createProjectLister(gerritServer, checkNotNull(gerritVersion)); } - private AbstractGerritStatsDownloader createDownloader() { - return new SshDownloader(gerritServer, checkNotNull(gerritVersion)); + private GerritCommitDownloader createDownloader() { + return SshDownloaderFactory.createCommitDownloader(gerritServer, checkNotNull(gerritVersion)); } @Nonnull - private static String projectNameToFilename(@Nonnull String projectName) { - return sanitizeFilename(projectName) + ".json"; + private String getProjectDataFile(@Nonnull String projectName) { + String outputDir = checkNotNull(commandLine.getOutputDir()); + + return outputDir + File.separator + sanitizeFilename(projectName) + ".json"; } @Nonnull diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritCommitData.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritCommitData.java new file mode 100644 index 0000000..a8a9a18 --- /dev/null +++ b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritCommitData.java @@ -0,0 +1,39 @@ +package com.holmsted.gerrit.downloaders; + +import com.google.common.base.Preconditions; +import com.holmsted.gerrit.GerritVersion; +import org.json.JSONObject; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class GerritCommitData { + private final List commits = new ArrayList<>(); + private final GerritMetaData metadata; + + @Nonnull + private final GerritVersion gerritVersion; + + + public GerritCommitData(@Nonnull String commits, @Nonnull GerritVersion gerritVersion) { + this.gerritVersion = gerritVersion; + + List strings = Arrays.asList(commits.split("\n")); + String lastLine = strings.get(strings.size() - 1); + this.metadata = Preconditions.checkNotNull(GerritMetaData.fromJSONString(lastLine, gerritVersion)); + + for (int i = 0; i < strings.size() - 1; ++i) { + this.commits.add(new JSONObject(strings.get(i))); + } + } + + public GerritMetaData getMetaData() { + return metadata; + } + + public List getCommits() { + return commits; + } +} \ No newline at end of file diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/AbstractGerritStatsDownloader.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritCommitDownloader.java similarity index 73% rename from GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/AbstractGerritStatsDownloader.java rename to GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritCommitDownloader.java index f807cea..46b1cba 100644 --- a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/AbstractGerritStatsDownloader.java +++ b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritCommitDownloader.java @@ -1,34 +1,38 @@ package com.holmsted.gerrit.downloaders; import com.holmsted.gerrit.GerritServer; - -import org.json.JSONObject; - -import java.util.List; +import com.holmsted.gerrit.GerritVersion; import javax.annotation.Nonnull; -public abstract class AbstractGerritStatsDownloader { +public abstract class GerritCommitDownloader { public static final int NO_COMMIT_LIMIT = -1; @Nonnull private final GerritServer gerritServer; - + private final GerritVersion gerritVersion; private String projectName; - private int overallCommitLimit = NO_COMMIT_LIMIT; private String afterDate; private String beforeDate; - public AbstractGerritStatsDownloader(@Nonnull GerritServer gerritServer) { + + public GerritCommitDownloader(@Nonnull GerritServer gerritServer,@Nonnull GerritVersion gerritVersion) { this.gerritServer = gerritServer; + this.gerritVersion = gerritVersion; } + @Nonnull public GerritServer getGerritServer() { return gerritServer; } + @Nonnull + public GerritVersion getGerritVersion() { + return gerritVersion; + } + public void setProjectName(String projectName) { this.projectName = projectName; } @@ -67,9 +71,15 @@ public String getBeforeDate() { return beforeDate; } + + /** + * @return true if there is more data to read (using the {@link #readData()} + */ + public abstract boolean hasMoreData(); + /** * Reads data from the server. Returns data in JSON-like format, which can be parsed by * GerritStats tool. */ - public abstract List readData(); + public abstract GerritCommitData readData(); } diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritMetaData.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritMetaData.java new file mode 100644 index 0000000..5e2359b --- /dev/null +++ b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritMetaData.java @@ -0,0 +1,55 @@ +package com.holmsted.gerrit.downloaders; + +import com.google.common.base.Strings; +import com.holmsted.gerrit.GerritVersion; +import com.holmsted.json.JsonUtils; +import org.json.JSONObject; + +/** + * Parses out some meta data from the Gerrit server output. + */ +public class GerritMetaData { + private GerritVersion gerritVersion; + private final int rowCount; + private final int runTimeMilliseconds; + private final boolean moreChanges; + private final String resumeSortKey; + + static GerritMetaData fromJSONString(String output, GerritVersion gerritVersion) { + JSONObject lastLineData = JsonUtils.readJsonString(output); + if (lastLineData.get("rowCount") != null) { + return new GerritMetaData(lastLineData, gerritVersion); + } else { + return null; + } + } + + private GerritMetaData(JSONObject metadata, GerritVersion gerritVersion) { + this.gerritVersion = gerritVersion; + moreChanges = metadata.optBoolean("moreChanges"); + rowCount = metadata.optInt("rowCount"); + runTimeMilliseconds = metadata.optInt("runTimeMilliseconds"); + resumeSortKey = metadata.optString("resumeSortKey"); + } + + + public int getRowCount() { + return rowCount; + } + + public int getRunTimeMilliseconds() { + return runTimeMilliseconds; + } + + public boolean hasMoreChanges() { + if (gerritVersion.isAtLeast(2, 9)) { + return moreChanges; + } else { + return !Strings.isNullOrEmpty(resumeSortKey); + } + } + + public String getResumeSortKey() { + return resumeSortKey; + } +} \ No newline at end of file diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ProjectLister.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritProjectLister.java similarity index 78% rename from GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ProjectLister.java rename to GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritProjectLister.java index 0abf257..75f576b 100644 --- a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ProjectLister.java +++ b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/GerritProjectLister.java @@ -6,12 +6,12 @@ import javax.annotation.Nonnull; -public abstract class ProjectLister { +public abstract class GerritProjectLister { @Nonnull private final GerritServer gerritServer; - public ProjectLister(@Nonnull GerritServer gerritServer) { + public GerritProjectLister(@Nonnull GerritServer gerritServer) { this.gerritServer = gerritServer; } diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/GerritSsh.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/GerritSsh.java deleted file mode 100644 index 4f9dfe0..0000000 --- a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/GerritSsh.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.holmsted.gerrit.downloaders.ssh; - -import com.holmsted.gerrit.GerritServer; -import com.holmsted.gerrit.GerritVersion; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import javax.annotation.Nonnull; - -public final class GerritSsh { - - public static List listProjects(@Nonnull GerritServer gerritServer) { - GerritSshCommand sshCommand = new GerritSshCommand(gerritServer); - String output = sshCommand.exec("ls-projects"); - - List projectList = new ArrayList<>(); - Collections.addAll(projectList, output.split("\n")); - return projectList; - } - - public static GerritVersion version(@Nonnull GerritServer gerritServer) { - GerritSshCommand sshCommand = new GerritSshCommand(gerritServer); - String output = sshCommand.exec("version"); - if (output != null) { - return GerritVersion.fromString(output.substring(output.lastIndexOf(' ') + 1)); - } else { - return null; - } - } - - private GerritSsh() { - } -} diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/GerritSshCommand.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshCommand.java similarity index 69% rename from GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/GerritSshCommand.java rename to GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshCommand.java index 554a616..95091d7 100644 --- a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/GerritSshCommand.java +++ b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshCommand.java @@ -1,18 +1,43 @@ package com.holmsted.gerrit.downloaders.ssh; import com.holmsted.gerrit.GerritServer; +import com.holmsted.gerrit.GerritVersion; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import javax.annotation.Nonnull; -public class GerritSshCommand { +public class SshCommand { @Nonnull private final GerritServer gerritServer; - public GerritSshCommand(@Nonnull GerritServer gerritServer) { + + public static List getProjectList(@Nonnull GerritServer gerritServer) { + SshCommand sshCommand = new SshCommand(gerritServer); + String output = sshCommand.exec("ls-projects"); + + List projectList = new ArrayList<>(); + Collections.addAll(projectList, output.split("\n")); + return projectList; + } + + public static GerritVersion getServerVersion(@Nonnull GerritServer gerritServer) { + SshCommand sshCommand = new SshCommand(gerritServer); + String output = sshCommand.exec("version"); + if (output != null) { + return GerritVersion.fromString(output.substring(output.lastIndexOf(' ') + 1)); + } else { + return null; + } + } + + + public SshCommand(@Nonnull GerritServer gerritServer) { this.gerritServer = gerritServer; } diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDefaultCommitDownloader.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDefaultCommitDownloader.java new file mode 100644 index 0000000..1adfbeb --- /dev/null +++ b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDefaultCommitDownloader.java @@ -0,0 +1,81 @@ +package com.holmsted.gerrit.downloaders.ssh; + +import com.google.common.base.Strings; +import com.holmsted.file.FileReader; +import com.holmsted.file.FileWriter; +import com.holmsted.gerrit.GerritServer; +import com.holmsted.gerrit.GerritVersion; +import com.holmsted.gerrit.downloaders.GerritCommitData; +import com.holmsted.gerrit.downloaders.GerritCommitDownloader; + +import javax.annotation.Nonnull; +import java.io.File; + + +/** + * Reads data from Gerrit versions 2.9 and higher. + */ +public class SshDefaultCommitDownloader extends GerritCommitDownloader { + private int startOffset; + private boolean hasMoreChanges = true; + private int rowCount = 0; + + + public SshDefaultCommitDownloader(@Nonnull GerritServer gerritServer, GerritVersion gerritVersion) { + super(gerritServer, gerritVersion); + } + + + public void setStartOffset(int startOffset) { + this.startOffset = startOffset; + } + + @Nonnull + @Override + public boolean hasMoreData() { + return hasMoreChanges + && (rowCount < getOverallCommitLimit() || getOverallCommitLimit() == NO_COMMIT_LIMIT); + } + + public GerritCommitData readData() { + String gerritQuery = getGerritQuery(); + String output = ""; + String file = "/home/ezivkoc/repo/gerritstats/data/ci-config/" + startOffset + ".txt"; + + SshCommand sshCommand = new SshCommand(getGerritServer()); + + output = sshCommand.exec(String.format("query %s " + + "--format=JSON " + + "--all-approvals " + + "--comments " + + "--all-reviewers " + + createStartOffsetArg(), + gerritQuery + )); + + FileWriter.writeFile(file, output); + + GerritCommitData commitData = new GerritCommitData(Strings.nullToEmpty(output), getGerritVersion()); + setStartOffset(startOffset + commitData.getMetaData().getRowCount()); + hasMoreChanges = commitData.getMetaData().hasMoreChanges(); + rowCount += commitData.getMetaData().getRowCount(); + return commitData; + } + + public String getGerritQuery() { + String query = String.format("project:{^%s}", getProjectName()); + + if (getAfterDate() != null) { + query += String.format(" after:{%s}", getAfterDate()); + } + if (getBeforeDate() != null) { + query += String.format(" before:{%s}", getBeforeDate()); + } + + return query; + } + + private String createStartOffsetArg() { + return startOffset != 0 ? "--start " + startOffset + " " : ""; + } +} \ No newline at end of file diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDownloader.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDownloader.java deleted file mode 100644 index 628ece4..0000000 --- a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDownloader.java +++ /dev/null @@ -1,308 +0,0 @@ -package com.holmsted.gerrit.downloaders.ssh; - -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.holmsted.gerrit.GerritServer; -import com.holmsted.gerrit.GerritVersion; -import com.holmsted.gerrit.downloaders.AbstractGerritStatsDownloader; -import com.holmsted.json.JsonUtils; - -import org.json.JSONObject; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import javax.annotation.Nonnull; - -public class SshDownloader extends AbstractGerritStatsDownloader { - - public static final int NO_COMMIT_LIMIT = -1; - - @Nonnull - private final GerritVersion gerritVersion; - - static class QueryMetadata { - final int rowCount; - final int runtimeMsec; - final boolean moreChanges; - final String resumeSortkey; - - static QueryMetadata fromOutputString(String output) { - JSONObject lastLineData = JsonUtils.readJsonString(output); - if (lastLineData.get("rowCount") != null) { - return new QueryMetadata(lastLineData); - } else { - return null; - } - } - - private QueryMetadata(JSONObject metadata) { - moreChanges = metadata.optBoolean("moreChanges"); - rowCount = metadata.optInt("rowCount"); - runtimeMsec = metadata.optInt("runTimeMilliseconds"); - resumeSortkey = metadata.optString("resumeSortKey"); - } - } - - static class GerritOutput { - private final List output = new ArrayList<>(); - private final QueryMetadata metadata; - - @Nonnull - private final GerritVersion gerritVersion; - - public GerritOutput(@Nonnull String output, @Nonnull GerritVersion gerritVersion) { - List strings = Arrays.asList(output.split("\n")); - - String lastLine = strings.get(strings.size() - 1); - this.metadata = Preconditions.checkNotNull(QueryMetadata.fromOutputString(lastLine)); - - for (int i = 0; i < strings.size() - 1; ++i) { - this.output.add(new JSONObject(strings.get(i))); - } - this.gerritVersion = gerritVersion; - } - - public boolean hasMoreChanges() { - if (gerritVersion.isAtLeast(2, 9)) { - return metadata.moreChanges; - } else { - return !Strings.nullToEmpty(metadata.resumeSortkey).isEmpty(); - } - } - - public String getResumeSortkey() { - return metadata.resumeSortkey; - } - - public int getRowCount() { - return metadata.rowCount; - } - - public List getOutput() { - return output; - } - } - - abstract static class DataReader { - private int overallCommitLimit; - protected String gerritQuery; - - private GerritServer gerritServer; - private GerritVersion gerritVersion; - - @Nonnull - public abstract List readUntilLimit(); - - public void setGerritServer(@Nonnull GerritServer gerritServer) { - this.gerritServer = gerritServer; - } - - public GerritServer getGerritServer() { - return gerritServer; - } - - public void setOverallCommitLimit(int overallCommitLimit) { - this.overallCommitLimit = overallCommitLimit; - } - - public int getOverallCommitLimit() { - return overallCommitLimit; - } - - public abstract void setGerritQuery(String projectNameList, String afterDate, String beforeDate); - - public String getGerritQuery() { - return gerritQuery; - } - - public void setGerritVersion(@Nonnull GerritVersion gerritVersion) { - this.gerritVersion = gerritVersion; - } - - public GerritVersion getGerritVersion() { - return gerritVersion; - } - } - - /** - * Data reader with Gerrit pre-2.9 support. The following problems are being worked around: - *

- * 1) if no status query is passed, only open commits are listed. So this reader manually - * queries for open, merged and abandoned commits. - * 2) the resume/limit behavior is not implemented in pre-2.9, so resume_sortKey is used instead. - */ - static class LegacyDataReader extends DataReader { - - private String resumeSortkey; - - private int rowCount; - - @Nonnull - @Override - public List readUntilLimit() { - rowCount = 0; - - List items = new ArrayList<>(); - - String[] statusQueries = {"status:merged", "status:open", "status:abandoned"}; - - for (String statusQuery : statusQueries) { - items.addAll(readOutputWithStatusQueryUntilLimit(statusQuery)); - } - - return items; - } - - @Override - public void setGerritQuery(String projectNameList, String afterDate, String beforeDate) { - if (projectNameList.isEmpty()) { - throw new IllegalStateException("No project name defined!"); - } - if (afterDate != null) { - System.out.println("--after-date parameter not supported with Gerrit prior to v2.9."); - } - this.gerritQuery = String.format("project:^%s", projectNameList); - } - - private List readOutputWithStatusQueryUntilLimit(@Nonnull String statusQuery) { - List items = new ArrayList<>(); - - boolean hasMoreChanges = true; - while (hasMoreChanges - && (rowCount < getOverallCommitLimit() || getOverallCommitLimit() == NO_COMMIT_LIMIT)) { - GerritOutput gerritOutput = readOutputWithStatusQuery(statusQuery); - items.addAll(gerritOutput.getOutput()); - - resumeSortkey = gerritOutput.getResumeSortkey(); - hasMoreChanges = gerritOutput.hasMoreChanges(); - rowCount += gerritOutput.getRowCount(); - } - - return items; - } - - private GerritOutput readOutputWithStatusQuery(String statusQuery) { - String gerritQuery = getGerritQuery(); - GerritSshCommand sshCommand = new GerritSshCommand(getGerritServer()); - String resumeSortkeyArg = !Strings.nullToEmpty(resumeSortkey).isEmpty() - ? "resume_sortkey:" + resumeSortkey : ""; - - String output = sshCommand.exec(String.format("query %s %s " - + "--format=JSON " - + "--all-approvals " - + "--comments " - + "%s ", - gerritQuery, - statusQuery, - resumeSortkeyArg - )); - - return new GerritOutput(Strings.nullToEmpty(output), getGerritVersion()); - } - } - - /** - * Reads data from Gerrit versions 2.9 and higher. - */ - static class DefaultDataReader extends DataReader { - private int startOffset; - - public void setStartOffset(int startOffset) { - this.startOffset = startOffset; - } - - public GerritOutput readData() { - String gerritQuery = getGerritQuery(); - GerritSshCommand sshCommand = new GerritSshCommand(getGerritServer()); - - String output = sshCommand.exec(String.format("query %s " - + "--format=JSON " - + "--all-approvals " - + "--comments " - + "--all-reviewers " - + createStartOffsetArg(), - gerritQuery - )); - - return new GerritOutput(Strings.nullToEmpty(output), getGerritVersion()); - } - - @Nonnull - @Override - public List readUntilLimit() { - List items = new ArrayList<>(); - boolean hasMoreChanges = true; - int rowCount = 0; - - while (hasMoreChanges - && (rowCount < getOverallCommitLimit() || getOverallCommitLimit() == NO_COMMIT_LIMIT)) { - GerritOutput gerritOutput = readData(); - items.addAll(gerritOutput.getOutput()); - - hasMoreChanges = gerritOutput.hasMoreChanges(); - rowCount += gerritOutput.getRowCount(); - setStartOffset(startOffset + gerritOutput.getRowCount()); - } - - return items; - } - - @Override - public void setGerritQuery(String projectNameList, String afterDate, String beforeDate) { - if (projectNameList.isEmpty()) { - throw new IllegalStateException("No project name defined!"); - } - this.gerritQuery = String.format("project:{^%s}", projectNameList); - if (afterDate != null) { - this.gerritQuery += String.format(" after:{%s}", afterDate); - } - if (beforeDate != null) { - this.gerritQuery += String.format(" before:{%s}", beforeDate); - } - } - - - private String createStartOffsetArg() { - return startOffset != 0 ? "--start " + startOffset + " " : ""; - } - } - - public SshDownloader(@Nonnull GerritServer gerritServer, @Nonnull GerritVersion gerritVersion) { - super(gerritServer); - this.gerritVersion = gerritVersion; - } - - /** - * Reads the data in json format from gerrit. - */ - @Nonnull - public List readData() { - if (getOverallCommitLimit() != NO_COMMIT_LIMIT) { - System.out.println(String.format("Reading data from %s for last %d commits", - getGerritServer(), getOverallCommitLimit())); - } else { - System.out.println("Reading all commit data from " + getGerritServer()); - } - - DataReader reader = createDataReader(); - return reader.readUntilLimit(); - } - - @Nonnull - private DataReader createDataReader() { - DataReader reader; - if (gerritVersion.isAtLeast(2, 9)) { - reader = new DefaultDataReader(); - } else { - reader = new LegacyDataReader(); - } - reader.setGerritServer(getGerritServer()); - reader.setOverallCommitLimit(getOverallCommitLimit()); - reader.setGerritQuery(getProjectName(), getAfterDate(), getBeforeDate()); - reader.setGerritVersion(gerritVersion); - - return reader; - } -} diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDownloaderFactory.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDownloaderFactory.java new file mode 100644 index 0000000..a28c3c7 --- /dev/null +++ b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshDownloaderFactory.java @@ -0,0 +1,28 @@ +package com.holmsted.gerrit.downloaders.ssh; + + +import com.holmsted.gerrit.GerritServer; +import com.holmsted.gerrit.GerritVersion; +import com.holmsted.gerrit.downloaders.GerritCommitDownloader; +import com.holmsted.gerrit.downloaders.GerritProjectLister; + +import javax.annotation.Nonnull; + +public class SshDownloaderFactory { + + public static GerritProjectLister createProjectLister(@Nonnull GerritServer gerritServer, @Nonnull GerritVersion gerritVersion) { + return new SshProjectLister(gerritServer); + } + + public static GerritCommitDownloader createCommitDownloader(@Nonnull GerritServer gerritServer, @Nonnull GerritVersion gerritVersion) { + GerritCommitDownloader reader; + + if (gerritVersion.isAtLeast(2, 9)) { + reader = new SshDefaultCommitDownloader(gerritServer, gerritVersion); + } else { + reader = new SshLegacyCommitDownloader(gerritServer, gerritVersion); + } + + return reader; + } +} diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshLegacyCommitDownloader.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshLegacyCommitDownloader.java new file mode 100644 index 0000000..bfdd4f4 --- /dev/null +++ b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshLegacyCommitDownloader.java @@ -0,0 +1,82 @@ +package com.holmsted.gerrit.downloaders.ssh; + +import com.google.common.base.Strings; +import com.holmsted.gerrit.GerritServer; +import com.holmsted.gerrit.GerritVersion; +import com.holmsted.gerrit.downloaders.GerritCommitData; +import com.holmsted.gerrit.downloaders.GerritCommitDownloader; + +import javax.annotation.Nonnull; + +/** + * Data reader with Gerrit pre-2.9 support. The following problems are being worked around: + *

+ * 1) if no status query is passed, only open commits are listed. So this reader manually + * queries for open, merged and abandoned commits. + * 2) the resume/limit behavior is not implemented in pre-2.9, so resume_sortKey is used instead. + */ +public class SshLegacyCommitDownloader extends GerritCommitDownloader { + private static final String[] STATUS_QUERIES = {"status:open", "status:merged", "status:abandoned"}; + + private int statusQueryIndex = 0; + private boolean hasMoreChanges = true; + private String resumeSortKey; + private int rowCount; + + + public SshLegacyCommitDownloader(@Nonnull GerritServer gerritServer, GerritVersion gerritVersion) { + super(gerritServer, gerritVersion); + } + + + @Nonnull + @Override + public boolean hasMoreData() { + if (!hasMoreChanges && statusQueryIndex < STATUS_QUERIES.length) { + statusQueryIndex++; + hasMoreChanges = true; + } + + return hasMoreChanges + && (rowCount < getOverallCommitLimit() || getOverallCommitLimit() == NO_COMMIT_LIMIT); + } + + @Nonnull + @Override + public GerritCommitData readData() { + return readStatusData(STATUS_QUERIES[statusQueryIndex]); + } + + private GerritCommitData readStatusData(String statusQuery) { + String gerritQuery = getGerritQuery(); + SshCommand sshCommand = new SshCommand(getGerritServer()); + String resumeSortkeyArg = !Strings.nullToEmpty(resumeSortKey).isEmpty() + ? "resume_sortkey:" + resumeSortKey : ""; + + String output = sshCommand.exec(String.format("query %s %s " + + "--format=JSON " + + "--all-approvals " + + "--comments " + + "%s ", + gerritQuery, + statusQuery, + resumeSortkeyArg + )); + + GerritCommitData commitData = new GerritCommitData(Strings.nullToEmpty(output), getGerritVersion()); + resumeSortKey = commitData.getMetaData().getResumeSortKey(); + hasMoreChanges = commitData.getMetaData().hasMoreChanges(); + rowCount += commitData.getMetaData().getRowCount(); + return commitData; + } + + public String getGerritQuery() { + String query = String.format("project:^%s", getProjectName()); + + if (getAfterDate() != null) { + System.out.println("WARNING: --after-date parameter not supported with Gerrit prior to v2.9."); + } + + return query; + } +} \ No newline at end of file diff --git a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshProjectLister.java b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshProjectLister.java index fb3d78f..d1bc02a 100644 --- a/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshProjectLister.java +++ b/GerritDownloader/src/main/java/com/holmsted/gerrit/downloaders/ssh/SshProjectLister.java @@ -1,7 +1,7 @@ package com.holmsted.gerrit.downloaders.ssh; import com.holmsted.gerrit.GerritServer; -import com.holmsted.gerrit.downloaders.ProjectLister; +import com.holmsted.gerrit.downloaders.GerritProjectLister; import java.util.List; @@ -10,7 +10,7 @@ /** * Creates a listing of all Gerrit projects on the given server. */ -public class SshProjectLister extends ProjectLister { +public class SshProjectLister extends GerritProjectLister { public SshProjectLister(@Nonnull GerritServer gerritServer) { super(gerritServer); @@ -18,6 +18,6 @@ public SshProjectLister(@Nonnull GerritServer gerritServer) { @Nonnull public List getProjectListing() { - return GerritSsh.listProjects(getGerritServer()); + return SshCommand.getProjectList(getGerritServer()); } }