Skip to content

Commit

Permalink
Add functionality in benchmarking code to optionally provide a hand-o…
Browse files Browse the repository at this point in the history 
…ptimized DDlog program file. In that case, /tmp/program.dl is not compiled using sql-ddlog compiler. Instead, we just copy over contents of the hand-optimized file into /tmp/program.dl.

Signed-off-by: Amy Tai <[email protected]>
  • Loading branch information
@amytai
amytai committed Nov 11, 2021
1 parent cf36222 commit 03b6379
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 20 deletions.
46 changes: 30 additions & 16 deletions k8s-scheduler/src/main/java/com/vmware/dcm/DDlogDBConnectionPool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.vmware.ddlog.util.sql.H2SqlStatement;
import ddlogapi.DDlogAPI;
import ddlogapi.DDlogException;
import org.apache.commons.io.FileUtils;
import org.jooq.DSLContext;
import org.jooq.SQLDialect;
import org.jooq.conf.ParamCastMode;
Expand All @@ -17,10 +18,9 @@
import org.jooq.impl.DSL;
import org.jooq.tools.jdbc.MockConnection;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import javax.annotation.Nullable;
import java.io.*;
import java.nio.Buffer;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
Expand All @@ -33,11 +33,18 @@ public class DDlogDBConnectionPool implements IConnectionPool {
private final List<String> scopedViews;

private DDlogJooqProvider provider;
private final String ddlogFile;

public DDlogDBConnectionPool() {
this.ddlogFile = null;
scopedViews = new ArrayList<>();
}

public DDlogDBConnectionPool(String ddlogFile) {
this.ddlogFile = ddlogFile;
this.scopedViews = new ArrayList<>();
}

@Override
public DSLContext getConnectionToDb() {
final MockConnection connection = new MockConnection(provider);
Expand All @@ -61,19 +68,26 @@ public void addScopedViews(List<String> statements) {
scopedViews.addAll(statements);
}

private static void compileAndLoad(final List<CalciteSqlStatement> ddl, final List<String> createIndexStatements)
private static void compileAndLoad(final List<CalciteSqlStatement> ddl, final List<String> createIndexStatements,
final String ddlogFile)
throws IOException, DDlogException {
final Translator t = new Translator(null);
CalciteToPrestoTranslator ctopTranslator = new CalciteToPrestoTranslator();
ddl.forEach(x -> t.translateSqlStatement(ctopTranslator.toPresto(x)));
createIndexStatements.forEach(t::translateCreateIndexStatement);

final DDlogProgram dDlogProgram = t.getDDlogProgram();
final String fileName = "/tmp/program.dl";
File tmp = new File(fileName);
BufferedWriter bw = new BufferedWriter(new FileWriter(tmp));
bw.write(dDlogProgram.toString());
bw.close();
if (ddlogFile != null) {
// don't compile, instead use this ddlogFile
// Copy ddlogFile contents to fileName
FileUtils.copyFile(new File(ddlogFile), new File(fileName));
} else {
final Translator t = new Translator(null);
CalciteToPrestoTranslator ctopTranslator = new CalciteToPrestoTranslator();
ddl.forEach(x -> t.translateSqlStatement(ctopTranslator.toPresto(x)));
createIndexStatements.forEach(t::translateCreateIndexStatement);

final DDlogProgram dDlogProgram = t.getDDlogProgram();
File tmp = new File(fileName);
BufferedWriter bw = new BufferedWriter(new FileWriter(tmp));
bw.write(dDlogProgram.toString());
bw.close();
}
DDlogAPI.CompilationResult result = new DDlogAPI.CompilationResult(true);
final String ddlogHome = System.getenv("DDLOG_HOME");
assertNotNull(ddlogHome);
Expand Down Expand Up @@ -104,7 +118,7 @@ private void setupDDlog(boolean seal) {

DDlogAPI ddlogAPI = null;
if (seal) {
compileAndLoad(tablesInCalcite, createIndexStatements);
compileAndLoad(tablesInCalcite, createIndexStatements, ddlogFile);

ddlogAPI = new DDlogAPI(1, false);
}
Expand Down
10 changes: 7 additions & 3 deletions k8s-scheduler/src/main/java/com/vmware/dcm/EmulatedCluster.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ class EmulatedCluster {

public void runTraceLocally(final int numNodes, final String traceFileName, final int cpuScaleDown,
final int memScaleDown, final int timeScaleDown, final int startTimeCutOff,
final int affinityRequirementsProportion, final boolean scopeOn)
final int affinityRequirementsProportion, final boolean scopeOn, final String ddlogFile)
throws Exception {
final IConnectionPool dbConnectionPool = new DDlogDBConnectionPool(); // new DBConnectionPool();
final IConnectionPool dbConnectionPool = new DDlogDBConnectionPool(ddlogFile); // new DBConnectionPool();
if (dbConnectionPool instanceof DDlogDBConnectionPool) {
((DDlogDBConnectionPool) dbConnectionPool).buildDDlog(false);
}
Expand Down Expand Up @@ -184,6 +184,8 @@ public static void runWorkload(final String[] args) throws Exception {
"P, from 0 to 100, indicating the proportion of pods that have affinity requirements");
options.addOption("S", "scopeOn", false,
"enable auto-scope in scheduler");
options.addOption("df", "ddlogFile", true,
"specify which ddlog program.dl to use");
final CommandLineParser parser = new DefaultParser();
final CommandLine cmd = parser.parse(options, args);
final int numNodes = Integer.parseInt(cmd.getOptionValue("numNodes"));
Expand All @@ -195,13 +197,15 @@ public static void runWorkload(final String[] args) throws Exception {
final int affinityRequirementsProportion = Integer.parseInt(cmd.hasOption("proportion") ?
cmd.getOptionValue("proportion") : "0");
final boolean scopeOn = cmd.hasOption("scopeOn");
final String ddlogFile = cmd.getOptionValue("ddlogFile");

assert affinityRequirementsProportion >= 0 && affinityRequirementsProportion <= 100;
LOG.info("Running experiment with parameters: numNodes: {}, traceFile: {}, cpuScaleDown: {}, " +
"memScaleDown: {}, timeScaleDown: {}, startTimeCutOff: {}, proportion: {}, scopeOn: {}",
numNodes, traceFile, cpuScaleDown, memScaleDown,
timeScaleDown, startTimeCutOff, affinityRequirementsProportion, scopeOn);
emulatedCluster.runTraceLocally(numNodes, traceFile, cpuScaleDown, memScaleDown, timeScaleDown,
startTimeCutOff, affinityRequirementsProportion, scopeOn);
startTimeCutOff, affinityRequirementsProportion, scopeOn, ddlogFile);
}

public static void main(final String[] args) throws Exception {
Expand Down
3 changes: 2 additions & 1 deletion k8s-scheduler/src/test/java/com/vmware/dcm/WorkloadReplayTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ public class WorkloadReplayTest {
@Test
public void runTest() throws Exception {
final String[] args =
{"-n", "500", "-f", "test-data.txt", "-c", "100", "-m", "200", "-t", "100", "-s", "1000"};
{"-n", "500", "-f", "test-data.txt", "-c", "100", "-m", "200", "-t", "100", "-s", "1000", "-df",
"../artifacts/optimized_ddlog_programs/preempt_program.dl.dl"};
EmulatedCluster.runWorkload(args);
}

Expand Down

0 comments on commit 03b6379

Please sign in to comment.