Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore: separate dataset index creation #4710

Merged
merged 2 commits into from
Jul 4, 2024
Merged

chore: separate dataset index creation #4710

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dd4a3cb
chore: separate dataset index creation
Sidddddarth May 25, 2024
f7844a9
fixup! chore: separate dataset index creation
Sidddddarth Jul 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 51 additions & 34 deletions jobsdb/jobsdb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1272,15 +1272,6 @@ func (jd *Handle) addNewDSInTx(tx *Tx, l lock.LockToken, dsList []dataSetT, ds d
return nil
}

func (jd *Handle) addDSInTx(tx *Tx, ds dataSetT) error {
defer jd.getTimerStat(
"add_new_ds",
&statTags{CustomValFilters: []string{jd.tablePrefix}},
).RecordDuration()()
jd.logger.Infof("Creating DS %+v", ds)
return jd.createDSInTx(tx, ds)
}

func (jd *Handle) computeNewIdxForAppend(l lock.LockToken) string {
dList, err := jd.doRefreshDSList(l)
jd.assertError(err)
Expand Down Expand Up @@ -1325,7 +1316,22 @@ func (jd *Handle) createDSInTx(tx *Tx, newDS dataSetT) error {
}

// Create the jobs and job_status tables
if _, err = tx.ExecContext(ctx, fmt.Sprintf(`CREATE TABLE %q (
if err = jd.createDSTablesInTx(ctx, tx, newDS); err != nil {
return fmt.Errorf("creating DS tables %w", err)
}
if err = jd.createDSIndicesInTx(ctx, tx, newDS); err != nil {
return fmt.Errorf("creating DS indices %w", err)
}

err = jd.journalMarkDoneInTx(tx, opID)
if err != nil {
return err
}
return nil
}

func (jd *Handle) createDSTablesInTx(ctx context.Context, tx *Tx, newDS dataSetT) error {
if _, err := tx.ExecContext(ctx, fmt.Sprintf(`CREATE TABLE %q (
job_id BIGSERIAL PRIMARY KEY,
workspace_id TEXT NOT NULL DEFAULT '',
uuid UUID NOT NULL,
Expand All @@ -1336,42 +1342,53 @@ func (jd *Handle) createDSInTx(tx *Tx, newDS dataSetT) error {
event_count INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
expire_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW());`, newDS.JobTable)); err != nil {
return err
}
if _, err = tx.ExecContext(ctx, fmt.Sprintf(`CREATE INDEX "idx_%[1]s_ws" ON %[1]q (workspace_id)`, newDS.JobTable)); err != nil {
return err
}
if _, err = tx.ExecContext(ctx, fmt.Sprintf(`CREATE INDEX "idx_%[1]s_cv" ON %[1]q (custom_val)`, newDS.JobTable)); err != nil {
return err
return fmt.Errorf("creating %s: %w", newDS.JobTable, err)
}
for _, param := range cacheParameterFilters {
if _, err = tx.ExecContext(ctx, fmt.Sprintf(`CREATE INDEX "idx_%[1]s_%[2]s" ON %[1]q USING BTREE ((parameters->>'%[2]s'))`, newDS.JobTable, param)); err != nil {
return err
}
}

if _, err = tx.ExecContext(ctx, fmt.Sprintf(`CREATE TABLE %q (
if _, err := tx.ExecContext(ctx, fmt.Sprintf(`CREATE TABLE %q (
id BIGSERIAL,
job_id BIGINT REFERENCES %q(job_id),
job_id BIGINT,
job_state VARCHAR(64),
attempt SMALLINT,
exec_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
retry_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
error_code VARCHAR(32),
error_response JSONB DEFAULT '{}'::JSONB,
parameters JSONB DEFAULT '{}'::JSONB,
PRIMARY KEY (job_id, job_state, id));`, newDS.JobStatusTable, newDS.JobTable)); err != nil {
return err
PRIMARY KEY (job_id, job_state, id));`, newDS.JobStatusTable)); err != nil {
return fmt.Errorf("creating %s: %w", newDS.JobStatusTable, err)
}
if _, err = tx.ExecContext(ctx, fmt.Sprintf(`CREATE INDEX "idx_%[1]s_jid_id" ON %[1]q(job_id asc,id desc)`, newDS.JobStatusTable)); err != nil {
return err
return nil
}

func (jd *Handle) createDSIndicesInTx(ctx context.Context, tx *Tx, newDS dataSetT) error {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we add more context around the error branches? Same for createDSTablesInTx as well.

if _, err := tx.ExecContext(ctx, fmt.Sprintf(`CREATE INDEX "idx_%[1]s_ws" ON %[1]q (workspace_id)`, newDS.JobTable)); err != nil {
return fmt.Errorf("creating workspace index: %w", err)
}
if _, err = tx.ExecContext(ctx, fmt.Sprintf(`CREATE VIEW "v_last_%[1]s" AS SELECT DISTINCT ON (job_id) * FROM %[1]q ORDER BY job_id ASC, id DESC`, newDS.JobStatusTable)); err != nil {
return err
if _, err := tx.ExecContext(ctx, fmt.Sprintf(`CREATE INDEX "idx_%[1]s_cv" ON %[1]q (custom_val)`, newDS.JobTable)); err != nil {
return fmt.Errorf("creating custom_val index: %w", err)
}
err = jd.journalMarkDoneInTx(tx, opID)
if err != nil {
return err
for _, param := range cacheParameterFilters {
if _, err := tx.ExecContext(ctx, fmt.Sprintf(`CREATE INDEX "idx_%[1]s_%[2]s" ON %[1]q USING BTREE ((parameters->>'%[2]s'))`, newDS.JobTable, param)); err != nil {
return fmt.Errorf("creating %s index: %w", param, err)
}
}
if _, err := tx.ExecContext(
ctx,
fmt.Sprintf(
`ALTER TABLE %[1]q
ADD CONSTRAINT "fk_%[1]s_job_id"
FOREIGN KEY (job_id)
REFERENCES %[2]q (job_id)`,
newDS.JobStatusTable,
newDS.JobTable,
)); err != nil {
return fmt.Errorf("adding foreign key constraint: %w", err)
}
if _, err := tx.ExecContext(ctx, fmt.Sprintf(`CREATE INDEX "idx_%[1]s_jid_id" ON %[1]q(job_id asc,id desc)`, newDS.JobStatusTable)); err != nil {
return fmt.Errorf("adding job_id_id index: %w", err)
}
if _, err := tx.ExecContext(ctx, fmt.Sprintf(`CREATE VIEW "v_last_%[1]s" AS SELECT DISTINCT ON (job_id) * FROM %[1]q ORDER BY job_id ASC, id DESC`, newDS.JobStatusTable)); err != nil {
return fmt.Errorf("create view: %w", err)
}
return nil
}
Expand Down
2 changes: 1 addition & 1 deletion jobsdb/jobsdb_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ func TestRefreshDSList(t *testing.T) {

require.Equal(t, 1, len(jobsDB.getDSList()), "jobsDB should start with a ds list size of 1")
require.NoError(t, jobsDB.WithTx(func(tx *Tx) error {
return jobsDB.addDSInTx(tx, newDataSet(prefix, "2"))
return jobsDB.createDSInTx(tx, newDataSet(prefix, "2"))
}))
require.Equal(t, 1, len(jobsDB.getDSList()), "addDS should not refresh the ds list")
jobsDB.dsListLock.WithLock(func(l lock.LockToken) {
Expand Down
8 changes: 5 additions & 3 deletions jobsdb/migration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,8 @@ func (jd *Handle) doMigrateDS(ctx context.Context) error {
return fmt.Errorf("failed to mark journal start: %w", err)
}

err = jd.addDSInTx(tx, destination)
if err != nil {
return fmt.Errorf("failed to add dataset: %w", err)
if err = jd.createDSTablesInTx(ctx, tx, destination); err != nil {
return fmt.Errorf("failed to create dataset tables: %w", err)
}

totalJobsMigrated := 0
Expand All @@ -139,6 +138,9 @@ func (jd *Handle) doMigrateDS(ctx context.Context) error {
}
totalJobsMigrated += noJobsMigrated
}
if err = jd.createDSIndicesInTx(ctx, tx, destination); err != nil {
return fmt.Errorf("create %v indices: %w", destination, err)
}
if err = jd.journalMarkDoneInTx(tx, opID); err != nil {
return fmt.Errorf("failed to mark journal done: %w", err)
}
Expand Down