Merge commit '9c65c22e28c7900ff8e4959f4d9f9d5358bd1c8f' into BED-4725…

…_zip-bom-decode

cmd/api/src/database/migration/migrations/v5.16.0.sql → cmd/api/src/database/migration/migrations/v6.0.0.sql

@@ -26,6 +26,31 @@ INSERT INTO parameters (key, name, description, value, created_at, updated_at) V
 -- must occur after insert to ensure reconciliation flag is set to whatever current value is
 DELETE FROM feature_flags WHERE key = 'reconciliation';
 
-
 -- Grant the Read-Only user SavedQueriesRead permissions
 INSERT INTO roles_permissions (role_id, permission_id) VALUES ((SELECT id FROM roles WHERE roles.name  = 'Read-Only'), (SELECT id FROM permissions WHERE permissions.authority  = 'saved_queries'  and permissions.name = 'Read')) ON CONFLICT DO NOTHING;
+
+-- Add OIDC Support feature flag
+INSERT INTO feature_flags (created_at, updated_at, key, name, description, enabled, user_updatable)
+VALUES (
+    current_timestamp,
+    current_timestamp,
+    'oidc_support',
+    'OIDC Support',
+    'Enables OpenID Connect authentication support for SSO Authentication.',
+    false,
+    false
+)
+ON CONFLICT DO NOTHING;
+
+do
+$$
+  begin
+    -- Update existing Edge tables with an additional constraint to support ON CONFLICT upserts
+    alter table edge drop constraint if exists edge_graph_id_start_id_end_id_kind_id_key;
+    alter table edge add constraint edge_graph_id_start_id_end_id_kind_id_key unique (graph_id, start_id, end_id, kind_id);
+  exception
+    -- This guards against the possibility that the edge table doesn't exist, in which case there's no constraint to
+    -- migrate
+    when undefined_table then null;
+  end
+$$;

cmd/api/src/model/appcfg/flag.go

@@ -36,6 +36,7 @@ const (
 	FeatureFedRAMPEULA                = "fedramp_eula"
 	FeatureDarkMode                   = "dark_mode"
 	FeatureAutoTagT0ParentObjects     = "auto_tag_t0_parent_objects"
+	FeatureOIDCSupport                = "oidc_support"
 )
 
 // FeatureFlag defines the most basic details of what a feature flag must contain to be actionable. Feature flags should be

packages/go/cypher/models/pgsql/format/format_test.go

@@ -79,7 +79,9 @@ func TestFormat_Update(t *testing.T) {
 
 func TestFormat_Insert(t *testing.T) {
 	formattedQuery, err := format.Statement(pgsql.Insert{
-		Table: pgsql.CompoundIdentifier{"table"},
+		Table: pgsql.TableReference{
+			Name: pgsql.CompoundIdentifier{"table"},
+		},
 		Shape: pgsql.RecordShape{
 			Columns: []pgsql.Identifier{"col1", "col2", "col3"},
 		},
@@ -94,7 +96,9 @@ func TestFormat_Insert(t *testing.T) {
 	require.Equal(t, "insert into table (col1, col2, col3) values ('1', 1, false);", formattedQuery)
 
 	formattedQuery, err = format.Statement(pgsql.Insert{
-		Table: pgsql.CompoundIdentifier{"table"},
+		Table: pgsql.TableReference{
+			Name: pgsql.CompoundIdentifier{"table"},
+		},
 		Shape: pgsql.RecordShape{
 			Columns: []pgsql.Identifier{"col1", "col2", "col3"},
 		},
@@ -121,7 +125,9 @@ func TestFormat_Insert(t *testing.T) {
 	require.Equal(t, "insert into table (col1, col2, col3) select * from other where other.col1 = '1234';", formattedQuery)
 
 	formattedQuery, err = format.Statement(pgsql.Insert{
-		Table: pgsql.CompoundIdentifier{"table"},
+		Table: pgsql.TableReference{
+			Name: pgsql.CompoundIdentifier{"table"},
+		},
 		Shape: pgsql.RecordShape{
 			Columns: []pgsql.Identifier{"col1", "col2", "col3"},
 		},
@@ -151,7 +157,9 @@ func TestFormat_Insert(t *testing.T) {
 	require.Equal(t, "insert into table (col1, col2, col3) select * from other where other.col1 = '1234' returning id;", formattedQuery)
 
 	formattedQuery, err = format.Statement(pgsql.Insert{
-		Table: pgsql.CompoundIdentifier{"table"},
+		Table: pgsql.TableReference{
+			Name: pgsql.CompoundIdentifier{"table"},
+		},
 		Shape: pgsql.RecordShape{
 			Columns: []pgsql.Identifier{"col1", "col2", "col3"},
 		},
@@ -200,7 +208,9 @@ func TestFormat_Insert(t *testing.T) {
 	require.Equal(t, "insert into table (col1, col2, col3) select * from other where other.col1 = '1234' on conflict on constraint other.hash_constraint do update set hit_count = hit_count + 1 where hit_count < 9999 returning id, hit_count;", formattedQuery)
 
 	formattedQuery, err = format.Statement(pgsql.Insert{
-		Table: pgsql.CompoundIdentifier{"table"},
+		Table: pgsql.TableReference{
+			Name: pgsql.CompoundIdentifier{"table"},
+		},
 		Shape: pgsql.RecordShape{
 			Columns: []pgsql.Identifier{"col1", "col2", "col3"},
 		},
@@ -223,7 +233,7 @@ func TestFormat_Insert(t *testing.T) {
 		},
 		OnConflict: &pgsql.OnConflict{
 			Target: &pgsql.ConflictTarget{
-				Columns: pgsql.CompoundIdentifier{"hash"},
+				Columns: []pgsql.Expression{pgsql.CompoundIdentifier{"hash"}},
 			},
 			Action: pgsql.DoUpdate{
 				Assignments: []pgsql.Assignment{pgsql.BinaryExpression{

packages/go/cypher/models/pgsql/functions.go

@@ -35,6 +35,7 @@ const (
 	FunctionNow                    Identifier = "now"
 	FunctionToLower                Identifier = "lower"
 	FunctionCoalesce               Identifier = "coalesce"
+	FunctionUnnest                 Identifier = "unnest"
 	FunctionJSONBSet               Identifier = "jsonb_set"
 	FunctionCount                  Identifier = "count"
 	FunctionEdgesToPath            Identifier = "edges_to_path"

packages/go/cypher/models/pgsql/model.go

@@ -269,6 +269,10 @@ func NewBinaryExpression(left Expression, operator Operator, right Expression) *
 	}
 }
 
+func NewJSONTextFieldLookup(reference CompoundIdentifier, field Identifier) *BinaryExpression {
+	return NewBinaryExpression(reference, OperatorJSONTextField, field)
+}
+
 func (s BinaryExpression) AsExpression() Expression {
 	return s
 }
@@ -421,6 +425,10 @@ type Parameter struct {
 	CastType   DataType
 }
 
+func (s Parameter) AsSelectItem() SelectItem {
+	return s
+}
+
 func (s Parameter) NodeType() string {
 	return "parameter"
 }
@@ -493,6 +501,16 @@ func (s Join) NodeType() string {
 
 type Identifier string
 
+func AsIdentifiers(strs ...string) []Identifier {
+	identifiers := make([]Identifier, len(strs))
+
+	for idx, str := range strs {
+		identifiers[idx] = Identifier(str)
+	}
+
+	return identifiers
+}
+
 func (s Identifier) AsSelectItem() SelectItem {
 	return s
 }
@@ -615,6 +633,22 @@ func (s CompoundIdentifier) NodeType() string {
 	return "compound_identifier"
 }
 
+func AsCompoundIdentifier[T string | Identifier](parts ...T) CompoundIdentifier {
+	compoundIdentifier := make(CompoundIdentifier, len(parts))
+
+	for idx, part := range parts {
+		switch typedPart := any(part).(type) {
+		case string:
+			compoundIdentifier[idx] = Identifier(typedPart)
+
+		case Identifier:
+			compoundIdentifier[idx] = typedPart
+		}
+	}
+
+	return compoundIdentifier
+}
+
 type TableReference struct {
 	Name    CompoundIdentifier
 	Binding models.Optional[Identifier]
@@ -805,7 +839,7 @@ func (s Merge) AsStatement() Statement {
 }
 
 type ConflictTarget struct {
-	Columns    []Identifier
+	Columns    []Expression
 	Constraint CompoundIdentifier
 }
 
@@ -853,7 +887,7 @@ func (s OnConflict) AsExpression() Expression {
 
 // Insert is a SQL statement that is evaluated to insert data into a target table.
 type Insert struct {
-	Table      CompoundIdentifier
+	Table      TableReference
 	Shape      RecordShape
 	OnConflict *OnConflict
 	Source     *Query

packages/go/cypher/models/pgsql/pgtypes.go

@@ -92,6 +92,7 @@ const (
 	Text                     DataType = "text"
 	TextArray                DataType = "text[]"
 	JSONB                    DataType = "jsonb"
+	JSONBArray               DataType = "jsonb[]"
 	Date                     DataType = "date"
 	TimeWithTimeZone         DataType = "time with time zone"
 	TimeWithoutTimeZone      DataType = "time without time zone"

packages/go/cypher/models/pgsql/translate/expansion.go

@@ -380,15 +380,19 @@ func (s *Translator) buildAllShortestPathsExpansionRoot(part *PatternPart, trave
 
 	var (
 		primerInsert = pgsql.Insert{
-			Table: pgsql.CompoundIdentifier{"next_pathspace"},
+			Table: pgsql.TableReference{
+				Name: pgsql.CompoundIdentifier{"next_pathspace"},
+			},
 			Shape: expansionColumns(),
 			Source: &pgsql.Query{
 				Body: expansion.PrimerStatement,
 			},
 		}
 
 		recursiveInsert = pgsql.Insert{
-			Table: pgsql.CompoundIdentifier{"next_pathspace"},
+			Table: pgsql.TableReference{
+				Name: pgsql.CompoundIdentifier{"next_pathspace"},
+			},
 			Shape: expansionColumns(),
 			Source: &pgsql.Query{
 				Body: expansion.RecursiveStatement,

packages/go/dawgs/drivers/pg/batch.go

@@ -380,7 +380,7 @@ func (s *batch) flushRelationshipUpdateByBuffer(updates *sql.RelationshipUpdateB
 	if graphTarget, err := s.innerTransaction.getTargetGraph(); err != nil {
 		return err
 	} else {
-		query := sql.FormatRelationshipPartitionUpsert(graphTarget)
+		query := sql.FormatRelationshipPartitionUpsert(graphTarget, updates.IdentityProperties)
 
 		if _, err := s.innerTransaction.tx.Exec(s.ctx, query, parameters.Format(graphTarget)...); err != nil {
 			return err

packages/go/dawgs/drivers/pg/query/format.go

@@ -129,13 +129,13 @@ func FormatNodeUpsert(graphTarget model.Graph, identityProperties []string) stri
 	)
 }
 
-func FormatRelationshipPartitionUpsert(graphTarget model.Graph) string {
-	return join(
-		"merge into ", graphTarget.Partitions.Edge.Name, " as e ",
-		"using (select $1::int4 as gid, unnest($2::int4[]) as sid, unnest($3::int4[]) as eid, unnest($4::int2[]) as kid, unnest($5::jsonb[]) as p) as ei ",
-		"on e.start_id = ei.sid and e.end_id = ei.eid and e.kind_id = ei.kid ",
-		"when matched then update set properties = e.properties || ei.p ",
-		"when not matched then insert (graph_id, start_id, end_id, kind_id, properties) values (ei.gid, ei.sid, ei.eid, ei.kid, ei.p);",
+func FormatRelationshipPartitionUpsert(graphTarget model.Graph, identityProperties []string) string {
+	return join("insert into ", graphTarget.Partitions.Edge.Name, " as e ",
+		"(graph_id, start_id, end_id, kind_id, properties) ",
+		"select $1::int4, unnest($2::int4[]), unnest($3::int4[]), unnest($4::int2[]), unnest($5::jsonb[]) ",
+		formatConflictMatcher(identityProperties, "graph_id, start_id, end_id, kind_id"),
+		"do update set properties = e.properties || excluded.properties ",
+		"returning id;",
 	)
 }
 

packages/go/dawgs/drivers/pg/query/sql/schema_up.sql

@@ -155,7 +155,9 @@ create table if not exists edge
   properties jsonb    not null,
 
   primary key (id, graph_id),
-  foreign key (graph_id) references graph (id) on delete cascade
+  foreign key (graph_id) references graph (id) on delete cascade,
+
+  unique (graph_id, start_id, end_id, kind_id)
 ) partition by list (graph_id);
 
 -- delete_node_edges is a trigger and associated plpgsql function to cascade delete edges when attached nodes are

packages/go/log/log.go

@@ -243,8 +243,7 @@ func LogAndMeasure(level Level, format string, args ...any) func() {
 		then    = time.Now()
 	)
 
-	// Only output the message header on debug
-	WithLevel(LevelDebug).Uint64(FieldMeasurementID, pairID).Msg(message)
+	WithLevel(level).Uint64(FieldMeasurementID, pairID).Msg(message)
 
 	return func() {
 		if elapsed := time.Since(then); elapsed >= measureThreshold {