Derive the session.id/order_number from the upload.id

database/models/reports.py

@@ -74,8 +74,8 @@ class CommitReport(CodecovBaseModel, MixinBaseClass):
 uploadflagmembership = Table(
     "reports_uploadflagmembership",
     CodecovBaseModel.metadata,
-    Column("upload_id", types.Integer, ForeignKey("reports_upload.id")),
-    Column("flag_id", types.Integer, ForeignKey("reports_repositoryflag.id")),
+    Column("upload_id", types.BigInteger, ForeignKey("reports_upload.id")),
+    Column("flag_id", types.BigInteger, ForeignKey("reports_repositoryflag.id")),
 )
 
 
@@ -84,7 +84,7 @@ class ReportResults(MixinBaseClass, CodecovBaseModel):
     state = Column(types.Text)
     completed_at = Column(types.DateTime(timezone=True), nullable=True)
     result = Column(postgresql.JSON)
-    report_id = Column(types.Integer, ForeignKey("reports_commitreport.id"))
+    report_id = Column(types.BigInteger, ForeignKey("reports_commitreport.id"))
     report = relationship("CommitReport", foreign_keys=[report_id])
 
 
@@ -131,7 +131,7 @@ class UploadError(CodecovBaseModel, MixinBaseClass):
 
 class ReportDetails(CodecovBaseModel, MixinBaseClass):
     __tablename__ = "reports_reportdetails"
-    report_id = Column(types.Integer, ForeignKey("reports_commitreport.id"))
+    report_id = Column(types.BigInteger, ForeignKey("reports_commitreport.id"))
     report: CommitReport = relationship(
         "CommitReport", foreign_keys=[report_id], back_populates="details"
     )
@@ -218,13 +218,13 @@ class Meta:
 
 class ReportLevelTotals(CodecovBaseModel, AbstractTotals):
     __tablename__ = "reports_reportleveltotals"
-    report_id = Column(types.Integer, ForeignKey("reports_commitreport.id"))
+    report_id = Column(types.BigInteger, ForeignKey("reports_commitreport.id"))
     report = relationship("CommitReport", foreign_keys=[report_id])
 
 
 class UploadLevelTotals(CodecovBaseModel, AbstractTotals):
     __tablename__ = "reports_uploadleveltotals"
-    upload_id = Column("upload_id", types.Integer, ForeignKey("reports_upload.id"))
+    upload_id = Column("upload_id", types.BigInteger, ForeignKey("reports_upload.id"))
     upload = relationship("Upload", foreign_keys=[upload_id])
 
 
@@ -234,7 +234,9 @@ class CompareFlag(MixinBaseClass, CodecovBaseModel):
     commit_comparison_id = Column(
         types.BigInteger, ForeignKey("compare_commitcomparison.id")
     )
-    repositoryflag_id = Column(types.Integer, ForeignKey("reports_repositoryflag.id"))
+    repositoryflag_id = Column(
+        types.BigInteger, ForeignKey("reports_repositoryflag.id")
+    )
     head_totals = Column(postgresql.JSON)
     base_totals = Column(postgresql.JSON)
     patch_totals = Column(postgresql.JSON)
@@ -304,22 +306,22 @@ class TestInstance(CodecovBaseModel, MixinBaseClass):
     test = relationship(Test, backref=backref("testinstances"))
     duration_seconds = Column(types.Float, nullable=False)
     outcome = Column(types.String(100), nullable=False)
-    upload_id = Column(types.Integer, ForeignKey("reports_upload.id"))
+    upload_id = Column(types.BigInteger, ForeignKey("reports_upload.id"))
     upload = relationship("Upload", backref=backref("testinstances"))
     failure_message = Column(types.Text)
     branch = Column(types.Text, nullable=True)
     commitid = Column(types.Text, nullable=True)
     repoid = Column(types.Integer, nullable=True)
 
     reduced_error_id = Column(
-        types.Integer, ForeignKey("reports_reducederror.id"), nullable=True
+        types.BigInteger, ForeignKey("reports_reducederror.id"), nullable=True
     )
     reduced_error = relationship("ReducedError", backref=backref("testinstances"))
 
 
 class TestResultReportTotals(CodecovBaseModel, MixinBaseClass):
     __tablename__ = "reports_testresultreporttotals"
-    report_id = Column(types.Integer, ForeignKey("reports_commitreport.id"))
+    report_id = Column(types.BigInteger, ForeignKey("reports_commitreport.id"))
     report = relationship("CommitReport", foreign_keys=[report_id])
     passed = Column(types.Integer)
     skipped = Column(types.Integer)
@@ -341,7 +343,7 @@ class Flake(CodecovBaseModel, MixinBaseClassNoExternalID):
     test = relationship(Test, backref=backref("flakes"))
 
     reduced_error_id = Column(
-        types.Integer, ForeignKey("reports_reducederror.id"), nullable=True
+        types.BigInteger, ForeignKey("reports_reducederror.id"), nullable=True
     )
     reduced_error = relationship(ReducedError, backref=backref("flakes"))
 

helpers/parallel_upload_processing.py

@@ -1,68 +1,4 @@
-import copy
-
 import sentry_sdk
-from shared.utils.sessions import SessionType
-
-from database.models.reports import Upload
-
-
-# Copied from shared/reports/resources.py Report.next_session_number()
-def next_session_number(session_dict):
-    start_number = len(session_dict)
-    while start_number in session_dict or str(start_number) in session_dict:
-        start_number += 1
-    return start_number
-
-
-# Copied and cut down from worker/services/report/raw_upload_processor.py
-# this version stripped out all the ATS label stuff
-def _adjust_sessions(
-    original_sessions: dict,
-    to_merge_flags,
-    current_yaml,
-):
-    session_ids_to_fully_delete = []
-    flags_under_carryforward_rules = [
-        f for f in to_merge_flags if current_yaml.flag_has_carryfoward(f)
-    ]
-    if flags_under_carryforward_rules:
-        for sess_id, curr_sess in original_sessions.items():
-            if curr_sess.session_type == SessionType.carriedforward:
-                if curr_sess.flags:
-                    if any(
-                        f in flags_under_carryforward_rules for f in curr_sess.flags
-                    ):
-                        session_ids_to_fully_delete.append(sess_id)
-    if session_ids_to_fully_delete:
-        # delete sessions from dict
-        for id in session_ids_to_fully_delete:
-            original_sessions.pop(id, None)
-    return
-
-
-def get_parallel_session_ids(
-    sessions, argument_list, db_session, report_service, commit_yaml
-):
-    num_sessions = len(argument_list)
-
-    mock_sessions = copy.deepcopy(sessions)  # the sessions already in the report
-    get_parallel_session_ids = []
-
-    # iterate over all uploads, get the next session id, and adjust sessions (remove CFF logic)
-    for i in range(num_sessions):
-        next_session_id = next_session_number(mock_sessions)
-
-        upload_pk = argument_list[i]["upload_pk"]
-        upload = db_session.query(Upload).filter_by(id_=upload_pk).first()
-        to_merge_session = report_service.build_session(upload)
-        flags = upload.flag_names
-
-        mock_sessions[next_session_id] = to_merge_session
-        _adjust_sessions(mock_sessions, flags, commit_yaml)
-
-        get_parallel_session_ids.append(next_session_id)
-
-    return get_parallel_session_ids
 
 
 @sentry_sdk.trace

helpers/reports.py

@@ -18,3 +18,22 @@ def delete_archive_setting(commit_yaml: UserYaml | dict) -> bool:
     return not read_yaml_field(
         commit_yaml, ("codecov", "archive", "uploads"), _else=True
     )
+
+
+def session_id_from_upload_id(upload_id: int) -> int:
+    """
+    Creates a unique `session_id` from an `upload_id`.
+
+    The purpose of the `session_id` is to unique within one report.
+    Turning the `upload_id` into the `session_id` trivially achieves this, as
+    the `upload_id` is a unique database id.
+
+    However in production, the `upload_id` exceeds 31 bits, which overflows
+    the datatype of the `session_id`, which is a signed 32-bit integer
+    (effectively 31-bits).
+
+    We work around this problem by just masking off the high bits, essentially
+    wrapping around at that number, and shifting a bit to avoid collisions
+    with existing low session numbers.
+    """
+    return (upload_id & (2**16 - 1)) << 12

services/report/__init__.py

@@ -44,6 +44,7 @@
     RepositoryWithoutValidBotError,
 )
 from helpers.labels import get_labels_per_session
+from helpers.reports import session_id_from_upload_id
 from helpers.telemetry import MetricContext
 from rollouts import (
     CARRYFORWARD_BASE_SEARCH_RANGE_BY_OWNER,
@@ -150,18 +151,15 @@
             Upload
         """
         db_session = commit_report.get_db_session()
+        name = normalized_arguments.get("name")
         upload = Upload(
             external_id=normalized_arguments.get("reportid"),
             build_code=normalized_arguments.get("build"),
             build_url=normalized_arguments.get("build_url"),
             env=None,
             report_id=commit_report.id_,
             job_code=normalized_arguments.get("job"),
-            name=(
-                normalized_arguments.get("name")[:100]
-                if normalized_arguments.get("name")
-                else None
-            ),
+            name=(name[:100] if name else None),
             provider=normalized_arguments.get("service"),
             state="started",
             storage_path=normalized_arguments.get("url"),
@@ -202,7 +200,7 @@
 
     @sentry_sdk.trace
     def initialize_and_save_report(
-        self, commit: Commit, report_code: str = None
+        self, commit: Commit, report_code: str | None = None
     ) -> CommitReport:
         """
             Initializes the commit report
@@ -412,7 +410,7 @@
         Does not include CF sessions if there is also an upload session with the same
         flag name.
         """
-        sessions = {}
+        sessions: dict[int, Session] = {}
 
         carryforward_sessions = {}
         uploaded_flags = set()
@@ -431,9 +429,9 @@
         for upload in report_uploads:
             session = self.build_session(upload)
             if session.session_type == SessionType.carriedforward:
-                carryforward_sessions[upload.order_number] = session
+                carryforward_sessions[session.id] = session
             else:
-                sessions[upload.order_number] = session
+                sessions[session.id] = session
                 uploaded_flags |= set(session.flags)
 
         for sid, session in carryforward_sessions.items():
@@ -873,7 +871,6 @@
         report: Report,
         raw_report_info: RawReportInfo,
         upload: Upload,
-        parallel_idx=None,
     ) -> ProcessingResult:
         """
         Processes an upload on top of an existing report `master` and returns
@@ -884,23 +881,19 @@
         """
         commit = upload.report.commit
         flags = upload.flag_names
-        service = upload.provider
-        build_url = upload.build_url
-        build = upload.build_code
-        job = upload.job_code
-        name = upload.name
         archive_url = upload.storage_path
         reportid = upload.external_id
 
         session = Session(
-            provider=service,
-            build=build,
-            job=job,
-            name=name,
+            id=session_id_from_upload_id(upload.id),
+            provider=upload.provider,
+            build=upload.build_code,
+            job=upload.job_code,
+            name=upload.name,
             time=int(time()),
             flags=flags,
             archive=archive_url,
-            url=build_url,
+            url=upload.build_url,
         )
         result = ProcessingResult(session=session)
 
@@ -919,7 +912,6 @@
                     reportid=reportid,
                     commit_yaml=self.current_yaml.to_dict(),
                     archive_url=archive_url,
-                    in_parallel=parallel_idx is not None,
                 ),
             )
             result.error = ProcessingError(
@@ -955,13 +947,11 @@
                     raw_report,
                     flags,
                     session,
-                    upload=upload,
-                    parallel_idx=parallel_idx,
+                    upload,
                 )
                 result.report = process_result.report
             log.info(
-                "Successfully processed report"
-                + (" (in parallel)" if parallel_idx is not None else ""),
+                "Successfully processed report",
                 extra=dict(
                     session=session.id,
                     ci=f"{session.provider}:{session.build}:{session.job}",
@@ -1066,7 +1056,7 @@
                 error_params=error.params,
             )
             db_session.add(error_obj)
-            db_session.flush()
+        db_session.flush()
 
     @sentry_sdk.trace
     def save_report(self, commit: Commit, report: Report, report_code=None):
@@ -1193,14 +1183,14 @@
                 ),
             )
             db_session.add(upload)
-            db_session.flush()
             self._attach_flags_to_upload(upload, session.flags if session.flags else [])
             if session.totals is not None:
                 upload_totals = UploadLevelTotals(upload_id=upload.id_)
                 db_session.add(upload_totals)
                 upload_totals.update_from_totals(
                     session.totals, precision=precision, rounding=rounding
                 )
+        db_session.flush()
         return res
 
     @sentry_sdk.trace