Increase main db timeout + logging tweaks (#1017)

* Set org runners for helmfile. * Remove incompatible fields. * Use helm runner. * Add logging to repo_helper to get timing info. * Increase timeout for token balance fetcher + production db connection pool. * Revert "Add logging to repo_helper to get timing info." This reverts commit 24bc5bc. * Use blockscout runners. * Increase timeouts further. * Run on 8-cpu runners. * Log missing block ranges. * Reduce block fetch concurrency. * Remove noisy sanitizer log. * Reorder runner labels. * Suppress logs + revert remote script. * More logging tweaks. * Format. * Set transaction cache lock timeout to 30 seconds. * Don't update tx cache from indexer. * Fix unused variable.
celo-org · Feb 6, 2024 · 5c45da6 · 5c45da6
1 parent 0652afb
commit 5c45da6
Show file tree

Hide file tree

Showing 11 changed files with 34 additions and 29 deletions.
diff --git a/.github/workflows/blockscout.yml b/.github/workflows/blockscout.yml
@@ -25,7 +25,7 @@ env:
 jobs:
   build-and-cache:
     name: Build and Cache deps
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: [ '8-cpu','self-hosted', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -94,7 +94,7 @@ jobs:
 
   credo:
     name: Credo
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -121,7 +121,7 @@ jobs:
 
   check_formatted:
     name: Code formatting checks
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -147,7 +147,7 @@ jobs:
       - run: mix format --check-formatted
   dialyzer:
     name: Dialyzer static analysis
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -190,7 +190,7 @@ jobs:
 
   gettext:
     name: Missing translation keys check
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -219,7 +219,7 @@ jobs:
         working-directory: "apps/block_scout_web"
   sobelow:
     name: Sobelow security analysis
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -250,7 +250,7 @@ jobs:
         working-directory: "apps/block_scout_web"
   eslint:
     name: ESLint
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -307,7 +307,7 @@ jobs:
 
   jest:
     name: JS Tests
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -355,7 +355,7 @@ jobs:
 
   test_nethermind_mox_ethereum_jsonrpc:
     name: EthereumJSONRPC Tests
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -423,7 +423,7 @@ jobs:
           path: _build/test/junit/ethereum_jsonrpc/*.xml
   test_nethermind_mox_explorer:
     name: Explorer Tests
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -512,7 +512,7 @@ jobs:
           path: _build/test/junit/explorer/*.xml
   test_nethermind_mox_indexer:
     name: Indexer Tests
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -585,7 +585,7 @@ jobs:
 
   test_nethermind_mox_block_scout_web:
     name: Blockscout Web Tests
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
       options: --user root
@@ -702,7 +702,7 @@ jobs:
 
   publish-test-results:
     name: "Publish Unit Tests Results"
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
     needs:
@@ -724,7 +724,7 @@ jobs:
           files: artifacts/**/*.xml
 
   set-docker-vars:
-    runs-on: ['self-hosted', 'org', '8-cpu']
+    runs-on: ['self-hosted', '8-cpu', 'blockscout']
     container:
       image: us-west1-docker.pkg.dev/devopsre/actions-runner-controller/blockscout:latest
     needs: [credo, check_formatted, dialyzer, gettext, sobelow, eslint, jest, test_nethermind_mox_ethereum_jsonrpc, test_nethermind_mox_explorer, test_nethermind_mox_indexer, test_nethermind_mox_block_scout_web]

diff --git a/apps/explorer/config/prod.exs b/apps/explorer/config/prod.exs
@@ -3,7 +3,7 @@ import Config
 # Configures the database
 config :explorer, Explorer.Repo.Local,
   prepare: :unnamed,
-  timeout: :timer.seconds(60),
+  timeout: :timer.minutes(6),
   migration_lock: nil,
   telemetry_prefix: [:explorer, :repo]
 

diff --git a/apps/explorer/lib/explorer/chain/cache/transactions.ex b/apps/explorer/lib/explorer/chain/cache/transactions.ex
@@ -17,6 +17,7 @@ defmodule Explorer.Chain.Cache.Transactions do
       token_transfers: :from_address,
       token_transfers: :to_address
     ],
+    acquire_lock_timeout: :timer.seconds(30),
     ttl_check_interval: Application.get_env(:explorer, __MODULE__)[:ttl_check_interval],
     global_ttl: Application.get_env(:explorer, __MODULE__)[:global_ttl]
 

diff --git a/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex b/apps/explorer/lib/explorer/chain/import/runner/address/token_balances.ex
@@ -14,7 +14,7 @@ defmodule Explorer.Chain.Import.Runner.Address.TokenBalances do
   @behaviour Import.Runner
 
   # milliseconds
-  @timeout 60_000
+  @timeout 300_000
 
   @type imported :: [TokenBalance.t()]
 

diff --git a/apps/explorer/lib/explorer/chain/import/runner/exchange_rate.ex b/apps/explorer/lib/explorer/chain/import/runner/exchange_rate.ex
@@ -14,7 +14,7 @@ defmodule Explorer.Chain.Import.Runner.ExchangeRate do
   @behaviour Import.Runner
 
   # milliseconds
-  @timeout 60_000
+  @timeout 30_000
 
   @type imported :: [ExchangeRate.t()]
 

diff --git a/apps/indexer/lib/indexer/block/catchup/fetcher.ex b/apps/indexer/lib/indexer/block/catchup/fetcher.ex
@@ -72,7 +72,7 @@ defmodule Indexer.Block.Catchup.Fetcher do
             |> Stream.map(&Enum.count/1)
             |> Enum.sum()
 
-          Logger.debug(fn -> "Missed blocks in ranges." end,
+          Logger.info(fn -> "Missed blocks in ranges." end,
             missing_block_range_count: range_count,
             missing_block_count: missing_block_count
           )

diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex
@@ -17,7 +17,7 @@ defmodule Indexer.Block.Fetcher do
   alias Explorer.Chain.{Address, Block, Hash, Import, Transaction, Wei}
   alias Explorer.Chain.Block.Reward
   alias Explorer.Chain.Cache.Blocks, as: BlocksCache
-  alias Explorer.Chain.Cache.{Accounts, BlockNumber, Transactions, Uncles}
+  alias Explorer.Chain.Cache.{Accounts, BlockNumber, Uncles}
   alias Indexer.Block.Fetcher.Receipts
 
   alias Explorer.Celo.Util
@@ -380,8 +380,10 @@ defmodule Indexer.Block.Fetcher do
 
   defp update_block_cache(_), do: :ok
 
-  defp update_transactions_cache(transactions) do
-    Transactions.update(transactions)
+  defp update_transactions_cache(_transactions) do
+    # Celo - Indexer and Web are not using the same cache instance, no need to update here
+    # Transactions.update(transactions)
+    :ok
   end
 
   defp update_addresses_cache(addresses), do: Accounts.drop(addresses)

diff --git a/apps/indexer/lib/indexer/block/realtime/fetcher.ex b/apps/indexer/lib/indexer/block/realtime/fetcher.ex
@@ -342,15 +342,17 @@ defmodule Indexer.Block.Realtime.Fetcher do
         end
 
       {:error, {:import = step, reason}} ->
-        Logger.error(fn -> inspect(reason) end, step: step)
+        Logger.error(fn -> " Error #{inspect(block_number_to_fetch)} #{inspect(step)} - #{inspect(reason)}" end,
+          step: step
+        )
 
       {:error, {step, reason}} ->
         Logger.error(
           fn ->
             [
               "failed to fetch: ",
               inspect(reason),
-              ".  Block will be retried by catchup indexer."
+              ".  Block #{inspect(block_number_to_fetch)} will be retried by catchup indexer."
             ]
           end,
           step: step

diff --git a/apps/indexer/lib/indexer/fetcher/empty_blocks_sanitizer.ex b/apps/indexer/lib/indexer/fetcher/empty_blocks_sanitizer.ex
@@ -57,7 +57,7 @@ defmodule Indexer.Fetcher.EmptyBlocksSanitizer do
         :sanitize_empty_blocks,
         %{interval: interval, json_rpc_named_arguments: json_rpc_named_arguments} = state
       ) do
-    Logger.info("Start sanitizing of empty blocks. Batch size is #{limit()}",
+    Logger.debug("Start sanitizing of empty blocks. Batch size is #{limit()}",
       fetcher: :empty_blocks_to_refetch
     )
 
@@ -95,7 +95,7 @@ defmodule Indexer.Fetcher.EmptyBlocksSanitizer do
           |> Enum.count()
 
         if transactions_count > 0 do
-          Logger.info(
+          Logger.debug(
             "Block with number #{block_number} and hash #{to_string(block_hash)} is full of transactions. We should set consensus=false for it in order to refetch.",
             fetcher: :empty_blocks_to_refetch
           )
@@ -112,7 +112,7 @@ defmodule Indexer.Fetcher.EmptyBlocksSanitizer do
       end
     end)
 
-    Logger.info("Batch of empty blocks is sanitized",
+    Logger.debug("Batch of empty blocks is sanitized",
       fetcher: :empty_blocks_to_refetch
     )
   end

diff --git a/apps/indexer/lib/indexer/fetcher/token_instance.ex b/apps/indexer/lib/indexer/fetcher/token_instance.ex
@@ -85,7 +85,7 @@ defmodule Indexer.Fetcher.TokenInstance do
       result ->
         Telemetry.event([:indexer, :nft, :ingestion_errors], %{count: 1})
 
-        Logger.error(
+        Logger.debug(
           [
             "failed to fetch token instance metadata for #{inspect({to_string(token_contract_address_hash), Decimal.to_integer(token_id)})}: ",
             inspect(result)

diff --git a/config/runtime.exs b/config/runtime.exs
@@ -512,8 +512,8 @@ config :indexer, Indexer.Block.Realtime.Supervisor, enabled: System.get_env("DIS
 config :indexer, Indexer.Fetcher.TokenInstance.Supervisor,
   disabled?: System.get_env("DISABLE_TOKEN_INSTANCE_FETCHER", "false") == "true"
 
-blocks_catchup_fetcher_batch_size_default_str = "10"
-blocks_catchup_fetcher_concurrency_default_str = "10"
+blocks_catchup_fetcher_batch_size_default_str = "5"
+blocks_catchup_fetcher_concurrency_default_str = "5"
 
 {blocks_catchup_fetcher_batch_size, _} =
   Integer.parse(System.get_env("INDEXER_CATCHUP_BLOCKS_BATCH_SIZE", blocks_catchup_fetcher_batch_size_default_str))