publish to prod

CHANGELOG.md

@@ -1,5 +1,17 @@
 # backend
 
+## 1.20.0
+
+### Minor Changes
+
+-   9d9b90d: Subgraph client will fallback to another URL on failure
+
+### Patch Changes
+
+-   d813678: dev setup with hot reloading
+-   5ea8f51: adding prodction base subgraph url
+-   7f76312: AAVE APRs on gnosis and wUSDM on OP
+
 ## 1.19.0
 
 ### Minor Changes

README.md

@@ -106,7 +106,7 @@ Query the SOR to swap 1 WETH to USDC
 
 ## Pricing of tokens
 
-First of all, for a token to be able to have a price it must be allowed, meaning it must be added to the [tokenlist](https://github.com/balancer/tokenlists). This must happen *before* any pricing can occur. 
+First of all, for a token to be able to have a price it must be allowed, meaning it must be added to the [tokenlist](https://github.com/balancer/tokenlists). This must happen _before_ any pricing can occur.
 
 To price a token there are various handlers that will try to price a token. These handlers take priority over each other. This means that as soon
 as a handler can price a token, it will not be price by another handler. These handlers, order by priority, are:
@@ -165,6 +165,12 @@ userInitWalletBalancesForAllPools
 userInitStakedBalances
 ```
 
+You can do that by starting the server in development mode with hot reloading and calling the methods via API playground, or curl from the shell:
+
+```
+curl -d '{"query":"mutation { poolSyncAllPoolsFromSubgraph }"}' -H 'Content-Type: application/json' -H 'chainId: 1' -H "AdminApiKey: $(grep '^ADMIN_API_KEY=' .env | cut -d '=' -f2)" http://localhost:4000/graphql
+```
+
 ### Setup database & Prisma from backup
 
 Retrieve the current pg_dump file under `https://api-db-dump.s3.eu-central-1.amazonaws.com/canary/api-dump.YYYYMMDD`.
@@ -178,7 +184,7 @@ The output at the very end saying `ERROR: role "rdsadmin" does not exist` is nor
 
 ## Run locally
 
-`yarn start:local`
+`yarn dev`
 
 ## Branching and deployment environments
 

apps/env.ts

@@ -33,6 +33,10 @@ export const schema = {
         optional: true,
         type: String,
     },
+    SATSUMA_API_KEY: {
+        optional: true,
+        type: String,
+    },
     WORKER_QUEUE_URL: {
         optional: true,
         type: String,

config/arbitrum.ts

@@ -12,7 +12,10 @@ export default <NetworkData>{
     },
     subgraphs: {
         startDate: '2021-08-23',
-        balancer: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmPbjY6L1NhPjpBv7wDTfG9EPx5FpCuBqeg1XxByzBTLcs`,
+        balancer: [
+            `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmPbjY6L1NhPjpBv7wDTfG9EPx5FpCuBqeg1XxByzBTLcs`,
+            `https://subgraph.satsuma-prod.com/${env.SATSUMA_API_KEY}/balancer/balancer-v2-arbitrum/api`,
+        ],
         cowAmm: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmTSU862YAXb5XMhGsE7JCajuvf5FPiZjrdvC9nnbzd86x`,
         beetsBar: 'https://',
         blocks: 'https://api.studio.thegraph.com/query/48427/arbitrum-blocks/version/latest',

config/avalanche.ts

@@ -12,7 +12,9 @@ export default <NetworkData>{
     },
     subgraphs: {
         startDate: '2023-06-06',
-        balancer: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmeJY1ZjmuJVPvmVghZSuiSxEx2a9kmpKnjr4Qw5hNdpLU`,
+        balancer: [
+            `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmeJY1ZjmuJVPvmVghZSuiSxEx2a9kmpKnjr4Qw5hNdpLU`,
+        ],
         beetsBar: 'https://',
         blocks: 'https://api.studio.thegraph.com/query/48427/avalanche-blocks/version/latest',
         gauge: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmYCJJToWTY31LgsJG2vZTNkKWpQ7i91cTVYgPUBsJ5nQZ`,

config/base.ts

@@ -12,7 +12,10 @@ export default <NetworkData>{
     },
     subgraphs: {
         startDate: '2023-07-10',
-        balancer: `https://api.studio.thegraph.com/query/24660/balancer-base-v2/version/latest`,
+        balancer: [
+            `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmRKBwBwPKtFz4mQp5jvH44USVprM4C77Nr4m77UGCbGv9`,
+            `https://api.studio.thegraph.com/query/24660/balancer-base-v2/version/latest`,
+        ],
         beetsBar: '',
         blocks: 'https://api.studio.thegraph.com/query/48427/bleu-base-blocks/version/latest',
         gauge: `https://api.studio.thegraph.com/query/24660/balancer-gauges-base/version/latest`,

config/fantom.ts

@@ -13,7 +13,10 @@ export default <NetworkData>{
     },
     subgraphs: {
         startDate: '2021-10-08',
-        balancer: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_FANTOM}/deployments/id/QmYN8qV7PEokFeQvhhWMinYD5wsspP1Sc87pGKEvAmjSCJ`,
+        balancer: [
+            `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_FANTOM}/deployments/id/QmYN8qV7PEokFeQvhhWMinYD5wsspP1Sc87pGKEvAmjSCJ`,
+            `https://subgraph.satsuma-prod.com/${env.SATSUMA_API_KEY}/balancer/balancer-v2-fantom/api`,
+        ],
         beetsBar: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_FANTOM}/deployments/id/QmXcxzZioHXV5ts2UcG6gNNEayoaZ9ip7D9JvPS88K2HXe`,
         blocks: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_FANTOM}/subgraphs/id/3drjZDpA9hAuYGA19ttEkhW432mVe2XHy5YarBDVYHbz`,
         masterchef: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_FANTOM}/deployments/id/QmZQJu1rxMEDwzZb5TSqDXjcFiS1DN8BRKCLKRv6ifEBhA`,

config/fraxtal.ts

@@ -11,8 +11,9 @@ export default <NetworkData>{
     },
     subgraphs: {
         startDate: '2024-05-22',
-        balancer:
+        balancer: [
             'https://api.goldsky.com/api/public/project_clwhu1vopoigi01wmbn514m1z/subgraphs/balancer-fraxtal-v2/latest/gn',
+        ],
         beetsBar: '',
         blocks: 'https://api.goldsky.com/api/public/project_clwhu1vopoigi01wmbn514m1z/subgraphs/fraxtal-blocks/1.0.0/gn',
         gauge: 'https://api.goldsky.com/api/public/project_clwhu1vopoigi01wmbn514m1z/subgraphs/balancer-gauges-fraxtal/latest/gn',

config/gnosis.ts

@@ -12,7 +12,9 @@ export default <NetworkData>{
     },
     subgraphs: {
         startDate: '2021-08-23',
-        balancer: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmXXSKeLh14DnJgR1ncHhAHciqacfRshcHKXasAGy7LP4Y`,
+        balancer: [
+            `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmXXSKeLh14DnJgR1ncHhAHciqacfRshcHKXasAGy7LP4Y`,
+        ],
         beetsBar: 'https://',
         blocks: 'https://api.studio.thegraph.com/query/48427/gnosis-blocks/version/latest',
         gauge: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/Qme9hQY1NZ8ReVDSSQb893s2fGpeLkgfwXd3YU5rndACaP`,
@@ -83,6 +85,27 @@ export default <NetworkData>{
                 isIbYield: true,
             },
         },
+        aave: {
+            v3: {
+                subgraphUrl: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/subgraphs/id/HtcDaL8L8iZ2KQNNS44EBVmLruzxuNAz1RkBYdui1QUT`,
+                tokens: {
+                    USDC: {
+                        underlyingAssetAddress: '0xddafbb505ad214d7b80b1f830fccc89b60fb7a83',
+                        aTokenAddress: '0xc6b7aca6de8a6044e0e32d0c841a89244a10d284',
+                        wrappedTokens: {
+                            stataGnoUSDC: '0x270ba1f35d8b87510d24f693fccc0da02e6e4eeb',
+                        },
+                    },
+                    USDCn: {
+                        underlyingAssetAddress: '0x2a22f9c3b484c3629090feed35f17ff8f88f76f0',
+                        aTokenAddress: '0xc0333cb85b59a788d8c7cae5e1fd6e229a3e5a65',
+                        wrappedTokens: {
+                            stataGnoUSDCe: '0xf0e7ec247b918311afa054e0aedb99d74c31b809',
+                        },
+                    },
+                },
+            },
+        },
     },
     gyro: {
         config: '0x00a2a9bbd352ab46274433faa9fec35fe3abb4a8',

config/mainnet.ts

@@ -22,7 +22,9 @@ export default <NetworkData>{
     subgraphs: {
         startDate: '2019-04-20',
         cowAmm: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmUvfS6hqU3nGQxFFzxoMBkufYJ7Jh3cYdTUM64hucgqe7`,
-        balancer: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmQ5TT2yYBZgoUxsat3bKmNe5Fr9LW9YAtDs8aeuc1BRhj`,
+        balancer: [
+            `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmQ5TT2yYBZgoUxsat3bKmNe5Fr9LW9YAtDs8aeuc1BRhj`,
+        ],
         beetsBar: 'https://',
         blocks: 'https://api.studio.thegraph.com/query/48427/ethereum-blocks/version/latest',
         gauge: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmdmQBHbBtwD6wNypHbuGKB1uKHpHNVuSHbo9FsvrMhXSn`,

config/mode.ts

@@ -11,7 +11,7 @@ export default <NetworkData>{
     },
     subgraphs: {
         startDate: '2024-05-22',
-        balancer: `https://api.studio.thegraph.com/query/75376/balancer-mode-v2/version/latest`,
+        balancer: [`https://api.studio.thegraph.com/query/75376/balancer-mode-v2/version/latest`],
         beetsBar: '',
         blocks: 'https://api.studio.thegraph.com/query/48427/bleu-mode-blocks/version/latest',
         gauge: `https://api.studio.thegraph.com/query/75376/balancer-gauges-mode/version/latest`,

config/optimism.ts

@@ -13,7 +13,10 @@ export default <NetworkData>{
     },
     subgraphs: {
         startDate: '2022-01-01',
-        balancer: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmWUgkiUM5c3BW1Z51DUkZfnyQfyfesE8p3BRnEtA9vyPL`,
+        balancer: [
+            `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmWUgkiUM5c3BW1Z51DUkZfnyQfyfesE8p3BRnEtA9vyPL`,
+            `https://subgraph.satsuma-prod.com/${env.SATSUMA_API_KEY}/balancer/balancer-v2-optimism/api`,
+        ],
         beetsBar: 'https://',
         blocks: 'https://api.studio.thegraph.com/query/48427/optimism-blocks/version/latest',
         gauge: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/Qmdtj1ix1nUCRtSoiyF7a3oKMSvrKT8KTEFJdep53EHtRy`,
@@ -246,6 +249,13 @@ export default <NetworkData>{
                 path: 'value',
                 isIbYield: true,
             },
+            wusdm: {
+                tokenAddress: '0x57f5e098cad7a3d1eed53991d4d66c45c9af7812',
+                sourceUrl: 'https://apy.prod.mountainprotocol.com',
+                path: 'value',
+                isIbYield: true,
+                scale: 1,
+            },
         },
     },
     datastudio: {

config/polygon.ts

@@ -12,7 +12,9 @@ export default <NetworkData>{
     },
     subgraphs: {
         startDate: '2021-06-16',
-        balancer: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmUqS6BAVQgvstEsVrxuwsu1DwQdfAdj3Q6gz2j3DbUYQ9`,
+        balancer: [
+            `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmUqS6BAVQgvstEsVrxuwsu1DwQdfAdj3Q6gz2j3DbUYQ9`,
+        ],
         beetsBar: 'https://',
         blocks: 'https://api.studio.thegraph.com/query/48427/polygon-blocks/version/latest',
         gauge: `https://gateway-arbitrum.network.thegraph.com/api/${env.THEGRAPH_API_KEY_BALANCER}/deployments/id/QmewSgLJf9TZt8trr61dECJhEGGyHxKFWbNQ3AnNZAdYyU`,

config/sepolia.ts

@@ -13,7 +13,7 @@ export default <NetworkData>{
     subgraphs: {
         startDate: '2023-05-03',
         cowAmm: 'https://api.studio.thegraph.com/proxy/75376/balancer-cow-amm-sepolia/version/latest',
-        balancer: 'https://api.studio.thegraph.com/query/24660/balancer-sepolia-v2/version/latest',
+        balancer: ['https://api.studio.thegraph.com/query/24660/balancer-sepolia-v2/version/latest'],
         balancerV3: 'https://api.studio.thegraph.com/query/31386/balancer-v3-sepolia-8th/version/latest',
         balancerPoolsV3: 'https://api.studio.thegraph.com/query/31386/balancer-pools-v3-sepolia-8th/version/latest',
         beetsBar: 'https://',

config/zkevm.ts

@@ -12,7 +12,7 @@ export default <NetworkData>{
     },
     subgraphs: {
         startDate: '2023-05-17',
-        balancer: `https://api.studio.thegraph.com/query/24660/balancer-polygon-zk-v2/version/latest`,
+        balancer: [`https://api.studio.thegraph.com/query/24660/balancer-polygon-zk-v2/version/latest`],
         beetsBar: 'https://',
         blocks: 'https://api.studio.thegraph.com/query/48427/bleu-polygon-zkevm-blocks/version/latest',
         gauge: `https://api.studio.thegraph.com/query/24660/balancer-gauges-polygon-zk/version/latest`,

env.local

@@ -4,6 +4,7 @@ DEPLOYMENT_ENV=canary
 
 # RPCs
 ALCHEMY_API_KEY=
+SATSUMA_API_KEY=
 
 # Database
 DATABASE_URL=postgresql://backend:let-me-in@localhost:5431/database?schema=public

modules/network/network-config-types.ts

@@ -61,7 +61,7 @@ export interface NetworkData {
     };
     subgraphs: {
         startDate: string;
-        balancer: string;
+        balancer: string[];
         balancerV3?: string;
         balancerPoolsV3?: string;
         blocks: string;

modules/protocol/protocol.service.ts

@@ -4,12 +4,13 @@ import { Cache } from 'memory-cache';
 import { Chain, PrismaLastBlockSyncedCategory, PrismaUserBalanceType } from '@prisma/client';
 import _ from 'lodash';
 import { networkContext } from '../network/network-context.service';
-import { AllNetworkConfigs, AllNetworkConfigsKeyedOnChain } from '../network/network-config';
+import { AllNetworkConfigs, AllNetworkConfigsKeyedOnChain, chainToIdMap } from '../network/network-config';
 import { GqlProtocolMetricsAggregated, GqlProtocolMetricsChain } from '../../schema';
 import { GraphQLClient } from 'graphql-request';
 import { getSdk } from '../subgraphs/balancer-subgraph/generated/balancer-subgraph-types';
 import axios from 'axios';
 import { tokenService } from '../token/token.service';
+import { getV2SubgraphClient } from '../subgraphs/balancer-subgraph';
 
 interface LatestSyncedBlocks {
     userWalletSyncBlock: string;
@@ -69,10 +70,12 @@ export class ProtocolService {
     public async cacheProtocolMetrics(chain: Chain): Promise<GqlProtocolMetricsChain> {
         const oneDayAgo = moment().subtract(24, 'hours').unix();
 
-        const client = new GraphQLClient(AllNetworkConfigsKeyedOnChain[chain].data.subgraphs.balancer);
-        const subgraphClient = getSdk(client);
+        const client = getV2SubgraphClient(
+            AllNetworkConfigsKeyedOnChain[chain].data.subgraphs.balancer,
+            Number(chainToIdMap[chain]),
+        );
 
-        const { balancers } = await subgraphClient.BalancerProtocolData({});
+        const { balancers } = await client.BalancerProtocolData({});
         const { totalSwapFee, totalSwapVolume, poolCount } = balancers[0];
 
         const pools = await prisma.prismaPool.findMany({

modules/sources/subgraphs/retry-on-failure.spec.ts

@@ -0,0 +1,52 @@
+import { retryOnFailureWithRotation } from './retry-on-failure';
+
+describe('retryOnFailureWithRotation', () => {
+    it('should retry with the next SDK client when the first one fails', async () => {
+        // Mock SDK clients
+        const sdkClient1 = {
+            fetchData: jest.fn().mockRejectedValueOnce(new Error('First client error')),
+        };
+        const sdkClient2 = {
+            fetchData: jest.fn().mockResolvedValueOnce('Second client success'),
+        };
+
+        const sdkClients = [sdkClient1, sdkClient2];
+
+        // Function to be retried
+        const fn = (sdk: any) => sdk.fetchData();
+
+        // Call the function
+        const result = await retryOnFailureWithRotation(sdkClients, fn, 2);
+
+        // Verify the result
+        expect(result).toBe('Second client success');
+
+        // Verify the calls
+        expect(sdkClient1.fetchData).toHaveBeenCalledTimes(1);
+        expect(sdkClient2.fetchData).toHaveBeenCalledTimes(1);
+    });
+
+    it('should throw an error after exhausting retries', async () => {
+        // Mock SDK clients
+        const sdkClient1 = {
+            fetchData: jest.fn().mockRejectedValue(new Error('First client error')),
+        };
+        const sdkClient2 = {
+            fetchData: jest.fn().mockRejectedValue(new Error('Second client error')),
+        };
+
+        const sdkClients = [sdkClient1, sdkClient2];
+
+        // Function to be retried
+        const fn = (sdk: any) => sdk.fetchData();
+
+        // Call the function and expect an error
+        await expect(retryOnFailureWithRotation(sdkClients, fn, 2)).rejects.toThrow(
+            'All SDK clients failed after retries.',
+        );
+
+        // Verify the calls
+        expect(sdkClient1.fetchData).toHaveBeenCalledTimes(1);
+        expect(sdkClient2.fetchData).toHaveBeenCalledTimes(1);
+    });
+});

modules/sources/subgraphs/retry-on-failure.ts

@@ -0,0 +1,53 @@
+export async function retryOnFailureWithRotation<T>(
+    sdkClients: any[],
+    fn: (sdk: any) => Promise<T>,
+    retries: number = 3,
+): Promise<T> {
+    let attempts = 0;
+    let currentSdkIndex = 0;
+
+    while (attempts < retries) {
+        try {
+            const sdk = sdkClients[currentSdkIndex]; // Get the current SDK client
+            return await fn(sdk); // Try the operation using the current SDK
+        } catch (error: any) {
+            attempts += 1;
+            console.log(`Subgraph URL from index ${currentSdkIndex} on ${attempts} attempt failed:`, error.message);
+
+            if (attempts < retries) {
+                // Rotate to the next SDK client
+                currentSdkIndex = (currentSdkIndex + 1) % sdkClients.length;
+                console.log(`Retrying with URL from index ${currentSdkIndex}...`);
+                if (error.message.includes('429')) {
+                    // Sleep for x seconds if the error is a 429, would be better to use the Retry-After header
+                    await new Promise((resolve) => setTimeout(resolve, 10000 * attempts));
+                }
+            } else {
+                throw new Error('All SDK clients failed after retries.');
+            }
+        }
+    }
+
+    throw new Error('Unexpected failure, retries exhausted.');
+}
+
+export function wrapSdkWithRetryAndRotation<T extends object>(sdkClients: T[], retries: number = 3): T {
+    const wrappedSdk: Partial<T> = {};
+
+    // Use one SDK as a template for wrapping all functions
+    const sdk = sdkClients[0];
+
+    for (const key of Object.keys(sdk)) {
+        const value = (sdk as any)[key];
+
+        if (typeof value === 'function') {
+            // Wrap each SDK method to use retry with rotation
+            wrappedSdk[key as keyof T] = ((...args: any[]) =>
+                retryOnFailureWithRotation(sdkClients, (sdk) => value.apply(sdk, args), retries)) as T[keyof T];
+        } else {
+            wrappedSdk[key as keyof T] = value;
+        }
+    }
+
+    return wrappedSdk as T;
+}