Rename maxAmount, fix Tez vs uTez logic, decrease min Balance in config

.github/workflows/deploy.yml

@@ -38,7 +38,7 @@ jobs:
             type=semver,pattern={{major}}
 
       - name: Login to GHCR
-        if: github.event_name != 'pull_request'
+        if: github.event_name == 'pull_request'
         uses: docker/login-action@v2
         with:
           registry: ghcr.io

README.md

@@ -1,6 +1,14 @@
-Keygen is a tool for creating keys to be used with Teaquito Integration tests
+Keygen is a tool for creating keys to be used with Taquito Integration tests
+
+# Context:
+In Taquito, we cannot run all integration tests using a single account. There are two reasons for this:
+
+1. Some tests require the account to not be `reveal`ed.
+1. In Tezos, each account can have only one `ManagerOperation` in each block. Because of that, if we need to run multiple integration tests t once, we should run them using different accounts.
+
+
 
 # Docker:
 
 
-# Local Hosting
+# Local Hosting

cspell.json

@@ -0,0 +1,21 @@
+{
+    "words": [
+        "funder",
+        "ghostnet",
+        "ligo",
+        "mumbainet",
+        "nairobinet",
+        "taqueria",
+        "Taquito",
+        "Tezos"
+    ],
+    "ignoreRegExpList": [
+        "KT1[a-zA-Z0-9]{33}",
+        "o[a-zA-Z0-9]{50}",
+        "tz[1-4][a-zA-Z0-9]{33}",
+        "tx[a-zA-Z0-9]{35}",
+        "B[LMK][a-zA-Z0-9]{49}",
+        "[\"':][a-zA-Z0-9]{54}[\"':]",
+        "[\"':][a-zA-Z0-9]{51}[\"':]"
+    ]
+}

ephemeral-config.json

@@ -1,6 +1,6 @@
 {
   "ephemeral-keys-ghostnet": {
-    "maxAmount": 30,
+    "minBalanceInTez": 2,
     "expire": 300,
     "pool-id": "ephemeral-keys-ghostnet"
   }

pools-config.json

@@ -1,20 +1,82 @@
 {
-
+  "ligo-ide-ghostnet": {
+    "funderPKH": "tz1bRt6Lo9KRNEUF9voCwkjR2pkMU9xJuYMB",
+    "remoteSignerUrl": "http://signatory:6732",
+    "batchSize": 100,
+    "rpcUrl": "http://ecad-ghostnet-rolling.i.tez.ie:8732",
+    "redisListName": "ligo-ide:ghostnet:address-pool"
+  },
+  "ligo-ide-mumbainet": {
+    "funderPKH": "tz1bRt6Lo9KRNEUF9voCwkjR2pkMU9xJuYMB",
+    "remoteSignerUrl": "http://signatory:6732",
+    "batchSize": 100,
+    "rpcUrl": "http://ecad-mumbainet-full.i.tez.ie:8732",
+    "redisListName": "ligo-ide:mumbainet:address-pool"
+  },
+  "ligo-ide-nairobinet": {
+    "funderPKH": "tz1bRt6Lo9KRNEUF9voCwkjR2pkMU9xJuYMB",
+    "remoteSignerUrl": "http://signatory:6732",
+    "batchSize": 100,
+    "rpcUrl": "http://ecad-nairobinet-full.i.tez.ie:8732",
+    "redisListName": "ligo-ide:nairobinet:address-pool"
+  },
   "taquito-example-ghostnet": {
-    "funderPKH": "tz1MtFJuM9Hsht9eKW8ikcadaA9GhSP3Ce1J",
-    "remoteSignerUrl": "http://localhost:6732",
-    "batchSize": 2,
-    "rpcUrl": "https://ghostnet.ecadinfra.com/",
+    "funderPKH": "tz1bRt6Lo9KRNEUF9voCwkjR2pkMU9xJuYMB",
+    "remoteSignerUrl": "http://signatory:6732",
+    "batchSize": 100,
+    "rpcUrl": "http://ecad-ghostnet-rolling.i.tez.ie:8732",
     "redisListName": "taquito-example:ghostnet:address-pool"
   },
+  "taquito-example-mumbainet": {
+    "funderPKH": "tz1bRt6Lo9KRNEUF9voCwkjR2pkMU9xJuYMB",
+    "remoteSignerUrl": "http://signatory:6732",
+    "batchSize": 100,
+    "rpcUrl": "http://ecad-mumbainet-full.i.tez.ie:8732",
+    "redisListName": "taquito-example:mumbainet:address-pool"
+  },
+  "taquito-example-nairobinet": {
+    "funderPKH": "tz1bRt6Lo9KRNEUF9voCwkjR2pkMU9xJuYMB",
+    "remoteSignerUrl": "http://signatory:6732",
+    "batchSize": 100,
+    "rpcUrl": "http://ecad-nairobinet-full.i.tez.ie:8732",
+    "redisListName": "taquito-example:nairobinet:address-pool"
+  },
+  "taqueria-ghostnet": {
+    "funderPKH": "tz1bRt6Lo9KRNEUF9voCwkjR2pkMU9xJuYMB",
+    "remoteSignerUrl": "http://signatory:6732",
+    "batchSize": 50,
+    "rpcUrl": "http://ecad-ghostnet-rolling.i.tez.ie:8732",
+    "redisListName": "taqueria:ghostnet:address-pool"
+  },
   "ephemeral-keys-ghostnet": {
-    "funderPKH": "tz1MtFJuM9Hsht9eKW8ikcadaA9GhSP3Ce1J",
-    "remoteSignerUrl": "http://localhost:6732",
-    "targetBuffer": 6,
-    "batchSize": 3,
-    "tzAmount": 2,
+    "funderPKH": "tz1bRt6Lo9KRNEUF9voCwkjR2pkMU9xJuYMB",
+    "remoteSignerUrl": "http://signatory:6732",
+    "targetBuffer": 100,
+    "batchSize": 100,
+    "tzAmount": 10,
     "autoRefillDurationMS": 30000,
-    "rpcUrl": "https://ghostnet.ecadinfra.com/",
+    "rpcUrl": "http://ecad-ghostnet-rolling.i.tez.ie:8732",
     "redisListName": "ephemeral-keys:ghostnet:address-pool"
+  },
+  "ephemeral-keys-mumbainet": {
+    "funderPKH": "tz1bRt6Lo9KRNEUF9voCwkjR2pkMU9xJuYMB",
+    "remoteSignerUrl": "http://signatory:6732",
+    "targetBuffer": 100,
+    "batchSize": 100,
+    "tzAmount": 10,
+    "autoRefillDurationMS": 30000,
+    "rpcUrl": "http://ecad-mumbainet-full.i.tez.ie:8732",
+    "redisListName": "ephemeral-keys:mumbainet:address-pool"
+  },
+  "ephemeral-keys-nairobinet": {
+    "funderPKH": "tz1bRt6Lo9KRNEUF9voCwkjR2pkMU9xJuYMB",
+    "remoteSignerUrl": "http://signatory:6732",
+    "targetBuffer": 100,
+    "batchSize": 100,
+    "tzAmount": 10,
+    "autoRefillDurationMS": 30000,
+    "rpcUrl": "http://ecad-nairobinet-full.i.tez.ie:8732",
+    "redisListName": "ephemeral-keys:nairobinet:address-pool"
   }
-}
+
+}

src/address-pool.ts

@@ -2,14 +2,14 @@ import { InMemorySigner } from '@taquito/signer';
 import { TezosToolkit } from '@taquito/taquito';
 import { b58cencode, prefix, Prefix } from '@taquito/utils';
 import { logger } from './logger';
-import { keyValue } from './storage/key-value';
 import { RedisQueue } from "./storage/redis-queue";
 import { RedisClient } from 'redis';
 import { RemoteSigner } from '@taquito/remote-signer';
 import { getKeyProducedCounter } from './metrics/keys-produced.counter';
 import { getKeyIssuedCounter } from './metrics/keys-issued.counter';
 
 const crypto = require('crypto');
+let lastJobLevel = -1;
 
 export interface PoolConfig {
   redisListName: string;
@@ -23,6 +23,10 @@ export interface PoolConfig {
   autoRefillDurationMS: number;
 }
 
+/* A pool of addresses maintained in a Redis list. If the config has autoRefillDurationMS,
+  then this address pool will automatically refill itself with new addresses. Otherwise, it will
+  only refill after after an address is popped.
+*/
 export class AddressPool {
 
   queue: RedisQueue;
@@ -83,9 +87,9 @@ export class AddressPool {
       const Tezos = await this.taquito()
       const { level } = await Tezos.rpc.getBlockHeader()
 
-      if (await this.queue.size() < this.config.targetBuffer && (!keyValue.has(this.config.lastJobKey) || keyValue.get(this.config.lastJobKey) < level)) {
+      if (await this.queue.size() < this.config.targetBuffer && lastJobLevel < level) {
         this.logger.info('Generating new keys...', { level })
-        keyValue.put(this.config.lastJobKey, level);
+        lastJobLevel = level;
         const dests: { key: string, pkh: string }[] = [];
 
         for (let i = 0; i < this.config.batchSize; i++) {

src/config.ts

@@ -23,7 +23,7 @@ let faucetKey = {
   "email": "[email protected]"
 }
 
-export class Config {
+export class EnvironmentConfig {
   faucetKey() {
     return faucetKey;
   }
@@ -42,4 +42,4 @@ export class Config {
 }
 
 
-export const config = new Config();
+export const environmentConfig = new EnvironmentConfig();

src/ephemeral/ephemeral-keys.ts

@@ -7,7 +7,7 @@ import { v4 as uuidv4 } from 'uuid';
 
 export interface EphemeralKeyConfig {
   expire: number,
-  maxAmount: number,
+  minBalanceInTez: number,
 }
 
 export class EphemeralKeyStore {
@@ -22,13 +22,17 @@ export class EphemeralKeyStore {
     }
   }
 
+  public getMinBalance() {
+    return this.config.minBalanceInTez;
+  }
+
   public getRPC() {
     return this.pool.getRPC();
   }
 
   async recycle(id: string) {
     const { secret, amount } = await this.get(id);
-    if (BigInt(amount) < BigInt(this.config.maxAmount)) {
+    if (BigInt(amount) < BigInt(this.config.minBalanceInTez * 1000000)) {
       logger.info('Discarding key', { key: secret, id, amount })
     } else {
       logger.info('Recycling key', { key: secret, id, amount })
@@ -47,7 +51,7 @@ export class EphemeralKeyStore {
     const pkh = await new InMemorySigner(secretKey).publicKeyHash()
     const Tezos = await this.pool.taquito();
     const balance = await Tezos.tz.getBalance(pkh)
-    const allowedAmount = balance.minus(this.config.maxAmount * 1000000).toNumber();
+    const allowedAmount = balance.minus(this.config.minBalanceInTez * 1000000).toNumber();
 
     const timeout = this.config.expire;
     this.client.set(`${this.id}:${id}:expire`, "", 'EX', timeout)

src/handlers/sign.ts

@@ -52,12 +52,12 @@ export const sign = async (req: Request, res: Response) => {
 
   const id = req.params.id;
 
-  const { secret, amount } = await ephemeralPool.get(id);
+  const { secret } = await ephemeralPool.get(id);
+  const minBalance = ephemeralPool.getMinBalance();
 
   const signer = new InMemorySigner(secret);
 
   const key = await signer.publicKeyHash();
-  const allowed = Number(amount);
   const balance = await Tezos.tz.getBalance(key)
   const bytes = JSON.parse(req.body);
 
@@ -98,7 +98,7 @@ export const sign = async (req: Request, res: Response) => {
       if (content.kind === 'transaction' || content.kind === 'origination' || content.kind === 'reveal') {
         allowedOperation.push(content as any);
       } else {
-        childLogger.debug('Signing request denied', {
+        childLogger.warn('Signing request denied', {
           reason: 'Kind not allowed',
           kindRequested: content.kind
         })
@@ -120,11 +120,11 @@ export const sign = async (req: Request, res: Response) => {
 
 
     // Check if balance allowed is higher than balance updates
-    if (balance.plus(balanceChange).lt(allowed)) {
-      childLogger.debug('Signing request denied', {
+    if (balance.plus(balanceChange).lt(100000)) {
+      childLogger.warn('Signing request denied', {
         reason: 'Not enough balance',
         amountRequest: balanceChange.toString(),
-        amountAllowed: allowed.toString()
+        currentBalance: balance,
       })
       res.status(403).send('Not enough balance');
       return

src/index.ts

@@ -1,5 +1,5 @@
 import { RedisClient } from "redis";
-import { config } from "./config";
+import { EnvironmentConfig, environmentConfig } from "./config";
 import { count, popKeys } from "./handlers/keys";
 import { logger } from "./logger";
 import { pools } from "./pools";
@@ -58,15 +58,15 @@ app.get('/:network/ephemeral/:id/keys/:key', middlewareLogger((req: any, res: an
 app.post('/:network/ephemeral/:id/keys/:key', middlewareLogger((req: any, res: any) => sign(req, res))) 
 
 export const client: RedisClient = redis.createClient({
-  host: config.redisHost,
-  port: config.redisPort,
-  password: config.redisPassword
+  host: environmentConfig.redisHost,
+  port: environmentConfig.redisPort,
+  password: environmentConfig.redisPassword
 });
 
 export const pubSub: RedisClient = redis.createClient({
-  host: config.redisHost,
-  port: config.redisPort,
-  password: config.redisPassword
+  host: environmentConfig.redisHost,
+  port: environmentConfig.redisPort,
+  password: environmentConfig.redisPassword
 });
 
 const ready = async () => {

src/logger.ts

@@ -1,4 +1,5 @@
-const winston = require('winston');
+import winston from "winston";
+
 const LEVEL = Symbol.for('level');
 
 /**

src/pools.ts

@@ -1,7 +1,7 @@
 import { AddressPool } from "./address-pool";
 import { RedisClient } from "redis";
 import { readFileSync } from 'fs';
-import { config } from "./config";
+import { environmentConfig } from "./config";
 import { getKeyPoolSizeGauge } from "./metrics/key_pool_size.gauge";
 import { getFundingAccountGauge } from "./metrics/funding_account_balance.gauge";
 import { EphemeralKeyStore } from "./ephemeral/ephemeral-keys";
@@ -24,13 +24,12 @@ export class Pools {
   init(redis: RedisClient) {
     const content = JSON.parse(readFileSync("pools-config.json").toString('utf8'))
     Object.keys(content).forEach((poolKey) => {
-      const pool = new AddressPool(poolKey, {
-        ...config,
+      const config = {
+        ...environmentConfig,
         // TODO validate schema
         ...content[poolKey]
-      },
-        redis
-      )
+      };
+      const pool = new AddressPool(poolKey, config, redis);
       pool.init();
       this.pools.set(poolKey, pool)