Feat/272 split sdk and postman (#381)

* feat: split postman and sdk

* fix: update postman client and sendMessage script

* fix: clean the sdk

* fix: update sdk dependencies

* fix: remove .env.sample file

* fix: remove testing helpers from the build

* fix: update gas provider in linea sdk and update postman client

* fix: update postman dependencies

* fix: update postman dockerfile and fix tests imports and types

* fix: remove unused code in the sdk + move typechain folder

* fix: remove unused code + fix imports in postman

* fix: pnpm lock file issue

* fix: import issue

* fix: case sensitive file issue

* fix: update sdk fees options and update exports

* fix: remove postman unused code and adjust imports and tests

* fix: update contracts abis + clean error parsing

* fix: update postman based on new SDk changes

* add readme + remove unused interface in postman

* fix: rename Base.ts file to BaseError.ts

* fix: rename Base.ts file to BaseError.ts in postman

* chore: update readme for the postman

* fix: rename maxFeePerGas to maxFeePerGasCap

* fix: update DefaultGasProvider fees check

* fix: default gas provider test issue

* fix: update main ci filter

* fix: issue in default gas provider

.dockerignore

@@ -0,0 +1,5 @@
+**/node_modules
+**/typechain
+**/tsconfig.build.tsbuildinfo
+**/coverage
+**/dist

.github/workflows/main.yml

@@ -52,7 +52,9 @@ jobs:
               - 'docker/compose-local-dev.overrides.yml'
               - 'docker/compose-local-dev-traces-v2.overrides.yml'
             postman:
+              - 'postman/**'
               - 'sdk/**'
+              - 'ts-libs/linea-native-libs/**'
               - '.github/workflows/postman-*.yml'
               - '.github/workflows/build-and-publish.yml'
               - '.github/workflows/main.yml'
@@ -193,7 +195,7 @@ jobs:
     secrets: inherit
 
   run-e2e-tests-geth-tracing:
-    needs: [ store-image-name-and-tags, docker-build, manual-docker-build-and-e2e-tests ]
+    needs: [ store-image-name-and-tags, docker-build, manual-docker-build-and-e2e-tests, filter-commit-changes ]
     # Make this execute for has-changes-requiring-build == 'false' so that we can get to the required job @ which is in reuse-run-e2e-tests.yml
     if: ${{ always() && needs.filter-commit-changes.outputs.has-changes-requiring-build == 'false' || needs.docker-build.result == 'success' }}
     concurrency:

.github/workflows/postman-build-and-publish.yml

@@ -93,7 +93,7 @@ jobs:
         if: ${{ env.PUSH_IMAGE == 'false' }}
         with:
           context: ./
-          file: ./sdk/Dockerfile
+          file: ./postman/Dockerfile
           platforms: linux/amd64
           load: true
           push: false
@@ -116,7 +116,7 @@ jobs:
         if: ${{ env.PUSH_IMAGE == 'true' || github.event_name == 'workflow_dispatch' }}
         with:
           context: ./
-          file: ./sdk/Dockerfile
+          file: ./postman/Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ env.TAGS }}

.github/workflows/postman-testing.yml

@@ -24,4 +24,5 @@ jobs:
         run: |
           pnpm run -F ./ts-libs/linea-native-libs build;
           pnpm run -F ./sdk build;
+          pnpm run -F ./postman test;
           pnpm run -F ./sdk test;

pnpm-workspace.yaml

@@ -2,6 +2,7 @@ packages:
   - 'contracts/**'
   - 'e2e/**'
   - 'sdk/**'
+  - 'postman/**'
   - 'operations/**'
   - 'bridge-ui/**'
   - 'ts-libs/**'

postman/.env.sample

@@ -0,0 +1,44 @@
+L1_RPC_URL=http://localhost:8445
+L1_CONTRACT_ADDRESS=0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9
+# Only use the following PRIVATE KEY for testing
+L1_SIGNER_PRIVATE_KEY=0x8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba
+L1_LISTENER_INTERVAL=2000
+# L1_LISTENER_INITIAL_FROM_BLOCK=0
+L1_LISTENER_BLOCK_CONFIRMATION=1
+L1_MAX_BLOCKS_TO_FETCH_LOGS=1000
+L1_MAX_GAS_FEE_ENFORCED=false
+L2_RPC_URL=http://localhost:8845
+L2_CONTRACT_ADDRESS=0xe537D669CA013d86EBeF1D64e40fC74CADC91987
+# Only use the following PRIVATE KEY for testing
+L2_SIGNER_PRIVATE_KEY=0xfcf854e0a0bc6fd7e97d7050e61a362c915cecd6767a32267b22e8b7af572e58 
+L2_LISTENER_INTERVAL=2000
+# L2_LISTENER_INITIAL_FROM_BLOCK=0
+L2_LISTENER_BLOCK_CONFIRMATION=0
+L2_MAX_BLOCKS_TO_FETCH_LOGS=1000
+L2_MESSAGE_TREE_DEPTH=5
+L2_MAX_GAS_FEE_ENFORCED=false
+MESSAGE_SUBMISSION_TIMEOUT=300000
+MAX_FETCH_MESSAGES_FROM_DB=1000
+MAX_NONCE_DIFF=10000
+MAX_FEE_PER_GAS_CAP=100000000000
+GAS_ESTIMATION_PERCENTILE=50
+PROFIT_MARGIN=0.0
+MAX_NUMBER_OF_RETRIES=100
+RETRY_DELAY_IN_SECONDS=30
+MAX_CLAIM_GAS_LIMIT=2560000
+MAX_TX_RETRIES=20
+L1_L2_EOA_ENABLED=true
+L1_L2_CALLDATA_ENABLED=true
+L1_L2_AUTO_CLAIM_ENABLED=true
+L2_L1_EOA_ENABLED=true
+L2_L1_CALLDATA_ENABLED=true
+L2_L1_AUTO_CLAIM_ENABLED=true
+POSTGRES_HOST=127.0.0.1
+POSTGRES_PORT=5432
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=postman_db
+DB_CLEANER_ENABLED=false
+DB_CLEANING_INTERVAL=10000
+DB_DAYS_BEFORE_NOW_TO_DELETE=1
+ENABLE_LINEA_ESTIMATE_GAS=false

postman/.eslintignore

@@ -0,0 +1,3 @@
+dist
+node_modules
+typechain

postman/.eslintrc.js

@@ -0,0 +1,15 @@
+module.exports = {
+  extends: "../.eslintrc.js",
+  env: {
+    commonjs: true,
+    es2021: true,
+    node: true,
+    jest: true,
+  },
+  parserOptions: {
+    sourceType: "module",
+  },
+  rules: {
+    "prettier/prettier": "error",
+  },
+};

postman/.prettierignore

@@ -0,0 +1,3 @@
+dist
+node_modules
+typechain

postman/.prettierrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+    ...require('../.prettierrc.js'),
+};

sdk/Dockerfile → postman/Dockerfile

@@ -18,16 +18,19 @@ ARG NATIVE_LIBS_RELEASE_TAG
 ENV NATIVE_LIBS_RELEASE_TAG=${NATIVE_LIBS_RELEASE_TAG}
 
 COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.json ./
+
+COPY ./postman/package.json ./postman/package.json
 COPY ./sdk/package.json ./sdk/package.json
 COPY ./ts-libs/linea-native-libs/package.json ./ts-libs/linea-native-libs/package.json
 
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile --prefer-offline --ignore-scripts
 
+COPY ./postman ./postman
 COPY ./sdk ./sdk
 COPY ts-libs/linea-native-libs ./ts-libs/linea-native-libs
 
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm run build \
-    && pnpm deploy --filter=./sdk --prod ./prod/sdk
+    && pnpm deploy --filter=./postman --prod ./prod/postman
 
 FROM node:lts-slim AS production
 
@@ -37,6 +40,6 @@ WORKDIR /usr/src/app
 
 USER node
 
-COPY --from=builder /usr/src/app/prod/sdk ./sdk
+COPY --from=builder /usr/src/app/prod/postman ./postman
 
-CMD [ "node", "./sdk/dist/scripts/runPostman.js" ]
+CMD [ "node", "./postman/dist/scripts/runPostman.js" ]

postman/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2023 Consensys Software Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

postman/README.md

@@ -0,0 +1,115 @@
+# Linea Postman Service
+
+The Linea Postman service is a component of the Linea blockchain infrastructure that facilitates cross-chain message delivery between Layer 1 (Ethereum) and Layer 2 (Linea).
+
+## Overview
+
+The Postman service monitors and processes messages between L1 and L2 chains, handling message submission, verification, and claiming. It operates as a Docker container and integrates with both L1 and L2 nodes.
+
+It offers the following key features:
+
+- Feature 1: Listening for message sent events on Ethereum and Linea
+- Feature 2: Listening for message hash anchoring events to check if a message is ready to be claimed
+- Feature 3: Automatic claiming of messages with a configurable retry mechanism
+- Feature 4: Checking receipt status for each transaction
+
+All messages are stored in a configurable Postgres DB.
+
+## Configuration
+
+### Environment Variables
+
+#### L1 Configuration
+- `L1_RPC_URL`: Ethereum node RPC endpoint
+- `L1_CONTRACT_ADDRESS`: Address of the LineaRollup contract on L1
+- `L1_SIGNER_PRIVATE_KEY`: Private key for L1 transactions
+- `L1_LISTENER_INTERVAL`: Block listening interval (ms)
+- `L1_LISTENER_INITIAL_FROM_BLOCK`: Starting block for event listening (optional)
+- `L1_LISTENER_BLOCK_CONFIRMATION`: Required block confirmations
+- `L1_MAX_BLOCKS_TO_FETCH_LOGS`: Maximum blocks to fetch in one request
+- `L1_MAX_GAS_FEE_ENFORCED`: Enable/disable gas fee enforcement
+
+#### L2 Configuration
+- `L2_RPC_URL`: Linea node RPC endpoint
+- `L2_CONTRACT_ADDRESS`: Address of the L2MessageService contract on L2
+- `L2_SIGNER_PRIVATE_KEY`: Private key for L2 transactions
+- `L2_LISTENER_INTERVAL`: Block listening interval (ms)
+- `L2_LISTENER_INITIAL_FROM_BLOCK`: Starting block for event listening (optional)
+- `L2_LISTENER_BLOCK_CONFIRMATION`: Required block confirmations
+- `L2_MAX_BLOCKS_TO_FETCH_LOGS`: Maximum blocks to fetch in one request
+- `L2_MAX_GAS_FEE_ENFORCED`: Enable/disable gas fee enforcement
+- `L2_MESSAGE_TREE_DEPTH`: Depth of the message Merkle tree
+
+#### Message Processing
+- `MESSAGE_SUBMISSION_TIMEOUT`: Timeout for message submission (ms)
+- `MAX_FETCH_MESSAGES_FROM_DB`: Maximum messages to fetch from database
+- `MAX_NONCE_DIFF`: Maximum allowed nonce difference between the DB and the chain
+- `MAX_FEE_PER_GAS_CAP`: Maximum gas fee cap
+- `GAS_ESTIMATION_PERCENTILE`: Gas estimation percentile
+- `PROFIT_MARGIN`: Profit margin for gas fees
+- `MAX_NUMBER_OF_RETRIES`: Maximum retry attempts
+- `RETRY_DELAY_IN_SECONDS`: Delay between retries
+- `MAX_CLAIM_GAS_LIMIT`: Maximum gas limit for claim transactions
+
+#### Feature Flags
+- `L1_L2_EOA_ENABLED`: Enable L1->L2 EOA messages
+- `L1_L2_CALLDATA_ENABLED`: Enable L1->L2 calldata messages
+- `L1_L2_AUTO_CLAIM_ENABLED`: Enable auto-claiming for L1->L2 messages
+- `L2_L1_EOA_ENABLED`: Enable L2->L1 EOA messages
+- `L2_L1_CALLDATA_ENABLED`: Enable L2->L1 calldata messages
+- `L2_L1_AUTO_CLAIM_ENABLED`: Enable auto-claiming for L2->L1 messages
+- `ENABLE_LINEA_ESTIMATE_GAS`: Enable `linea_estimateGas`endpoint usage for L2 chain gas fees estimation
+- `DB_CLEANER_ENABLED`: Enable DB cleaning to delete old claimed messages
+
+#### DB cleaning
+- `DB_CLEANING_INTERVAL`: DB cleaning polling interval (ms)
+- `DB_DAYS_BEFORE_NOW_TO_DELETE`: Number of days to retain messages in the database before deletion. Messages older than this number of days will be automatically cleaned up if they are in a final state (CLAIMED_SUCCESS, CLAIMED_REVERTED, EXCLUDED, or ZERO_FEE)
+
+#### Database Configuration
+- `POSTGRES_HOST`: PostgreSQL host
+- `POSTGRES_PORT`: PostgreSQL port
+- `POSTGRES_USER`: Database user
+- `POSTGRES_PASSWORD`: Database password
+- `POSTGRES_DB`: Database name
+
+## Development
+
+### Running
+
+#### Start the docker local stack
+
+From the root folder, run the following command:
+```bash
+make fresh-start-all
+```
+
+Stop the postman docker container manually.
+
+#### Run the postman locally:
+
+From the postman folder run the following commands:
+
+```bash
+# Create a new .env file
+cp .env.sample .env
+
+# Run the postman
+ts-node scripts/runPostman.ts
+```
+
+### Building
+```bash
+# Build the Postman service
+pnpm run build
+```
+
+### Testing
+
+```bash
+# Run unit tests
+pnpm run test
+```
+
+## License
+
+This package is licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for more information.

postman/jest.config.js

@@ -0,0 +1,25 @@
+module.exports = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  rootDir: ".",
+  testRegex: "test.ts$",
+  verbose: true,
+  collectCoverage: true,
+  collectCoverageFrom: ["src/**/*.ts"],
+  coverageReporters: ["html", "lcov", "text"],
+  testPathIgnorePatterns: [
+    "src/clients/blockchain/typechain",
+    "src/application/postman/persistence/migrations/",
+    "src/application/postman/persistence/repositories/",
+    "src/index.ts",
+    "src/utils/WinstonLogger.ts",
+  ],
+  coveragePathIgnorePatterns: [
+    "src/clients/blockchain/typechain",
+    "src/application/postman/persistence/migrations/",
+    "src/application/postman/persistence/repositories/",
+    "src/index.ts",
+    "src/utils/WinstonLogger.ts",
+    "src/utils/testing/",
+  ],
+};

postman/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@consensys/linea-postman",
+  "version": "1.0.0",
+  "author": "Consensys Software Inc.",
+  "license": "Apache-2.0",
+  "description": "",
+  "main": "dist/src/index.js",
+  "types": "dist/src/index.d.ts",
+  "scripts": {
+    "lint:ts": "npx eslint '**/*.ts'",
+    "lint:ts:fix": "npx eslint --fix '**/*.ts'",
+    "prettier": "prettier -c '**/*.ts'",
+    "prettier:fix": "prettier -w '**/*.ts'",
+    "clean": "rimraf dist node_modules coverage tsconfig.build.tsbuildinfo",
+    "build": "tsc -p tsconfig.build.json",
+    "build:runSdk": "tsc ./scripts/runSdk.ts",
+    "test": "npx jest --bail --detectOpenHandles --forceExit",
+    "lint:fix": "pnpm run lint:ts:fix && pnpm run prettier:fix"
+  },
+  "dependencies": {
+    "@consensys/linea-native-libs": "workspace:*",
+    "@consensys/linea-sdk": "workspace:*",
+    "better-sqlite3": "11.6.0",
+    "class-validator": "0.14.1",
+    "dotenv": "16.4.5",
+    "ethers": "6.13.4",
+    "pg": "8.13.1",
+    "typeorm": "0.3.20",
+    "typeorm-naming-strategies": "4.1.0",
+    "winston": "3.17.0"
+  },
+  "devDependencies": {
+    "@jest/globals": "29.7.0",
+    "@types/jest": "29.5.14",
+    "@types/yargs": "17.0.33",
+    "jest": "29.7.0",
+    "jest-mock-extended": "3.0.5",
+    "ts-jest": "29.2.5",
+    "yargs": "17.7.2"
+  },
+  "files": [
+    "dist/**/*"
+  ]
+}

sdk/scripts/runPostman.ts → postman/scripts/runPostman.ts

@@ -32,7 +32,7 @@ async function main() {
           ? parseInt(process.env.MESSAGE_SUBMISSION_TIMEOUT)
           : undefined,
         maxNonceDiff: process.env.MAX_NONCE_DIFF ? parseInt(process.env.MAX_NONCE_DIFF) : undefined,
-        maxFeePerGas: process.env.MAX_FEE_PER_GAS ? BigInt(process.env.MAX_FEE_PER_GAS) : undefined,
+        maxFeePerGasCap: process.env.MAX_FEE_PER_GAS_CAP ? BigInt(process.env.MAX_FEE_PER_GAS_CAP) : undefined,
         gasEstimationPercentile: process.env.GAS_ESTIMATION_PERCENTILE
           ? parseInt(process.env.GAS_ESTIMATION_PERCENTILE)
           : undefined,
@@ -72,7 +72,7 @@ async function main() {
           ? parseInt(process.env.MESSAGE_SUBMISSION_TIMEOUT)
           : undefined,
         maxNonceDiff: process.env.MAX_NONCE_DIFF ? parseInt(process.env.MAX_NONCE_DIFF) : undefined,
-        maxFeePerGas: process.env.MAX_FEE_PER_GAS ? BigInt(process.env.MAX_FEE_PER_GAS) : undefined,
+        maxFeePerGasCap: process.env.MAX_FEE_PER_GAS_CAP ? BigInt(process.env.MAX_FEE_PER_GAS_CAP) : undefined,
         gasEstimationPercentile: process.env.GAS_ESTIMATION_PERCENTILE
           ? parseInt(process.env.GAS_ESTIMATION_PERCENTILE)
           : undefined,