diff --git a/.github/workflows/filecoin-calibration.yml b/.github/workflows/filecoin-calibration.yml new file mode 100644 index 00000000..8198c3ea --- /dev/null +++ b/.github/workflows/filecoin-calibration.yml @@ -0,0 +1,82 @@ +name: Filecoin Calibration + +on: + push: + paths: ['services/filecoin/**', 'backend/**', 'frontend/**', '.github/workflows/filecoin-calibration.yml'] + pull_request: + paths: ['services/filecoin/**', 'backend/**', 'frontend/**', '.github/workflows/filecoin-calibration.yml'] + workflow_dispatch: + +permissions: + contents: read + +jobs: + frontend: + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.4.4' + - run: bun install --frozen-lockfile + - run: bun run test:filecoin + - run: bun run test:proxy + - run: bun x tsc --noEmit + sdk: + runs-on: ubuntu-latest + defaults: + run: + working-directory: services/filecoin + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: services/filecoin/package-lock.json + - run: npm ci --ignore-scripts + - run: npm test + - run: npm run check + backend: + runs-on: ubuntu-latest + services: + mongodb: + image: mongo:7 + ports: ['27017:27017'] + options: >- + --health-cmd "mongosh --quiet --eval 'db.runCommand({ping:1})'" + --health-interval 10s --health-timeout 5s --health-retries 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: actions/setup-node@v4 + with: + node-version: '22' + - run: npm ci --ignore-scripts --prefix services/filecoin + - name: Install integration test dependencies + run: >- + python -m pip install pytest ruff 'pydantic>=2.8,<3' 'httpx>=0.27,<1' + 'beanie>=1.27,<3' 'pymongo>=4.11,<5' 'web3>=7.6,<8' + 'fastapi>=0.115,<1' pyyaml python-dotenv 'pyunormalize<18' + 'py-cid==0.5.0' 'libp2p==0.6.0' 'python-multipart==0.0.32' + 'qiskit==2.3.0' 'qiskit-aer==0.17.2' 'qiskit-algorithms==0.4.0' + - run: python -m pip install --no-deps -e backend + - name: Lint Filecoin integration + run: >- + ruff check backend/src/quantum_backend_v2/artifacts/filecoin.py + backend/src/quantum_backend_v2/application/filecoin_*.py + backend/src/quantum_backend_v2/api/routers/filecoin.py + backend/scripts/filecoin.py backend/tests/unit/test_filecoin*.py + - name: Filecoin and adjacent regressions + env: + QMESH_TEST_MONGODB_URI: mongodb://localhost:27017 + run: >- + python -m pytest backend/tests/unit/test_filecoin*.py + backend/tests/unit/test_peer_tasks.py backend/tests/unit/test_provenance_job.py + backend/tests/unit/test_provenance_storage_check.py + backend/tests/unit/test_financial_api.py backend/tests/unit/test_finance_provenance.py -q diff --git a/backend/docs/evidence/filecoin-calibration-20261111/README.md b/backend/docs/evidence/filecoin-calibration-20261111/README.md new file mode 100644 index 00000000..df16ed68 --- /dev/null +++ b/backend/docs/evidence/filecoin-calibration-20261111/README.md @@ -0,0 +1,40 @@ +# Live Filecoin Calibration evidence + +Implementation: [PR #145](https://github.com/QMeshPy/distributed-quantum/pull/145), [issue #144](https://github.com/QMeshPy/distributed-quantum/issues/144). + +Five public objects were confirmed in dataset **52518**, provider **9**, using faucet test FIL and test USDFC. No custom contract was deployed. Every object was downloaded in a fresh, key-free verification process and checked against its PieceCID, SHA256 and byte length; package/dataset/checkpoint schemas were also validated. + +| Object | Bytes | Piece ID | Evidence | +| --- | ---: | ---: | --- | +| Bell experiment + NEAR provenance | 7,292 | 0 | [Transaction](https://beryx.io/fil/calibration/txs/0xece0512aca548d03f5222406c305e389aa9ac565a5f741f3bdf36810e87ad8b2) / [receipt](bell-receipt.json) | +| Synthetic finance CSV | 847 | 1 | [Transaction](https://beryx.io/fil/calibration/txs/0xe895434a95b4ca30c653278236016c5a1358f17f9507dbc599a5c186dda2529f) / [receipt](finance-dataset-receipt.json) | +| NCN toy Hamiltonian dataset | 1,106 | 2 | [Transaction](https://beryx.io/fil/calibration/txs/0x77e79da32215090106677d56d85dd5590eefbc84faf2552637d4189ef15404a3) / [receipt](pharma-dataset-receipt.json) | +| Completed VQE task checkpoint | 448 | 3 | [Transaction](https://beryx.io/fil/calibration/txs/0xeec3df365fb20c6f6d4bfabd89df94e379e85502dfb50a6c25b2d325338abc8f) / [receipt](pharma-checkpoint-receipt.json) | +| Finance result, input CSV and circuit archive | 20,499 | 4 | [Transaction](https://beryx.io/fil/calibration/txs/0xedc6401aa5aa80085aafb3298baddbb3c504da45f4fe233bf3132efb44859449) / [receipt](finance-package-receipt.json) | + +[Machine-readable observations](retrieval-and-status.json) record live registration for all five pieces. **PDP was pending for the current period** at these observations; successful retrieval is independently verified and is not presented as a completed PDP proof. This demonstration used one provider, not replicated storage. + +## What ran + +- **Bell archive:** the downloaded package's input, result, execution plan and bundle commitments passed verification, as did both existing NEAR testnet anchors. See [verification report](bell-near-verification.json). +- **Pharma:** the actual local VQE handler consumed the Filecoin-hosted NCN/STO-3G toy Hamiltonian (four qubits, two active electrons). It completed 35 evaluations at **-148.33704862401328 Ha** and published a completed-task checkpoint. A separate process retrieved the dataset and checkpoint, restored the identical outcome hash, and made **zero compute calls and zero uploads**. This reused an earlier Hamiltonian; it was not a new molecule-generation or safety-validation experiment. +- **Finance:** the real portfolio builder consumed the Filecoin-hosted **synthetic** CSV (13 rows, four assets). It selected SYN_A and SYN_B with objective **0.554655045676**, matching exact enumeration across six feasible portfolios. Its inputs, result and QASM circuit were archived together. These are local CPU demonstrations, not live mesh or market-performance measurements. + +[Summary and result hashes](summary.json) distinguish execution, storage and provenance evidence. + +## Independently retrieve + +Install the backend and `npm ci --ignore-scripts --prefix services/filecoin`, then run from the repository root (no wallet key required): + +```sh +python backend/scripts/filecoin.py download backend/docs/evidence/filecoin-calibration-20261111/finance-package-receipt.json finance-package.json +python backend/scripts/filecoin.py status backend/docs/evidence/filecoin-calibration-20261111/pharma-checkpoint-receipt.json +``` + +For the Bell archive's independent NEAR check, set `QB2_NEAR_REGISTRY_CONTRACT` and `QB2_NEAR_REGISTRY_PUBLISHER` to `qmesh-provenance-20261110.testnet`, then run `filecoin.py verify` with `bell-receipt.json`. Status is a fresh observation and can change as funding and provider health change. + +## Funding and operational boundaries + +The active demo wallet is `0xA63A33e52589c761F90ce46F7c86aeEe8644112f`. Its [one-test-USDFC deposit](https://beryx.io/fil/calibration/txs/0xbee3642018b40a0fdbd4708d6e3a4f26835123a5a0e05916ff94079235986e9a) and [finite operator approval](https://beryx.io/fil/calibration/txs/0xde85d6ed9a228071593c37a3fb8caf2c4a1ec8bed2fbf26bcab8a560077acc4e) used existing faucet funds. Approval limits were 0.25 test USDFC/month, one test USDFC lockup and the contract-required 30-day lockup period. A seven-day runway target does not schedule deletion. Earlier uncertain setup activity was quarantined by revoking the old wallet's operator approval; it is excluded from successful-object counts. + +The production UI built successfully and its interactions were exercised with an explicitly isolated account/API fixture. The real email-OTP session could not be tested because the available computer-use runtime failed to initialize. Hosted use still requires operator configuration of MongoDB, authentication and a funded agent key as documented in [setup](../../filecoin-calibration.md). diff --git a/backend/docs/evidence/filecoin-calibration-20261111/bell-near-verification.json b/backend/docs/evidence/filecoin-calibration-20261111/bell-near-verification.json new file mode 100644 index 00000000..728c9b9a --- /dev/null +++ b/backend/docs/evidence/filecoin-calibration-20261111/bell-near-verification.json @@ -0,0 +1,87 @@ +{ + "bundle_id": "provenance-near-demo-20261110T170703Z-bell-pair", + "status": "verified", + "checks": [ + { + "kind": "hash", + "name": "bundle_hash", + "status": "passed", + "message": "Commitment matches", + "reference": null, + "url": null + }, + { + "kind": "hash", + "name": "input_hash", + "status": "passed", + "message": "Commitment matches", + "reference": null, + "url": null + }, + { + "kind": "hash", + "name": "plan_id", + "status": "passed", + "message": "Commitment matches", + "reference": null, + "url": null + }, + { + "kind": "hash", + "name": "result_hash", + "status": "passed", + "message": "Commitment matches", + "reference": null, + "url": null + }, + { + "kind": "hash", + "name": "fragment_hashes", + "status": "passed", + "message": "Commitment matches", + "reference": null, + "url": null + }, + { + "kind": "storage", + "name": "storage_content", + "status": "not_applicable", + "message": "No storage references recorded", + "reference": null, + "url": null + }, + { + "kind": "anchor", + "name": "anchor_proof", + "status": "passed", + "message": "Merkle inclusion matches", + "reference": "near:testnet:FbM5wgccrreiuA9UDc1Dzyn34qwNBoWu6UjvpvNQ7vvX", + "url": null + }, + { + "kind": "anchor", + "name": "anchor_transaction", + "status": "passed", + "message": "Confirmed anchor root matches", + "reference": "near:testnet:FbM5wgccrreiuA9UDc1Dzyn34qwNBoWu6UjvpvNQ7vvX", + "url": "https://testnet.nearblocks.io/txns/FbM5wgccrreiuA9UDc1Dzyn34qwNBoWu6UjvpvNQ7vvX" + }, + { + "kind": "anchor", + "name": "anchor_proof", + "status": "passed", + "message": "Merkle inclusion matches", + "reference": "near:testnet:Cge8aNb4w3HXH61roPFLY9qyYwoSfHm8kvuao2rQeDax", + "url": null + }, + { + "kind": "anchor", + "name": "anchor_transaction", + "status": "passed", + "message": "Confirmed anchor root matches", + "reference": "near:testnet:Cge8aNb4w3HXH61roPFLY9qyYwoSfHm8kvuao2rQeDax", + "url": "https://testnet.nearblocks.io/txns/Cge8aNb4w3HXH61roPFLY9qyYwoSfHm8kvuao2rQeDax" + } + ], + "checked_at": "2026-11-10T21:31:03.625300Z" +} diff --git a/backend/docs/evidence/filecoin-calibration-20261111/bell-receipt.json b/backend/docs/evidence/filecoin-calibration-20261111/bell-receipt.json new file mode 100644 index 00000000..57f646da --- /dev/null +++ b/backend/docs/evidence/filecoin-calibration-20261111/bell-receipt.json @@ -0,0 +1,22 @@ +{ + "network": "calibration", + "chain_id": 314159, + "piece_cid": "bafkzcibdyqdarmlrnos4iu6tpqjyr3b7ln4vwxoo35qzumgi75oemx6qefdoqmzc", + "sha256": "abd163263016cdbdd5c05aae5b40bb0fadea0d8f7f6046734d27ce88875cba39", + "size_bytes": 7292, + "complete": true, + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "0", + "transaction_hash": "0xece0512aca548d03f5222406c305e389aa9ac565a5f741f3bdf36810e87ad8b2", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibdyqdarmlrnos4iu6tpqjyr3b7ln4vwxoo35qzumgi75oemx6qefdoqmzc" + } + ], + "transaction_hashes": [ + "0xece0512aca548d03f5222406c305e389aa9ac565a5f741f3bdf36810e87ad8b2" + ], + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" +} diff --git a/backend/docs/evidence/filecoin-calibration-20261111/finance-dataset-receipt.json b/backend/docs/evidence/filecoin-calibration-20261111/finance-dataset-receipt.json new file mode 100644 index 00000000..42a9c957 --- /dev/null +++ b/backend/docs/evidence/filecoin-calibration-20261111/finance-dataset-receipt.json @@ -0,0 +1,22 @@ +{ + "network": "calibration", + "chain_id": 314159, + "piece_cid": "bafkzcibdveaqk6e5peub6bsn2n7ds2n6xgnwkbsgdkge6zdpa2juzamgcgwh3gb4", + "sha256": "05881c4397aabfdf25f0e5211af280b6a582d11bc7e611095c73f6ccf47f2393", + "size_bytes": 847, + "complete": true, + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "1", + "transaction_hash": "0xe895434a95b4ca30c653278236016c5a1358f17f9507dbc599a5c186dda2529f", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibdveaqk6e5peub6bsn2n7ds2n6xgnwkbsgdkge6zdpa2juzamgcgwh3gb4" + } + ], + "transaction_hashes": [ + "0xe895434a95b4ca30c653278236016c5a1358f17f9507dbc599a5c186dda2529f" + ], + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" +} diff --git a/backend/docs/evidence/filecoin-calibration-20261111/finance-package-receipt.json b/backend/docs/evidence/filecoin-calibration-20261111/finance-package-receipt.json new file mode 100644 index 00000000..0beda7a5 --- /dev/null +++ b/backend/docs/evidence/filecoin-calibration-20261111/finance-package-receipt.json @@ -0,0 +1,22 @@ +{ + "network": "calibration", + "chain_id": 314159, + "piece_cid": "bafkzcibd5voqunvk3t67szzdlt67vl4ipgjlh5vlamrjefkkuuxxypqrjcaiiwba", + "sha256": "06145b9339ca58698d9977f6ef6fc2c5675e1004de20ff6c9a8c167bec1a5000", + "size_bytes": 20499, + "complete": true, + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "4", + "transaction_hash": "0xedc6401aa5aa80085aafb3298baddbb3c504da45f4fe233bf3132efb44859449", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibd5voqunvk3t67szzdlt67vl4ipgjlh5vlamrjefkkuuxxypqrjcaiiwba" + } + ], + "transaction_hashes": [ + "0xedc6401aa5aa80085aafb3298baddbb3c504da45f4fe233bf3132efb44859449" + ], + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" +} diff --git a/backend/docs/evidence/filecoin-calibration-20261111/pharma-checkpoint-receipt.json b/backend/docs/evidence/filecoin-calibration-20261111/pharma-checkpoint-receipt.json new file mode 100644 index 00000000..b226016e --- /dev/null +++ b/backend/docs/evidence/filecoin-calibration-20261111/pharma-checkpoint-receipt.json @@ -0,0 +1,22 @@ +{ + "network": "calibration", + "chain_id": 314159, + "piece_cid": "bafkzcibchqcjsv7tvkjyon2znkusp6hywkruuk2vfoia37pgacdj2ryrg3kjoky", + "sha256": "66bbd36b5684e404e0c242725cc4fbfa52b4db17a67be0f64aa19e2f06306cd1", + "size_bytes": 448, + "complete": true, + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "3", + "transaction_hash": "0xeec3df365fb20c6f6d4bfabd89df94e379e85502dfb50a6c25b2d325338abc8f", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibchqcjsv7tvkjyon2znkusp6hywkruuk2vfoia37pgacdj2ryrg3kjoky" + } + ], + "transaction_hashes": [ + "0xeec3df365fb20c6f6d4bfabd89df94e379e85502dfb50a6c25b2d325338abc8f" + ], + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" +} diff --git a/backend/docs/evidence/filecoin-calibration-20261111/pharma-dataset-receipt.json b/backend/docs/evidence/filecoin-calibration-20261111/pharma-dataset-receipt.json new file mode 100644 index 00000000..1978c1fe --- /dev/null +++ b/backend/docs/evidence/filecoin-calibration-20261111/pharma-dataset-receipt.json @@ -0,0 +1,22 @@ +{ + "network": "calibration", + "chain_id": 314159, + "piece_cid": "bafkzcibdtydqnjm3hkbkx6yxryhhl2dug2ny4ansdaqydteemvmboba4iu5477yk", + "sha256": "eac417517674d3b53e73321e5a1f58034bc005e0876474f5ecf051af0e892f41", + "size_bytes": 1106, + "complete": true, + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "2", + "transaction_hash": "0x77e79da32215090106677d56d85dd5590eefbc84faf2552637d4189ef15404a3", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibdtydqnjm3hkbkx6yxryhhl2dug2ny4ansdaqydteemvmboba4iu5477yk" + } + ], + "transaction_hashes": [ + "0x77e79da32215090106677d56d85dd5590eefbc84faf2552637d4189ef15404a3" + ], + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" +} diff --git a/backend/docs/evidence/filecoin-calibration-20261111/retrieval-and-status.json b/backend/docs/evidence/filecoin-calibration-20261111/retrieval-and-status.json new file mode 100644 index 00000000..a49b0a37 --- /dev/null +++ b/backend/docs/evidence/filecoin-calibration-20261111/retrieval-and-status.json @@ -0,0 +1,242 @@ +{ + "bell": { + "receipt": { + "network": "calibration", + "chain_id": 314159, + "piece_cid": "bafkzcibdyqdarmlrnos4iu6tpqjyr3b7ln4vwxoo35qzumgi75oemx6qefdoqmzc", + "sha256": "abd163263016cdbdd5c05aae5b40bb0fadea0d8f7f6046734d27ce88875cba39", + "size_bytes": 7292, + "complete": true, + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "0", + "transaction_hash": "0xece0512aca548d03f5222406c305e389aa9ac565a5f741f3bdf36810e87ad8b2", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibdyqdarmlrnos4iu6tpqjyr3b7ln4vwxoo35qzumgi75oemx6qefdoqmzc" + } + ], + "transaction_hashes": [ + "0xece0512aca548d03f5222406c305e389aa9ac565a5f741f3bdf36810e87ad8b2" + ], + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" + }, + "retrieved_sha256": "abd163263016cdbdd5c05aae5b40bb0fadea0d8f7f6046734d27ce88875cba39", + "retrieval": "verified", + "possession": { + "network": "calibration", + "chain_id": 314159, + "observed_block": "4234005", + "observed_at": "2026-11-10T21:36:13.534Z", + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "0", + "transaction_hash": "0xece0512aca548d03f5222406c305e389aa9ac565a5f741f3bdf36810e87ad8b2", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibdyqdarmlrnos4iu6tpqjyr3b7ln4vwxoo35qzumgi75oemx6qefdoqmzc", + "registered": true, + "live": true, + "service_end_epoch": "0", + "next_challenge_epoch": "4234215", + "proof_status": "pending_this_period", + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" + } + ] + } + }, + "finance-dataset": { + "receipt": { + "network": "calibration", + "chain_id": 314159, + "piece_cid": "bafkzcibdveaqk6e5peub6bsn2n7ds2n6xgnwkbsgdkge6zdpa2juzamgcgwh3gb4", + "sha256": "05881c4397aabfdf25f0e5211af280b6a582d11bc7e611095c73f6ccf47f2393", + "size_bytes": 847, + "complete": true, + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "1", + "transaction_hash": "0xe895434a95b4ca30c653278236016c5a1358f17f9507dbc599a5c186dda2529f", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibdveaqk6e5peub6bsn2n7ds2n6xgnwkbsgdkge6zdpa2juzamgcgwh3gb4" + } + ], + "transaction_hashes": [ + "0xe895434a95b4ca30c653278236016c5a1358f17f9507dbc599a5c186dda2529f" + ], + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" + }, + "retrieved_sha256": "05881c4397aabfdf25f0e5211af280b6a582d11bc7e611095c73f6ccf47f2393", + "retrieval": "verified", + "possession": { + "network": "calibration", + "chain_id": 314159, + "observed_block": "4234005", + "observed_at": "2026-11-10T21:36:24.824Z", + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "1", + "transaction_hash": "0xe895434a95b4ca30c653278236016c5a1358f17f9507dbc599a5c186dda2529f", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibdveaqk6e5peub6bsn2n7ds2n6xgnwkbsgdkge6zdpa2juzamgcgwh3gb4", + "registered": true, + "live": true, + "service_end_epoch": "0", + "next_challenge_epoch": "4234215", + "proof_status": "pending_this_period", + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" + } + ] + } + }, + "pharma-dataset": { + "receipt": { + "network": "calibration", + "chain_id": 314159, + "piece_cid": "bafkzcibdtydqnjm3hkbkx6yxryhhl2dug2ny4ansdaqydteemvmboba4iu5477yk", + "sha256": "eac417517674d3b53e73321e5a1f58034bc005e0876474f5ecf051af0e892f41", + "size_bytes": 1106, + "complete": true, + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "2", + "transaction_hash": "0x77e79da32215090106677d56d85dd5590eefbc84faf2552637d4189ef15404a3", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibdtydqnjm3hkbkx6yxryhhl2dug2ny4ansdaqydteemvmboba4iu5477yk" + } + ], + "transaction_hashes": [ + "0x77e79da32215090106677d56d85dd5590eefbc84faf2552637d4189ef15404a3" + ], + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" + }, + "retrieved_sha256": "eac417517674d3b53e73321e5a1f58034bc005e0876474f5ecf051af0e892f41", + "retrieval": "verified", + "possession": { + "network": "calibration", + "chain_id": 314159, + "observed_block": "4234006", + "observed_at": "2026-11-10T21:36:35.854Z", + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "2", + "transaction_hash": "0x77e79da32215090106677d56d85dd5590eefbc84faf2552637d4189ef15404a3", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibdtydqnjm3hkbkx6yxryhhl2dug2ny4ansdaqydteemvmboba4iu5477yk", + "registered": true, + "live": true, + "service_end_epoch": "0", + "next_challenge_epoch": "4234215", + "proof_status": "pending_this_period", + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" + } + ] + } + }, + "pharma-checkpoint": { + "receipt": { + "network": "calibration", + "chain_id": 314159, + "piece_cid": "bafkzcibchqcjsv7tvkjyon2znkusp6hywkruuk2vfoia37pgacdj2ryrg3kjoky", + "sha256": "66bbd36b5684e404e0c242725cc4fbfa52b4db17a67be0f64aa19e2f06306cd1", + "size_bytes": 448, + "complete": true, + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "3", + "transaction_hash": "0xeec3df365fb20c6f6d4bfabd89df94e379e85502dfb50a6c25b2d325338abc8f", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibchqcjsv7tvkjyon2znkusp6hywkruuk2vfoia37pgacdj2ryrg3kjoky" + } + ], + "transaction_hashes": [ + "0xeec3df365fb20c6f6d4bfabd89df94e379e85502dfb50a6c25b2d325338abc8f" + ], + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" + }, + "retrieved_sha256": "66bbd36b5684e404e0c242725cc4fbfa52b4db17a67be0f64aa19e2f06306cd1", + "retrieval": "verified", + "possession": { + "network": "calibration", + "chain_id": 314159, + "observed_block": "4234006", + "observed_at": "2026-11-10T21:36:46.800Z", + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "3", + "transaction_hash": "0xeec3df365fb20c6f6d4bfabd89df94e379e85502dfb50a6c25b2d325338abc8f", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibchqcjsv7tvkjyon2znkusp6hywkruuk2vfoia37pgacdj2ryrg3kjoky", + "registered": true, + "live": true, + "service_end_epoch": "0", + "next_challenge_epoch": "4234215", + "proof_status": "pending_this_period", + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" + } + ] + } + }, + "finance-package": { + "receipt": { + "network": "calibration", + "chain_id": 314159, + "piece_cid": "bafkzcibd5voqunvk3t67szzdlt67vl4ipgjlh5vlamrjefkkuuxxypqrjcaiiwba", + "sha256": "06145b9339ca58698d9977f6ef6fc2c5675e1004de20ff6c9a8c167bec1a5000", + "size_bytes": 20499, + "complete": true, + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "4", + "transaction_hash": "0xedc6401aa5aa80085aafb3298baddbb3c504da45f4fe233bf3132efb44859449", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibd5voqunvk3t67szzdlt67vl4ipgjlh5vlamrjefkkuuxxypqrjcaiiwba" + } + ], + "transaction_hashes": [ + "0xedc6401aa5aa80085aafb3298baddbb3c504da45f4fe233bf3132efb44859449" + ], + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" + }, + "retrieved_sha256": "06145b9339ca58698d9977f6ef6fc2c5675e1004de20ff6c9a8c167bec1a5000", + "retrieval": "verified", + "possession": { + "network": "calibration", + "chain_id": 314159, + "observed_block": "4234006", + "observed_at": "2026-11-10T21:36:58.405Z", + "copies": [ + { + "provider_id": "9", + "data_set_id": "52518", + "piece_id": "4", + "transaction_hash": "0xedc6401aa5aa80085aafb3298baddbb3c504da45f4fe233bf3132efb44859449", + "provider_address": "0xa3971A7234a3379A1813d9867B531e7EeB20ae07", + "retrieval_url": "https://calib.ezpdpz.net/piece/bafkzcibd5voqunvk3t67szzdlt67vl4ipgjlh5vlamrjefkkuuxxypqrjcaiiwba", + "registered": true, + "live": true, + "service_end_epoch": "0", + "next_challenge_epoch": "4234215", + "proof_status": "pending_this_period", + "payer_address": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f" + } + ] + } + } +} diff --git a/backend/docs/evidence/filecoin-calibration-20261111/summary.json b/backend/docs/evidence/filecoin-calibration-20261111/summary.json new file mode 100644 index 00000000..05e3d3fa --- /dev/null +++ b/backend/docs/evidence/filecoin-calibration-20261111/summary.json @@ -0,0 +1,43 @@ +{ + "network": "calibration", + "chain_id": 314159, + "issue": 144, + "pull_request": 145, + "wallet": "0xA63A33e52589c761F90ce46F7c86aeEe8644112f", + "new_custom_contracts": 0, + "confirmed_objects": 5, + "total_object_bytes": 30192, + "data_set_id": "52518", + "provider_id": "9", + "execution": "Local CPU; no live mesh peers or production HTTP jobs in this evidence run", + "pharma": { + "scope": "Previously generated NCN RHF/STO-3G active-space Hamiltonian", + "new_molecule_generation": false, + "energy_hartree": -148.33704862401328, + "evaluations": 35, + "same_hamiltonian_absolute_error_hartree": 4.343806381257309e-08, + "fresh_resume": true, + "resume_compute_calls": 0, + "resume_uploads": 0, + "outcomes_hash": "7861a4f9bcba6b9a939edf62c30a86e1e0c232937a229c2411d7d647091ce23c" + }, + "finance": { + "data_source": "Synthetic price series retrieved from Filecoin; no observed market prices", + "row_count": 13, + "asset_count": 4, + "selected_assets": [ + "SYN_B", + "SYN_A" + ], + "objective": 0.554655045676, + "feasible_portfolios": 6, + "classical_objective_gap": 0.0, + "result_hash": "aa03fc3ad7fd22fd49e2e6e34cfbfa15ae968071491c3504d1393fd4b1f29239" + }, + "near": { + "package_verification": "verified", + "confirmed_anchors": 2 + }, + "pdp": "pending_this_period at captured observations; not a completed possession-proof claim", + "ui_validation": "Production build and typecheck; isolated browser fixture with simulated account/API. Real Brave email OTP not exercised because computer-use initialization failed." +} diff --git a/backend/docs/filecoin-calibration.md b/backend/docs/filecoin-calibration.md new file mode 100644 index 00000000..721ad030 --- /dev/null +++ b/backend/docs/filecoin-calibration.md @@ -0,0 +1,154 @@ +# QMeshPy + Filecoin (Calibration) + +Filecoin stores the research bytes needed after a worker disappears. NEAR remains +the independent commitment registry. This integration is opt-in and accepts only +Calibration chain ID **314159**. It uses faucet test FIL and test USDFC; it cannot +switch to mainnet or purchase tokens. No new custom Filecoin contract is deployed. + +## What is connected + +| Capability | Product path | +| --- | --- | +| Experiment preservation | Explicit completed-circuit export includes inputs, result, execution plan, provenance and a hashed file manifest. The typed Filecoin receipt is attached without changing the existing provenance commitment. Arbitrary research packages can use the artifact endpoint or CLI. | +| Shared datasets | Finance accepts an immutable CSV receipt and verifies downloaded bytes before building the portfolio. Pharma tasks load a versioned dataset before dispatch; existing workers receive the verified input JSON. | +| Task recovery | Pharma VQE task batches checkpoint completed outcomes. A fresh coordinator retrieves the checkpoint, checks namespace/input/group bindings, and executes only unfinished tasks. This does not resume inside an optimizer iteration. | +| Storage checks | Status reports observed on-chain dataset/piece registration, service state and PDP period evidence separately from a fresh download and checksum check. Pending possession evidence is not labelled proven. | +| Agent retention funding | Operators configure a distinct Calibration wallet and finite budget for each existing agent. Its owner can deposit test USDFC, approve bounded storage, replenish funding, revoke, terminate a dataset, or withdraw available funds. Existing Base AgentKit wallets are not silently reused. | +| Filecoin + NEAR verification | A fresh process retrieves the archived bytes and recomputes input/result/bundle commitments, then checks the existing finalized NEAR registry anchors. Storage proofs establish possession; they do not establish compute quality. | + +Uploads require explicit `publish_publicly:true`. These endpoints publish plaintext +public research; do not send confidential inputs. Normal computation and existing +Kubo pinning continue when Filecoin is disabled. Completion, confirmed storage, +successful retrieval, and PDP proof observations are distinct states. + +## Configure + +Install Node 22+ and the pinned bridge dependencies: + +```sh +cd services/filecoin +npm ci --ignore-scripts +``` + +Use an existing QMeshPy agent ID and its actual owner ID. Provision a dedicated +Calibration key through the deployment's secret store; never commit it: + +```text +QMESH_FILECOIN_ENABLED=true +QMESH_FILECOIN_CHAIN_ID=314159 +QMESH_FILECOIN_RPC_URL=https://api.calibration.node.glif.io/rpc/v1 +QMESH_FILECOIN_AGENTS={"agent-id":{"owner_id":"user-id","key_env":"AGENT_A_FILECOIN_KEY","address":"0x...","max_monthly_usdfc":"0.25","max_lockup_usdfc":"1","max_deposit_usdfc":"1","retention_days":7}} +AGENT_A_FILECOIN_KEY= +``` + +MongoDB is required for operation journals, task records and coordinator leases. +Set `QMESH_FILECOIN_JOURNAL_DIR` to a persistent directory for the bridge's public +transaction events; the local default is `tmp/filecoin-journal`. Events are flushed +before confirmation waits so a killed subprocess does not erase known transaction hashes. +The SDK path is resolved relative to the repository; deploy `services/filecoin` +alongside `backend`. The bridge checks the RPC chain and expected wallet before +signing. Agent ownership is checked against both configuration and `ai_agents`. + +The monthly and lockup values are finite on-chain allowance ceilings. The deposit +ceiling applies per request; the bridge also limits the funded account to 10 test +USDFC. `retention_days` requests funded runway, not automatic deletion or a fixed +expiry guarantee. Monitor funding and provider status; renew with an explicit +deposit. Ending a dataset affects all its pieces, so review its contents first. +The current storage contract requires a 30-day operator lockup approval even +when the requested funding runway is seven days. The bridge reads that requirement +before publishing and rejects shorter approvals. Small datasets also incur a +fixed service fee and lifecycle reserve; the default limits cover a small demo, +not an unlimited archive. + +## Vault interface + +Open **My Vault → Filecoin** (`/vault/filecoin`) while signed in. The page lists +only the current owner's configured agents and recent operations. Archive a +completed circuit run, publish a dataset, inspect saved receipts, check retrieval +and PDP status independently, or verify an experiment's NEAR commitments. +The same page provides explicit funding controls and task resume; finance links +to the receipt-based CSV import. Publication and payment actions show a +confirmation before submission. An interrupted request retains its operation ID +in the browser so the user can inspect its durable backend record before writing +again. Private keys remain in the backend's configured secret store. + +## API sequence + +All routes use the existing authenticated backend identity: + +1. `GET /api/v1/filecoin/agents/{agent_id}/balance`. +2. `POST /api/v1/filecoin/payments` with `agent_id`, unique `operation_id`, + `action:"deposit"`, `amount_usdfc:"1"`; then another operation with + `action:"approve"`. Approval limits come from server policy, not the request. +3. `POST /api/v1/filecoin/runs/{run_id}/archive` with `agent_id`, `operation_id`, + `publish_publicly:true`; or `/artifacts` with the same fields and `data_base64`. +4. `POST /api/v1/filecoin/retrieve` or `/status` with the returned receipt. +5. For finance, `POST /api/v1/finance/filecoin-submit` with a CSV `receipt` and + normal portfolio configuration. Bytes must be the CSV itself, not a package. +6. For pharma, upload bytes produced by `create_dataset("pharma.vqe", inputs)`; + call `POST /api/v1/filecoin/tasks/{run_id}/run` with `agent_id`, `dataset` + receipt and `publish_publicly:true`. Poll `GET /api/v1/filecoin/tasks/{run_id}`. + After an interruption, submit the same run ID and dataset to resume. Active + leases cannot be reclaimed; expired leases can. Checkpoints are saved per + successful task, using the same agent's finite allowance. + +Files are limited to 16 MiB and the SDK requires at least 127 bytes. Wrap smaller +research artifacts in a package. The current public task endpoint bounds VQE +problem sizes and batch length; it uses the existing real pharma kernel. + +## Independent verification + +```sh +python backend/scripts/filecoin.py download receipt.json downloaded-package.json +QB2_NEAR_REGISTRY_CONTRACT=qmesh-provenance-20261110.testnet \ +QB2_NEAR_REGISTRY_PUBLISHER=qmesh-provenance-20261110.testnet \ +python backend/scripts/filecoin.py verify receipt.json +python backend/scripts/filecoin.py status receipt.json +``` + +`download` needs only public receipt metadata and registered provider lookup; +it ignores any caller-provided retrieval URL. It validates length, SHA256 and +PieceCID. `verify` additionally requires the experiment package schema and reports +each commitment and anchor check. Missing NEAR configuration remains unavailable. + +`pack DIRECTORY OUTPUT --identity RUN_ID` creates a manifest package; +`upload INPUT RECEIPT --publish-publicly` publishes it using an operator-supplied +Calibration key. The CLI uses an exclusive `.intent` file; it does not overwrite +receipts or automatically repeat interrupted writes. + +## Interrupted writes and recovery + +Every API write records intent before calling the SDK. Repeating a confirmed +operation returns its receipt; reusing its ID for another request is rejected. +A safe preflight rejection releases the agent lock. An uncertain submission keeps +the lock and any known transaction hashes; inspect +`GET /api/v1/filecoin/agents/{agent_id}/operations/{operation_id}` before proceeding. +Never clear a lock solely because a request timed out. An operator must compare +the transaction, account, dataset and piece state, persist recovered evidence, +then resolve the journal. There is no automatic repayment or blind retry. + +Task uploads record an exact owner/agent/run scope. Recovery reads the latest +confirmed checkpoint from the operation journal even if the task callback died. +Per-run leases renew while work is active; stale coordinators cannot persist +results. A cancelled local compute thread may finish its current calculation. + +## Validation and ecosystem contribution + +The tests cover mainnet rejection, finite budgets, corrupted bytes, user ownership, +immutable finance inputs, task restart, grouped tasks, uncertain writes, archive +replay and coordinator leases. CI also tests real Mongo atomic claims. Live proof +results, when available, are recorded separately with public transaction links. + +The [Calibration evidence pack](evidence/filecoin-calibration-20261111/README.md) +contains five confirmed objects, fresh retrieval checks, the finance result, +the pharma checkpoint/resume result, and independent NEAR verification. + +The reusable package format and verifier are Apache-2.0 project code. They make +research artifacts useful to Filecoin clients outside QMeshPy and make actual +retrieval reliability measurable. Testnet activity is development evidence, not +mainnet revenue or guaranteed grant eligibility. + +Primary references: [Synapse](https://docs.filecoin.cloud/developer-guides/synapse/), +[PDP](https://docs.filecoin.cloud/core-concepts/pdp-overview/), +[Filecoin Pay](https://docs.filecoin.cloud/core-concepts/filecoin-pay-overview/), +[Calibration](https://github.com/filecoin-project/testnet-calibration). diff --git a/backend/scripts/filecoin.py b/backend/scripts/filecoin.py new file mode 100644 index 00000000..595ebaae --- /dev/null +++ b/backend/scripts/filecoin.py @@ -0,0 +1,97 @@ +"""Calibration artifact CLI. No wallet funding or write retries are implicit. + +Run with the installed backend environment. Credentials come from environment; +receipts and journal files contain public data only. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +from pathlib import Path + +from quantum_backend_v2.application.filecoin_archive import verify_export +from quantum_backend_v2.artifacts.filecoin import FilecoinReceipt, FilecoinStore, Package +from quantum_backend_v2.config.models import NearRegistrySettings +from quantum_backend_v2.provenance.near_registry import NearRegistryAdapter + + +async def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + pack = commands.add_parser("pack") + pack.add_argument("directory", type=Path) + pack.add_argument("output", type=Path) + pack.add_argument("--identity", required=True) + upload = commands.add_parser("upload") + upload.add_argument("input", type=Path) + upload.add_argument("receipt", type=Path) + upload.add_argument("--publish-publicly", action="store_true", required=True) + upload.add_argument("--monthly-budget", default="0.25") + upload.add_argument("--lockup-budget", default="1") + for name in ("download", "verify", "status"): + command = commands.add_parser(name) + command.add_argument("receipt", type=Path) + if name == "download": + command.add_argument("output", type=Path) + args = parser.parse_args() + if args.command == "pack": + files = {} + root = args.directory.resolve(strict=True) + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise ValueError("Package directories must not contain symbolic links") + if path.is_file(): + if not path.resolve().is_relative_to(root): + raise ValueError("Package file escaped its directory") + files[path.relative_to(root).as_posix()] = path.read_bytes() + package = Package.create(kind="experiment", identity=args.identity, files=files) + with args.output.open("xb") as handle: + handle.write(package.to_bytes()) + print(json.dumps({"package": str(args.output), "files": len(files)})) + return + store = FilecoinStore( + max_monthly_usdfc=getattr(args, "monthly_budget", "0.25"), + max_lockup_usdfc=getattr(args, "lockup_budget", "1"), + ) + if args.command == "upload": + journal = args.receipt.with_suffix(args.receipt.suffix + ".intent") + if args.receipt.exists(): + raise ValueError("Receipt exists; verify it instead of uploading again") + # Exclusive intent survives timeout/kill; never automatically clear uncertain writes. + with journal.open("x") as handle: + json.dump({"state": "started", "input": str(args.input)}, handle) + receipt = await store.upload(args.input.read_bytes()) + with args.receipt.open("x") as handle: + handle.write(receipt.model_dump_json(indent=2)) + journal.write_text(json.dumps({"state": "confirmed", "receipt": str(args.receipt)})) + print(receipt.model_dump_json()) + return + receipt = FilecoinReceipt.model_validate_json(args.receipt.read_bytes()) + if args.command == "status": + print(json.dumps(await store.status(receipt))) + return + data = await store.download(receipt) + if args.command == "download": + with args.output.open("xb") as handle: + handle.write(data) + print(json.dumps({"retrieval": "verified", "sha256": receipt.sha256})) + return + settings = NearRegistrySettings( + contract_id=os.getenv("QB2_NEAR_REGISTRY_CONTRACT"), + publisher_id=os.getenv("QB2_NEAR_REGISTRY_PUBLISHER"), + rpc_url=os.getenv("QB2_NEAR_RPC_URL", "https://rpc.testnet.near.org"), + ) + adapters = {"near:testnet": NearRegistryAdapter(settings)} if settings.configured else {} + report = await verify_export(data, anchor_adapters=adapters) + print(report.model_dump_json(indent=2)) + if any(check.status == "failed" for check in report.checks): + raise SystemExit(1) + if report.status != "verified": + raise SystemExit(2) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/src/quantum_backend_v2/api/app.py b/backend/src/quantum_backend_v2/api/app.py index e35b4bcd..f82a0c35 100644 --- a/backend/src/quantum_backend_v2/api/app.py +++ b/backend/src/quantum_backend_v2/api/app.py @@ -37,6 +37,8 @@ from quantum_backend_v2.api.routers.credits import build_credits_router from quantum_backend_v2.api.routers.enrollment import build_enrollment_router from quantum_backend_v2.api.routers.financial import build_financial_router +from quantum_backend_v2.api.routers.filecoin import build_filecoin_router +from quantum_backend_v2.application.filecoin_service import FilecoinService from quantum_backend_v2.api.routers.options import build_options_router from quantum_backend_v2.api.routers.payments_admin import build_payments_admin_router from quantum_backend_v2.api.routers.pharma import build_pharma_router @@ -88,6 +90,7 @@ def create_app( runtime_recovery_service: RuntimeRecoveryService | None = None, provenance_service: ProvenanceService | None = None, anchor_service: ProvenanceAnchorService | None = None, + filecoin_service: FilecoinService | None = None, ) -> FastAPI: """Create the backend FastAPI application.""" configure_auth( @@ -130,6 +133,8 @@ def create_app( @asynccontextmanager async def lifespan(app: FastAPI) -> Any: await persistence_runtime.startup() + if filecoin_service is not None: + await filecoin_service.recover_tasks() if pharma_job_service is not None and persistence_runtime.mongodb is not None: # Pipelines run inside this process, so any job still active died with the last one. interrupted = await pharma_job_service.recover_interrupted() @@ -232,6 +237,8 @@ async def favicon() -> Response: register_exception_handlers(app) app.include_router(build_chains_router(registry=chain_registry)) + if filecoin_service is not None: + app.include_router(build_filecoin_router(service=filecoin_service)) app.include_router( system_router( diff --git a/backend/src/quantum_backend_v2/api/models/financial.py b/backend/src/quantum_backend_v2/api/models/financial.py index f7d086c6..e2696164 100644 --- a/backend/src/quantum_backend_v2/api/models/financial.py +++ b/backend/src/quantum_backend_v2/api/models/financial.py @@ -4,10 +4,12 @@ from datetime import date, datetime from enum import Enum -from typing import Any +from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field +from quantum_backend_v2.artifacts.filecoin import FilecoinReceipt + class FinancialJobStatus(str, Enum): """Status of a financial analysis job.""" @@ -26,6 +28,25 @@ class FinancialSubmitResponse(BaseModel): problem_type: str +class FilecoinPortfolioRequest(BaseModel): + """Analyze a public, immutable CSV; storage does not establish its market source.""" + + model_config = ConfigDict(extra="forbid") + + receipt: FilecoinReceipt + filename: str = Field(default="filecoin-dataset.csv", min_length=5, max_length=128, + pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*\.[cC][sS][vV]$") + budget: int | None = Field(default=None, ge=1) + risk_aversion: float = Field(default=0.5, ge=0.0, le=10.0, allow_inf_nan=False) + max_assets_considered: int = Field(default=6, ge=2, le=8) + qaoa_reps: int = Field(default=1, ge=1, le=4) + parameter_search_steps: int = Field(default=9, ge=3, le=25) + date_column: str | None = Field(default=None, max_length=128) + ticker_column: str | None = Field(default=None, max_length=128) + value_column: str | None = Field(default=None, max_length=128) + value_mode: Literal["auto", "prices", "returns"] = "auto" + + class FinancialJobSummary(BaseModel): """Summary of a financial job for list view.""" diff --git a/backend/src/quantum_backend_v2/api/routers/filecoin.py b/backend/src/quantum_backend_v2/api/routers/filecoin.py new file mode 100644 index 00000000..e0d96e25 --- /dev/null +++ b/backend/src/quantum_backend_v2/api/routers/filecoin.py @@ -0,0 +1,166 @@ +"""Opt-in Calibration storage, funding, verification and task recovery endpoints.""" + +from __future__ import annotations + +import base64 +from decimal import Decimal +from typing import Literal + +from fastapi import APIRouter, BackgroundTasks, HTTPException, Response +from pydantic import BaseModel, ConfigDict, Field + +from quantum_backend_v2.api.deps.auth import CurrentUser +from quantum_backend_v2.application.filecoin_service import FilecoinService +from quantum_backend_v2.artifacts.filecoin import MAX_BYTES, FilecoinReceipt + + +class WriteRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + agent_id: str = Field(min_length=1, max_length=128) + operation_id: str = Field(min_length=8, max_length=128, pattern=r"^[a-zA-Z0-9_-]+$") + + +class UploadRequest(WriteRequest): + publish_publicly: Literal[True] + data_base64: str = Field(min_length=1, max_length=MAX_BYTES * 4 // 3 + 4) + + +class ExportRequest(WriteRequest): + publish_publicly: Literal[True] + + +class PaymentRequest(WriteRequest): + action: Literal["deposit", "withdraw", "approve", "revoke", "terminate"] + amount_usdfc: Decimal | None = Field(default=None, gt=0, le=10) + data_set_id: str | None = Field(default=None, pattern=r"^[0-9]+$") + + +class TaskRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + agent_id: str = Field(min_length=1, max_length=128) + dataset: FilecoinReceipt + publish_publicly: Literal[True] + + +def build_filecoin_router(*, service: FilecoinService) -> APIRouter: + router = APIRouter(prefix="/api/v1/filecoin", tags=["filecoin"]) + + async def call(awaitable): + try: + return await awaitable + except LookupError: + raise HTTPException(404, "Resource not found") from None + except ValueError as exc: + raise HTTPException(409, str(exc)) from None + except (OSError, TimeoutError): + raise HTTPException( + 503, "Filecoin unavailable; reconcile the operation before retrying writes" + ) from None + + @router.get("/accounts") + async def accounts(current_user: CurrentUser): + return {"accounts": await call(service.accounts(current_user.user_id))} + + @router.get("/artifacts") + async def history(current_user: CurrentUser): + return {"artifacts": await call(service.artifacts(current_user.user_id))} + + @router.post("/artifacts", response_model=FilecoinReceipt) + async def upload(request: UploadRequest, current_user: CurrentUser): + try: + data = base64.b64decode(request.data_base64, validate=True) + except ValueError: + raise HTTPException(422, "Invalid base64") from None + return await call( + service.upload( + agent_id=request.agent_id, + owner_id=current_user.user_id, + operation_id=request.operation_id, + data=data, + ) + ) + + @router.post("/runs/{run_id}/archive", response_model=FilecoinReceipt) + async def archive(run_id: str, request: ExportRequest, current_user: CurrentUser): + return await call( + service.export( + agent_id=request.agent_id, + owner_id=current_user.user_id, + operation_id=request.operation_id, + run_id=run_id, + ) + ) + + @router.post("/retrieve") + async def retrieve(receipt: FilecoinReceipt, current_user: CurrentUser): + data = await call(service.reader.download(receipt)) + return Response( + data, + media_type="application/octet-stream", + headers={"Cache-Control": "private, no-store"}, + ) + + @router.post("/status") + async def storage_status(receipt: FilecoinReceipt, current_user: CurrentUser): + return await call(service.reader.status(receipt)) + + @router.post("/verify") + async def verify(receipt: FilecoinReceipt, current_user: CurrentUser): + return await call(service.verify(receipt)) + + @router.get("/agents/{agent_id}/balance") + async def balance(agent_id: str, current_user: CurrentUser): + return await call( + service.payment( + agent_id=agent_id, + owner_id=current_user.user_id, + operation_id=None, + action="balance", + ) + ) + + @router.post("/payments") + async def payment(request: PaymentRequest, current_user: CurrentUser): + return await call(service.payment(owner_id=current_user.user_id, **request.model_dump())) + + @router.get("/agents/{agent_id}/operations/{operation_id}") + async def operation(agent_id: str, operation_id: str, current_user: CurrentUser): + from quantum_backend_v2.provenance.hashing import hash_json + + await call(service.account(agent_id, current_user.user_id)) + row = await service.operations.collection.find_one( + {"_id": hash_json([current_user.user_id, agent_id, operation_id])} + ) + if row is None: + raise HTTPException(404, "Operation not found") + row.pop("_id", None) + return row + + @router.post("/tasks/{run_id}/run", status_code=202) + async def tasks( + run_id: str, + request: TaskRequest, + current_user: CurrentUser, + background_tasks: BackgroundTasks, + ): + if not 1 <= len(run_id) <= 128: + raise HTTPException(422, "Run ID must contain 1 to 128 characters") + key = await call( + service.prepare_tasks( + run_id=run_id, + owner_id=current_user.user_id, + agent_id=request.agent_id, + dataset=request.dataset, + ) + ) + background_tasks.add_task(service.execute_tasks, key) + return {"run_id": run_id, "state": "running"} + + @router.get("/tasks/{run_id}") + async def task_status(run_id: str, current_user: CurrentUser): + row = await service.task(run_id, current_user.user_id) + if row is None: + raise HTTPException(404, "Task run not found") + return row + + return router diff --git a/backend/src/quantum_backend_v2/api/routers/financial.py b/backend/src/quantum_backend_v2/api/routers/financial.py index b62da648..33ff7a91 100644 --- a/backend/src/quantum_backend_v2/api/routers/financial.py +++ b/backend/src/quantum_backend_v2/api/routers/financial.py @@ -20,6 +20,7 @@ from quantum_backend_v2.api.errors.models import not_found from quantum_backend_v2.api.finance_download import finance_provenance_download from quantum_backend_v2.api.models.financial import ( + FilecoinPortfolioRequest, FinancialComparisonResponse, FinancialJobResponse, FinancialJobSummary, @@ -44,6 +45,26 @@ def build_financial_router( """Build the financial analysis router.""" router = APIRouter(prefix="/api/v1/finance", tags=["financial"]) + @router.post("/filecoin-submit", response_model=FinancialSubmitResponse, status_code=201) + async def submit_filecoin_dataset( + request: FilecoinPortfolioRequest, + background_tasks: BackgroundTasks, + current_user: CurrentUser, + ) -> FinancialSubmitResponse: + try: + record = await financial_job_service.submit_filecoin( + owner_user_id=current_user.user_id, filename=request.filename, + receipt=request.receipt, + config=PortfolioOptimizationConfig( + **request.model_dump(exclude={"receipt", "filename"}) + ), + ) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from None + background_tasks.add_task(financial_job_service.process, job_id=record.id, csv_bytes=b"") + return FinancialSubmitResponse(job_id=record.id, status=record.status, + problem_type="portfolio_optimization") + async def queue_tokenized_job( request: TokenizedPortfolioRequest, background_tasks: BackgroundTasks, diff --git a/backend/src/quantum_backend_v2/application/filecoin_archive.py b/backend/src/quantum_backend_v2/application/filecoin_archive.py new file mode 100644 index 00000000..1355f6b2 --- /dev/null +++ b/backend/src/quantum_backend_v2/application/filecoin_archive.py @@ -0,0 +1,296 @@ +"""Explicit public research exports with durable, non-replaying write journals.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from datetime import datetime, timezone +from typing import Any + +from pymongo.errors import DuplicateKeyError + +from quantum_backend_v2.artifacts.filecoin import ( + FilecoinBridgeError, + FilecoinReceipt, + FilecoinStore, + Package, +) +from quantum_backend_v2.provenance.hashing import bundle_hash, canonical_json, hash_json +from quantum_backend_v2.provenance.models import ProvenanceBundle, StorageReference + + +class FilecoinOperations: + """A lost write result freezes the signer until an operator reconciles it.""" + + def __init__(self, collection: Any, locks: Any) -> None: + self.collection, self.locks = collection, locks + + async def recover_confirmed_lock(self, agent_id: str, owner_id: str) -> None: + lock = await self.locks.find_one({"_id": agent_id}) + if lock is None: + return + operation = await self.collection.find_one({"_id": lock["operation_key"]}) + if ( + operation + and operation.get("state") == "confirmed" + and operation.get("owner_id") == owner_id + and operation.get("agent_id") == agent_id + ): + await self.locks.delete_one({"_id": agent_id, "operation_key": lock["operation_key"]}) + + async def execute( + self, + *, + owner_id: str, + agent_id: str, + operation_id: str, + payload: dict[str, Any], + action: Callable[[], Awaitable[dict]], + scope: str | None = None, + ) -> dict: + key = hash_json([owner_id, agent_id, operation_id]) + digest = hash_json(payload) + await self.recover_confirmed_lock(agent_id, owner_id) + existing = await self.collection.find_one({"_id": key}) + if existing: + if existing["request_hash"] != digest: + raise ValueError("Operation ID already binds different content") + if existing["state"] == "confirmed": + return existing["result"] + if existing["state"] != "rejected": + raise ValueError("Operation needs reconciliation; it will not be retried") + try: + await self.locks.insert_one({"_id": agent_id, "operation_key": key}) + except DuplicateKeyError: + raise ValueError( + "Agent has an active or uncertain operation requiring reconciliation" + ) from None + try: + intent = { + "_id": key, + "owner_id": owner_id, + "agent_id": agent_id, + "operation_id": operation_id, + "request_hash": digest, + "state": "started", + "created_at": datetime.now(timezone.utc), + "scope": scope, + } + if existing is not None: + result = await self.collection.update_one( + {"_id": key, "state": "rejected"}, + {"$set": {k: v for k, v in intent.items() if k != "_id"}}, + ) + if not result.matched_count: + raise ValueError("Operation state changed; inspect it before retrying") + else: + await self.collection.insert_one(intent) + except Exception: + await self.locks.delete_one({"_id": agent_id, "operation_key": key}) + raise + try: + result = await action() + await self.collection.update_one( + {"_id": key}, + { + "$set": { + "state": "confirmed", + "result": result, + "updated_at": datetime.now(timezone.utc), + } + }, + ) + except BaseException as exc: + # Keep both the write intent and signer lock even on cancellation/process shutdown. + # A new process cannot distinguish a preflight failure from a lost confirmed receipt. + safe_rejection = isinstance(exc, FilecoinBridgeError) and not exc.uncertain + await self.collection.update_one( + {"_id": key}, + { + "$set": { + "state": "rejected" if safe_rejection else "uncertain", + "error_code": exc.code + if isinstance(exc, FilecoinBridgeError) + else type(exc).__name__, + "transaction_hashes": getattr(exc, "transaction_hashes", []), + "submitted_receipt": getattr(exc, "submitted_receipt", None), + "updated_at": datetime.now(timezone.utc), + } + }, + ) + if safe_rejection: + await self.locks.delete_one({"_id": agent_id, "operation_key": key}) + raise + await self.locks.delete_one({"_id": agent_id, "operation_key": key}) + return result + + +class JournaledFilecoinStore: + """Persist every task checkpoint's write intent, so resume never blindly repays.""" + + def __init__( + self, + store: FilecoinStore, + operations: FilecoinOperations, + *, + owner_id: str, + agent_id: str, + operation_prefix: str, + before_upload: Callable[[],Awaitable[None]] | None = None, + ) -> None: + self.store, self.operations = store, operations + self.owner_id, self.agent_id, self.operation_prefix = owner_id, agent_id, operation_prefix + self.before_upload = before_upload + + async def upload(self, data: bytes) -> FilecoinReceipt: + import hashlib + + if self.before_upload is not None: + await self.before_upload() + + digest = hashlib.sha256(data).hexdigest() + + async def action() -> dict: + if self.before_upload is not None: + await self.before_upload() + return (await self.store.upload(data)).model_dump(mode="json") + + result = await self.operations.execute( + owner_id=self.owner_id, + agent_id=self.agent_id, + operation_id=f"{self.operation_prefix}:{digest}", + payload={"sha256": digest}, + scope=self.operation_prefix, + action=action, + ) + return FilecoinReceipt.model_validate(result) + + async def download(self, receipt: FilecoinReceipt) -> bytes: + return await self.store.download(receipt) + + +class FilecoinBundleStorage: + def __init__(self, store: FilecoinStore) -> None: + self.store = store + + async def retrieve(self, reference: StorageReference) -> ProvenanceBundle: + if reference.filecoin is None: + raise ValueError("Missing Filecoin receipt") + package = Package.model_validate_json(await self.store.download(reference.filecoin)) + if package.kind != "experiment" or "provenance.json" not in package.files: + raise ValueError("Filecoin package is not a provenance export") + bundle = ProvenanceBundle.model_validate_json(package.files["provenance.json"].data()) + if package.identity != bundle.workflow_run_id: + raise ValueError("Package identity does not match its provenance") + return bundle + + +async def verify_export(data: bytes, *, anchor_adapters: dict | None = None) -> Any: + """Verify downloaded evidence without a local run database or original files.""" + import json + + from quantum_backend_v2.provenance.verification import RunEvidence, verify_bundle + + package = Package.model_validate_json(data) + if package.kind != "experiment": + raise ValueError("Expected an experiment package") + required = {"provenance.json", "inputs.json", "result.json", "plan.json", "manifest.json"} + if not required <= package.files.keys(): + raise ValueError("Experiment package is missing reproducibility files") + bundle = ProvenanceBundle.model_validate_json(package.files["provenance.json"].data()) + manifest = json.loads(package.files["manifest.json"].data()) + plan = json.loads(package.files["plan.json"].data()) + if ( + package.identity != bundle.workflow_run_id + or manifest.get("workflow_run_id") != package.identity + or manifest.get("plan_id") != bundle.plan_id + or not isinstance(plan, dict) + ): + raise ValueError("Experiment package identity or plan does not match provenance") + return await verify_bundle( + bundle.model_copy(update={"storage": ()}), + expected_hash=manifest.get("bundle_hash"), + run=RunEvidence( + run_id=bundle.workflow_run_id, + workflow_definition_id=bundle.workflow_definition_id, + owner_user_id=bundle.owner_user_id, + plan_id=manifest["plan_id"], + input_snapshot=json.loads(package.files["inputs.json"].data()), + result=json.loads(package.files["result.json"].data()), + ), + anchor_adapters=anchor_adapters, + ) + + +async def prepare_run_export(run_id: str, *, owner_id: str) -> tuple[ProvenanceBundle, bytes]: + from quantum_backend_v2.application.provenance import run_evidence + from quantum_backend_v2.persistence.mongodb import ( + ExecutionPlanDocument, + ProvenanceBundleDocument, + WorkflowRunDocument, + ) + from quantum_backend_v2.provenance.verification import verify_bundle + + run = await WorkflowRunDocument.get(run_id) + if run is None or run.owner_user_id != owner_id: + raise LookupError("Run not found") + if run.status != "COMPLETED": + raise ValueError("Only completed runs can be archived") + record = await ProvenanceBundleDocument.find_one({"workflow_run_id": run_id}) + if record is None: + raise ValueError("Run has no recorded provenance") + bundle = ProvenanceBundle.model_validate(record.payload) + report = await verify_bundle(bundle, expected_hash=record.content_hash, run=run_evidence(run)) + if any(check.status != "passed" for check in report.checks if check.kind == "hash"): + raise ValueError("Run data no longer matches its provenance") + plan = ( + await ExecutionPlanDocument.get(run.artifact_bundle_id) if run.artifact_bundle_id else None + ) + if plan is None: + raise ValueError("Recorded execution plan is unavailable") + files = { + "provenance.json": canonical_json( + bundle.model_copy(update={"storage": ()}).model_dump(mode="json") + ), + "inputs.json": canonical_json(run.input_snapshot), + "result.json": canonical_json(run.output_snapshot["result"]), + "plan.json": canonical_json(plan.payload), + "manifest.json": canonical_json( + { + "bundle_hash": bundle_hash(bundle), + "workflow_run_id": run.id, + "plan_id": run.artifact_bundle_id, + } + ), + } + package = Package.create(kind="experiment", identity=run.id, files=files) + return bundle, package.to_bytes() + + +async def export_run( + run_id: str, + *, + owner_id: str, + store: FilecoinStore, + prepared: tuple[ProvenanceBundle, bytes] | None = None, +) -> FilecoinReceipt: + from quantum_backend_v2.persistence.mongodb import ProvenanceBundleDocument + + bundle, data = prepared or await prepare_run_export(run_id, owner_id=owner_id) + receipt = await store.upload(data) + reference = StorageReference(backend="filecoin", cid=receipt.piece_cid, filecoin=receipt) + try: + await ProvenanceBundleDocument.get_pymongo_collection().update_one( + {"bundle_id": bundle.bundle_id}, + {"$addToSet": {"payload.storage": reference.model_dump(mode="json")}}, + ) + except Exception as exc: + error = FilecoinBridgeError( + { + "code": "receipt_attachment_failed", + "uncertain": True, + "transaction_hashes": list(receipt.transaction_hashes), + } + ) + error.submitted_receipt = receipt.model_dump(mode="json") + raise error from exc + return receipt diff --git a/backend/src/quantum_backend_v2/application/filecoin_service.py b/backend/src/quantum_backend_v2/application/filecoin_service.py new file mode 100644 index 00000000..f4aaafcc --- /dev/null +++ b/backend/src/quantum_backend_v2/application/filecoin_service.py @@ -0,0 +1,470 @@ +"""Owner-scoped Filecoin agent accounts, explicit payments and resumable task jobs.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import uuid +from contextlib import suppress +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator +from pymongo.errors import DuplicateKeyError + +from quantum_backend_v2.application.filecoin_archive import ( + FilecoinOperations, + JournaledFilecoinStore, + export_run, + prepare_run_export, + verify_export, +) +from quantum_backend_v2.artifacts.filecoin import ( + MAX_BYTES, + FilecoinBridgeError, + FilecoinReceipt, + FilecoinStore, +) +from quantum_backend_v2.provenance.hashing import hash_json + + +class AgentStoragePolicy(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + owner_id: str = Field(min_length=1) + key_env: str = Field(pattern=r"^[A-Z][A-Z0-9_]{1,100}$") + address: str = Field(pattern=r"^0x[0-9a-fA-F]{40}$") + max_monthly_usdfc: Decimal = Field(default=Decimal("0.25"), gt=0, le=10) + max_lockup_usdfc: Decimal = Field(default=Decimal("1"), gt=0, le=10) + max_deposit_usdfc: Decimal = Field(default=Decimal("1"), gt=0, le=10) + retention_days: int = Field(default=7, ge=1, le=30) + + +class FilecoinConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + enabled: bool = False + chain_id: Literal[314159] = 314159 + agents: dict[str, AgentStoragePolicy] = Field(default_factory=dict) + + @model_validator(mode="after") + def distinct_wallets(self) -> FilecoinConfig: + addresses = [policy.address.lower() for policy in self.agents.values()] + if len(set(addresses)) != len(addresses): + raise ValueError("Each Filecoin agent must have a distinct wallet") + return self + + @classmethod + def from_env(cls, env: dict[str, str]) -> FilecoinConfig: + return cls( + enabled=env.get("QMESH_FILECOIN_ENABLED", "false").lower() == "true", + chain_id=int(env.get("QMESH_FILECOIN_CHAIN_ID", "314159")), + agents=json.loads(env.get("QMESH_FILECOIN_AGENTS", "{}")), + ) + + +class FilecoinService: + def __init__( + self, database: Any, config: FilecoinConfig, env: dict[str, str], *, dispatcher: Any + ) -> None: + self.database, self.config, self.env, self.dispatcher = database, config, env, dispatcher + self.operations = FilecoinOperations( + database["filecoin_operations"], database["filecoin_locks"] + ) + # Readers never inherit an operator's signing key. + reader_env = {k: v for k, v in env.items() if k != "QMESH_FILECOIN_PRIVATE_KEY"} + self.reader = FilecoinStore(env=reader_env) + + async def verify(self, receipt: FilecoinReceipt): + from quantum_backend_v2.config.models import NearRegistrySettings + from quantum_backend_v2.provenance.near_registry import NearRegistryAdapter + + settings = NearRegistrySettings( + contract_id=self.env.get("QB2_NEAR_REGISTRY_CONTRACT"), + publisher_id=self.env.get("QB2_NEAR_REGISTRY_PUBLISHER"), + rpc_url=self.env.get("QB2_NEAR_RPC_URL", "https://rpc.testnet.near.org"), + ) + adapters = {"near:testnet": NearRegistryAdapter(settings)} if settings.configured else {} + return await verify_export(await self.reader.download(receipt), anchor_adapters=adapters) + + async def accounts(self, owner_id: str) -> list[dict]: + accounts = [] + for agent_id, policy in self.config.agents.items(): + if policy.owner_id != owner_id: + continue + agent = await self.database["ai_agents"].find_one( + {"agent_id": agent_id, "owner_id": owner_id} + ) + if agent is not None: + accounts.append( + { + "agent_id": agent_id, + "agent_name": agent.get("agent_name", agent_id), + **policy.model_dump(mode="json", exclude={"owner_id", "key_env"}), + } + ) + return accounts + + async def artifacts(self, owner_id: str) -> list[dict]: + return ( + await self.operations.collection.find( + {"owner_id": owner_id}, + { + "_id": 0, + "operation_id": 1, + "agent_id": 1, + "state": 1, + "created_at": 1, + "result": 1, + "submitted_receipt": 1, + "transaction_hashes": 1, + "error_code": 1, + }, + ) + .sort("created_at", -1) + .limit(50) + .to_list(length=50) + ) + + async def account( + self, agent_id: str, owner_id: str + ) -> tuple[AgentStoragePolicy, FilecoinStore]: + policy = self.config.agents.get(agent_id) + if policy is None or policy.owner_id != owner_id: + raise LookupError("Configured Filecoin agent not found") + agent = await self.database["ai_agents"].find_one( + {"agent_id": agent_id, "owner_id": owner_id} + ) + if agent is None: + raise LookupError("Agent not found") + key = self.env.get(policy.key_env) + if not key: + raise ValueError("Filecoin agent signing key is not configured") + env = { + **self.env, + "QMESH_FILECOIN_PRIVATE_KEY": key, + "QMESH_FILECOIN_EXPECTED_ADDRESS": policy.address, + } + return policy, FilecoinStore( + env=env, + retention_days=policy.retention_days, + max_monthly_usdfc=str(policy.max_monthly_usdfc), + max_lockup_usdfc=str(policy.max_lockup_usdfc), + ) + + async def upload( + self, *, agent_id: str, owner_id: str, operation_id: str, data: bytes + ) -> FilecoinReceipt: + if not 127 <= len(data) <= MAX_BYTES: + raise ValueError("Artifact must be 127 bytes to 16 MiB; package smaller files") + _, store = await self.account(agent_id, owner_id) + + async def action() -> dict: + return (await store.upload(data)).model_dump(mode="json") + + result = await self.operations.execute( + owner_id=owner_id, + agent_id=agent_id, + operation_id=operation_id, + payload={"upload": base64.b64encode(data).decode()}, + action=action, + ) + return FilecoinReceipt.model_validate(result) + + async def export( + self, *, agent_id: str, owner_id: str, operation_id: str, run_id: str + ) -> FilecoinReceipt: + _, store = await self.account(agent_id, owner_id) + existing = await self.operations.collection.find_one( + {"_id": hash_json([owner_id, agent_id, operation_id])} + ) + prepared = ( + None + if existing and existing["state"] == "confirmed" + else await prepare_run_export(run_id, owner_id=owner_id) + ) + + async def action() -> dict: + return ( + await export_run(run_id, owner_id=owner_id, store=store, prepared=prepared) + ).model_dump(mode="json") + + result = await self.operations.execute( + owner_id=owner_id, + agent_id=agent_id, + operation_id=operation_id, + payload={"archive_run": run_id}, + action=action, + ) + return FilecoinReceipt.model_validate(result) + + async def payment( + self, + *, + agent_id: str, + owner_id: str, + operation_id: str | None, + action: str, + amount_usdfc: Decimal | None = None, + data_set_id: str | None = None, + ) -> dict: + policy, store = await self.account(agent_id, owner_id) + arguments: dict[str, Any] = {"action": action} + if action == "balance": + return await store.command("payments", action="balance") + if action in {"deposit", "withdraw"}: + if amount_usdfc is None or not 0 < amount_usdfc <= policy.max_deposit_usdfc: + raise ValueError("Amount exceeds the configured test-token budget") + arguments["amount_usdfc"] = str(amount_usdfc) + elif action == "approve": + arguments.update( + max_monthly_usdfc=str(policy.max_monthly_usdfc), + max_lockup_usdfc=str(policy.max_lockup_usdfc), + max_lockup_days=30, + ) + elif action == "terminate": + if data_set_id is None or not data_set_id.isdecimal(): + raise ValueError("A decimal data set ID is required") + arguments["data_set_id"] = data_set_id + elif action != "revoke": + raise ValueError("Unsupported payment action") + if not operation_id: + raise ValueError("Writes require an operation ID") + return await self.operations.execute( + owner_id=owner_id, + agent_id=agent_id, + operation_id=operation_id, + payload=arguments, + action=lambda: store.command("payments", **arguments), + ) + + async def recover_tasks(self) -> None: + await self.database["filecoin_task_runs"].update_many( + {"state": "running", "lease_until": {"$lte": datetime.now(timezone.utc)}}, + {"$set": {"state": "interrupted"}}, + ) + + async def task(self, run_id: str, owner_id: str) -> dict | None: + row = await self.database["filecoin_task_runs"].find_one( + {"_id": hash_json([owner_id, run_id])} + ) + return ( + {key: value for key, value in row.items() if key not in {"_id", "lease_token"}} + if row + else None + ) + + async def prepare_tasks( + self, *, run_id: str, owner_id: str, agent_id: str, dataset: FilecoinReceipt + ) -> str: + await self.account(agent_id, owner_id) + key = hash_json([owner_id, run_id]) + row = { + "_id": key, + "run_id": run_id, + "owner_id": owner_id, + "agent_id": agent_id, + "dataset": dataset.model_dump(mode="json"), + "state": "ready", + "checkpoint": None, + } + try: + await self.database["filecoin_task_runs"].insert_one(row) + except DuplicateKeyError: + existing = await self.database["filecoin_task_runs"].find_one({"_id": key}) + if existing["dataset"] != row["dataset"] or existing["agent_id"] != agent_id: + raise ValueError("Run ID already binds different inputs or agent") from None + now, token = datetime.now(timezone.utc), uuid.uuid4().hex + claimed = await self.database["filecoin_task_runs"].update_one( + { + "_id": key, + "$or": [ + {"state": {"$in": ["ready", "interrupted", "failed"]}}, + {"state": "running", "lease_until": {"$lte": now}}, + ], + }, + { + "$set": { + "state": "running", + "error": None, + "lease_token": token, + "lease_until": now + timedelta(seconds=90), + "updated_at": now, + } + }, + ) + if not claimed.modified_count: + raise ValueError("Run is already running or completed") + return f"{key}:{token}" + + async def execute_tasks(self, claim: str) -> None: + from dataclasses import asdict + + from quantum_backend_v2.application.compute_tasks import TASK_KINDS + from quantum_backend_v2.application.filecoin_tasks import ( + CheckpointPersistenceError, + resolve_inputs, + ) + + key, token = claim.rsplit(":", 1) + collection = self.database["filecoin_task_runs"] + + def active() -> dict: + return { + "_id": key, + "state": "running", + "lease_token": token, + "lease_until": {"$gt": datetime.now(timezone.utc)}, + } + + row = await collection.find_one(active()) + if row is None: + return + current = asyncio.current_task() + assert current is not None + + async def renew() -> None: + try: + while True: + await asyncio.sleep(20) + updated = await collection.update_one( + active(), + { + "$set": { + "lease_until": datetime.now(timezone.utc) + timedelta(seconds=90) + } + }, + ) + if not updated.matched_count: + current.cancel() + return + except Exception: + current.cancel() + + heartbeat = asyncio.create_task(renew()) + try: + _, store = await self.account(row["agent_id"], row["owner_id"]) + scope = f"tasks:{row['owner_id']}:{row['run_id']}" + operation_scope = { + "owner_id": row["owner_id"], + "agent_id": row["agent_id"], + "scope": scope, + } + await self.operations.recover_confirmed_lock(row["agent_id"], row["owner_id"]) + if await self.operations.collection.find_one( + {**operation_scope, "state": {"$in": ["started", "uncertain"]}} + ) or await self.operations.locks.find_one({"_id": row["agent_id"]}): + raise ValueError("Checkpoint operation needs reconciliation before resuming") + # The operation journal commits before the task callback; it survives that crash gap. + confirmed = await self.operations.collection.find_one( + {**operation_scope, "state": "confirmed"}, + sort=[("created_at", -1), ("_id", -1)], + ) + saved = confirmed["result"] if confirmed else row["checkpoint"] + saved_receipt = FilecoinReceipt.model_validate(saved) if saved else None + + async def before_upload() -> None: + renewed = await collection.update_one( + active(), + {"$set": {"lease_until": datetime.now(timezone.utc) + timedelta(seconds=90)}}, + ) + if not renewed.matched_count: + raise FilecoinBridgeError({"code": "lease_expired", "uncertain": False}) + + journaled = JournaledFilecoinStore( + store, + self.operations, + owner_id=row["owner_id"], + agent_id=row["agent_id"], + operation_prefix=scope, + before_upload=before_upload, + ) + dataset = FilecoinReceipt.model_validate(row["dataset"]) + inputs = await resolve_inputs("pharma.vqe", None, journaled, dataset) + for value in inputs: + validate_vqe_input(value) + if not 1 <= len(inputs) <= 16: + raise ValueError("Task datasets require 1 to 16 inputs") + + async def checkpoint(receipt: FilecoinReceipt) -> None: + updated = await collection.update_one( + active(), + { + "$set": { + "checkpoint": receipt.model_dump(mode="json"), + "updated_at": datetime.now(timezone.utc), + } + }, + ) + if not updated.matched_count: + raise ValueError("Task coordinator lease expired") + + if saved_receipt is not None: + await checkpoint(saved_receipt) + outcomes = await self.dispatcher.run( + "pharma.vqe", + inputs, + run_locally=TASK_KINDS["pharma.vqe"], + filecoin_store=journaled, + checkpoint_namespace=key, + checkpoint_receipt=saved_receipt, + on_checkpoint=checkpoint, + ) + await collection.update_one( + active(), + { + "$set": { + "state": "completed", + "outcomes": [asdict(item) if item else None for item in outcomes], + } + }, + ) + except asyncio.CancelledError: + with suppress(Exception): + await collection.update_one( + active(), {"$set": {"state": "interrupted", "error": "CoordinatorInterrupted"}} + ) + raise + except Exception as exc: + failure: dict[str, Any] = {"state": "failed", "error": type(exc).__name__} + if isinstance(exc, CheckpointPersistenceError) and exc.receipt is not None: + failure["checkpoint"] = exc.receipt.model_dump(mode="json") + await collection.update_one(active(), {"$set": failure}) + finally: + heartbeat.cancel() + with suppress(asyncio.CancelledError): + await heartbeat + + +def validate_vqe_input(value: dict) -> None: + """Bound the public task endpoint before expensive local or remote execution.""" + import math + + problem = value.get("problem", {}) + qubits, electrons = problem.get("n_qubits"), problem.get("n_electrons") + iterations, terms = value.get("max_iterations"), problem.get("terms") + if ( + type(qubits) is not int + or not 2 <= qubits <= 8 + or type(electrons) is not int + or not 0 < electrons < qubits + or type(iterations) is not int + or not 1 <= iterations <= 200 + or not isinstance(terms, list) + or not 1 <= len(terms) <= 4096 + ): + raise ValueError("Invalid or oversized pharma VQE task") + for term in terms: + if ( + not isinstance(term, list) + or len(term) != 2 + or not isinstance(term[0], str) + or len(term[0]) != qubits + or set(term[0]) - set("IXYZ") + or type(term[1]) not in (float, int) + or not math.isfinite(term[1]) + ): + raise ValueError("Invalid Hamiltonian term") + seed = value.get("seed") + if seed is not None and (type(seed) is not int or not 0 <= seed < 2**32): + raise ValueError("Invalid random seed") diff --git a/backend/src/quantum_backend_v2/application/filecoin_tasks.py b/backend/src/quantum_backend_v2/application/filecoin_tasks.py new file mode 100644 index 00000000..e257d6d0 --- /dev/null +++ b/backend/src/quantum_backend_v2/application/filecoin_tasks.py @@ -0,0 +1,169 @@ +"""Versioned Filecoin inputs and explicit checkpoints for independent compute tasks.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Hashable, Sequence +from dataclasses import asdict +from typing import TYPE_CHECKING, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + +from quantum_backend_v2.artifacts.filecoin import MAX_BYTES, FilecoinReceipt, FilecoinStore +from quantum_backend_v2.provenance.hashing import canonical_json, hash_json + +if TYPE_CHECKING: + from quantum_backend_v2.application.peer_tasks import TaskOutcome + +CheckpointCallback = Callable[[FilecoinReceipt], Awaitable[None]] + + +class TaskDataset(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + schema_version: Literal["qmesh-filecoin-task-dataset-v1"] = "qmesh-filecoin-task-dataset-v1" + kind: str = Field(min_length=3, max_length=64) + inputs: list[dict[str, Any]] = Field(min_length=1, max_length=64) + + +class SavedOutcome(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + output: dict[str, Any] + executed_by: str = Field(min_length=1, max_length=256) + attempts: int = Field(ge=1) + elapsed_seconds: float = Field(ge=0, allow_inf_nan=False) + + +class TaskCheckpoint(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + schema_version: Literal["qmesh-filecoin-task-checkpoint-v1"] = ( + "qmesh-filecoin-task-checkpoint-v1" + ) + namespace: str = Field(min_length=1, max_length=256) + kind: str = Field(min_length=3, max_length=64) + input_hashes: list[str] = Field(min_length=1, max_length=64) + group_indices: list[int] = Field(min_length=1, max_length=64) + completed: dict[str, SavedOutcome] = Field(max_length=64) + + +def create_dataset(kind: str, inputs: list[dict[str, Any]]) -> bytes: + """Create the portable dataset consumed by PeerTaskDispatcher.run().""" + dataset = TaskDataset(kind=kind, inputs=inputs) + data = canonical_json(dataset.model_dump()) + if len(data) > MAX_BYTES: + raise ValueError("Task dataset exceeds the Filecoin size limit") + return data + + +async def resolve_inputs( + kind: str, + inputs: list[dict[str, Any]] | None, + store: FilecoinStore, + receipt: FilecoinReceipt | None, +) -> list[dict[str, Any]]: + """Hydrate verified inputs on the coordinator before either remote or local execution.""" + if receipt is not None: + if not receipt.complete: + raise ValueError("Task dataset storage is incomplete") + dataset = TaskDataset.model_validate_json(await store.download(receipt)) + if dataset.kind != kind: + raise ValueError("Task dataset kind does not match the request") + if inputs is not None and hash_json(inputs) != hash_json(dataset.inputs): + raise ValueError("Task dataset inputs do not match the request") + inputs = dataset.inputs + if inputs is None: + raise ValueError("Task inputs or a dataset receipt are required") + create_dataset(kind, inputs) + return inputs + + +class CheckpointPersistenceError(RuntimeError): + """Do not retry a write: receipt is the last known, retrievable checkpoint.""" + + def __init__(self, receipt: FilecoinReceipt | None) -> None: + super().__init__("Filecoin checkpoint persistence failed; inspect receipts before resuming") + self.receipt = receipt + + +class CheckpointJournal: + """Serialize checkpoint writes and stop on uncertain storage or receipt persistence.""" + + def __init__( + self, + *, + store: FilecoinStore, + namespace: str, + kind: str, + inputs: list[dict[str, Any]], + groups: Sequence[Hashable], + on_checkpoint: CheckpointCallback, + ) -> None: + group_numbers: dict[Hashable, int] = {} + group_indices = [ + group_numbers.setdefault(group, index) for index, group in enumerate(groups) + ] + self.template = TaskCheckpoint( + namespace=namespace, + kind=kind, + input_hashes=[hash_json(value) for value in inputs], + group_indices=group_indices, + completed={}, + ) + self.store = store + self.on_checkpoint = on_checkpoint + self.receipt: FilecoinReceipt | None = None + self._last_hash: str | None = None + self._failed = False + self._lock = asyncio.Lock() + + async def restore(self, receipt: FilecoinReceipt) -> dict[int, SavedOutcome]: + if not receipt.complete: + raise ValueError("Filecoin checkpoint storage is incomplete") + saved = TaskCheckpoint.model_validate_json(await self.store.download(receipt)) + if saved.model_dump(exclude={"completed"}) != self.template.model_dump( + exclude={"completed"} + ): + raise ValueError("Filecoin checkpoint does not match the run, inputs, kind, or groups") + restored: dict[int, SavedOutcome] = {} + completed_groups: set[int] = set() + for key, outcome in saved.completed.items(): + try: + index = int(key) + except ValueError: + raise ValueError("Invalid Filecoin checkpoint task index") from None + if key != str(index) or not 0 <= index < len(self.template.input_hashes): + raise ValueError("Invalid Filecoin checkpoint task index") + group = self.template.group_indices[index] + if group in completed_groups: + raise ValueError("Filecoin checkpoint has multiple winners for one group") + completed_groups.add(group) + restored[index] = outcome + self.receipt = receipt + self._last_hash = hash_json(saved.model_dump()) + return restored + + async def save(self, outcomes: Sequence[TaskOutcome | None]) -> None: + async with self._lock: + if self._failed: + raise CheckpointPersistenceError(self.receipt) + try: + payload = { + **self.template.model_dump(), + "completed": { + str(index): asdict(outcome) + for index, outcome in enumerate(outcomes) + if outcome is not None + }, + } + checkpoint = TaskCheckpoint.model_validate(payload) + digest = hash_json(checkpoint.model_dump()) + if digest == self._last_hash: + return + data = canonical_json(checkpoint.model_dump()) + if len(data) > MAX_BYTES: + raise ValueError("Task checkpoint exceeds the Filecoin size limit") + self.receipt = await self.store.upload(data) + await self.on_checkpoint(self.receipt) + self._last_hash = digest + except Exception as exc: + self._failed = True + raise CheckpointPersistenceError(self.receipt) from exc diff --git a/backend/src/quantum_backend_v2/application/parity.py b/backend/src/quantum_backend_v2/application/parity.py index 2253f2f9..33db9c44 100644 --- a/backend/src/quantum_backend_v2/application/parity.py +++ b/backend/src/quantum_backend_v2/application/parity.py @@ -35,6 +35,7 @@ tokenized_request, tokenized_result, ) +from quantum_backend_v2.artifacts.filecoin import FilecoinReceipt, FilecoinStore from quantum_backend_v2.chains.market_data import MarketDataSource from quantum_backend_v2.chains.prices import PriceUnavailableError from quantum_backend_v2.identity.models import UserTokenClaims @@ -405,9 +406,11 @@ def __init__( libp2p_runtime: Any, price_source: PriceSource | None = None, history_source: TokenPriceHistorySource | None = None, + filecoin_store: FilecoinStore | None = None, ) -> None: self._price_source = price_source self._history_source = history_source + self._filecoin_store = filecoin_store self._discovery_service = discovery_service self._libp2p_runtime = libp2p_runtime @@ -445,6 +448,35 @@ async def submit( def tokenized_market_data(self) -> MarketDataSource | None: return None if self._history_source is None else self._history_source.market_data + async def submit_filecoin( + self, + *, + owner_user_id: str, + filename: str, + receipt: FilecoinReceipt, + config: PortfolioOptimizationConfig, + ) -> FinancialJobDocument: + """Queue an immutable CSV reference; fetch and verify it before computation.""" + if self._filecoin_store is None: + raise ValueError("Filecoin dataset retrieval is not configured") + if receipt.size_bytes > 4 * 1024 * 1024 or not receipt.complete: + raise ValueError("Filecoin CSV must be complete and at most 4 MB") + if not filename.lower().endswith(".csv"): + raise ValueError("Filecoin finance datasets must be CSV files") + await _ensure_platform_user(owner_user_id) + doc = FinancialJobDocument( + id=f"fin-{uuid.uuid4()}", owner_user_id=owner_user_id, filename=filename, + status=_FIN_STATUS_INGESTING, + result_payload={"request": _submission_request_snapshot(config)}, + request_payload={ + "filecoin_dataset": receipt.model_dump(mode="json"), + "config": _submission_request_snapshot(config), + "price_observations": [], + }, + ) + await doc.insert() + return doc + async def submit_tokenized( self, *, @@ -484,6 +516,22 @@ async def process(self, *, job_id: str, csv_bytes: bytes) -> None: await doc.save() try: + if doc.request_payload and "filecoin_dataset" in doc.request_payload: + if self._filecoin_store is None: + raise ValueError("Filecoin dataset retrieval is not configured") + try: + receipt = FilecoinReceipt.model_validate( + doc.request_payload["filecoin_dataset"] + ) + if receipt.size_bytes > 4 * 1024 * 1024 or not receipt.complete: + raise ValueError("Invalid Filecoin CSV receipt") + verified_csv = await self._filecoin_store.download(receipt) + doc.request_payload["csv_text"] = verified_csv.decode("utf-8-sig") + except (OSError, ValueError, TimeoutError): + raise ValueError( + "Filecoin dataset could not be retrieved and verified" + ) from None + await doc.save() if doc.request_payload and "tokenized_assets" in doc.request_payload: doc.request_payload = await resolve_tokenized_history( doc.request_payload, self._history_source @@ -508,12 +556,12 @@ async def process(self, *, job_id: str, csv_bytes: bytes) -> None: artifacts.payload.update( tokenized_result(artifacts.payload, doc.request_payload) ) - else: + elif "filecoin_dataset" not in doc.request_payload: artifacts.payload["valuation"] = calculate_portfolio_quantities( artifacts.payload, doc.request_payload ) if doc.owner_user_id is None: - raise ValueError("oracle portfolio jobs require an owner") + raise ValueError("portfolio provenance requires an owner") artifacts.payload["provenance"] = build_finance_provenance( job_id, doc.owner_user_id, doc.request_payload, artifacts.payload, workflow="portfolio-optimization", diff --git a/backend/src/quantum_backend_v2/application/peer_tasks.py b/backend/src/quantum_backend_v2/application/peer_tasks.py index 6217ea0d..02702282 100644 --- a/backend/src/quantum_backend_v2/application/peer_tasks.py +++ b/backend/src/quantum_backend_v2/application/peer_tasks.py @@ -2,7 +2,7 @@ Peers pull work from a shared list, so a fast laptop simply takes more tasks. A peer that fails (unreachable, missing a library, protocol unknown) is dropped for the rest of the call and its task -goes to another peer; whatever no peer could run is run on the coordinator, so a call always finishes. +goes to another peer; whatever no peer could run is run on the coordinator. """ from __future__ import annotations @@ -12,13 +12,16 @@ import os import time import uuid -from collections.abc import Callable, Hashable +from collections.abc import Callable, Coroutine, Hashable from dataclasses import dataclass -from typing import Any +from typing import TYPE_CHECKING, Any -from quantum_backend_v2.libp2p.protocol_ids import build_execution_protocol_ids from quantum_backend_v2.protocols.compute import ComputeTaskRequest, ComputeTaskResponse +if TYPE_CHECKING: + from quantum_backend_v2.application.filecoin_tasks import CheckpointCallback + from quantum_backend_v2.artifacts.filecoin import FilecoinReceipt, FilecoinStore + logger = logging.getLogger(__name__) LOCAL = "local" @@ -54,6 +57,8 @@ def __init__( def _protocol_id(self) -> str: if self._explicit_protocol_id is not None: return self._explicit_protocol_id + from quantum_backend_v2.libp2p.protocol_ids import build_execution_protocol_ids + namespace = self._discovery.settings.rendezvous_namespace return build_execution_protocol_ids(namespace).compute_task @@ -83,20 +88,44 @@ def label_of(self, peer_id: str) -> str: async def run( self, kind: str, - inputs: list[dict[str, Any]], + inputs: list[dict[str, Any]] | None, *, run_locally: Callable[[dict[str, Any]], dict[str, Any]], on_event: Callable[[str], None] = lambda message: None, on_task: TaskListener = lambda index, state, peer_id, detail: None, groups: list[Hashable] | None = None, + filecoin_store: FilecoinStore | None = None, + dataset_receipt: FilecoinReceipt | None = None, + checkpoint_receipt: FilecoinReceipt | None = None, + checkpoint_namespace: str | None = None, + on_checkpoint: CheckpointCallback | None = None, ) -> list[TaskOutcome | None]: """Run the inputs as `kind` tasks; outcomes come back in input order. Tasks that share a group are interchangeable (redundant starts of one fragment): the first to finish wins and the rest are abandoned, so a slow peer never holds the call up. An abandoned or unneeded task has no outcome (None). + + Filecoin is opt-in. Inputs are verified before dispatch; checkpoints resume + completed tasks, never an optimizer's partial iterations. The callback must + durably save each receipt; a failed upload or callback stops this call. """ + checkpoint_requested = any( + value is not None for value in (checkpoint_receipt, checkpoint_namespace, on_checkpoint) + ) + if (dataset_receipt is not None or checkpoint_requested) and filecoin_store is None: + raise ValueError("Filecoin task inputs/checkpoints require a configured store") + if checkpoint_requested and (checkpoint_namespace is None or on_checkpoint is None): + raise ValueError("Filecoin checkpoints require a namespace and receipt callback") + if filecoin_store is not None: + from quantum_backend_v2.application.filecoin_tasks import resolve_inputs + + inputs = await resolve_inputs(kind, inputs, filecoin_store, dataset_receipt) + if inputs is None: + raise ValueError("Task inputs are required") count = len(inputs) + if groups is not None and len(groups) != count: + raise ValueError("Task groups must have one entry per input") group_of: list[Hashable] = groups if groups is not None else list(range(count)) outcomes: list[TaskOutcome | None] = [None] * count attempts = [0] * count @@ -105,8 +134,33 @@ async def run( dropped: set[str] = set() inflight: dict[int, asyncio.Future[dict[str, Any]]] = {} done_groups: set[Hashable] = set() + journal = None + if checkpoint_requested: + from quantum_backend_v2.application.filecoin_tasks import CheckpointJournal + + assert filecoin_store is not None and checkpoint_namespace is not None + assert on_checkpoint is not None + journal = CheckpointJournal( + store=filecoin_store, + namespace=checkpoint_namespace, + kind=kind, + inputs=inputs, + groups=group_of, + on_checkpoint=on_checkpoint, + ) + if checkpoint_receipt is not None: + for index, saved in (await journal.restore(checkpoint_receipt)).items(): + outcomes[index] = TaskOutcome(**saved.model_dump()) + done_groups.add(group_of[index]) + pending = [index for index in pending if group_of[index] not in done_groups] for index in range(count): - on_task(index, "queued", None, None) + restored = outcomes[index] + if restored is not None: + on_task(index, "done", restored.executed_by, "restored from Filecoin checkpoint") + elif group_of[index] in done_groups: + on_task(index, "superseded", None, None) + else: + on_task(index, "queued", None, None) def finish_group(winner: int) -> None: done_groups.add(group_of[winner]) @@ -158,6 +212,8 @@ async def peer_loop(peer: Any) -> None: return finally: inflight.pop(index, None) + if group_of[index] in done_groups: + return outcomes[index] = TaskOutcome( output=output, executed_by=peer.peer_id, @@ -166,12 +222,24 @@ async def peer_loop(peer: Any) -> None: ) on_task(index, "done", peer.peer_id, None) finish_group(index) + if journal is not None: + await journal.save(outcomes) - # A drop re-queues a task after the other peers may have gone idle, so run passes until quiet. + async def gather_workers(work: list[Coroutine[Any, Any, None]]) -> None: + workers = [asyncio.create_task(item) for item in work] + try: + await asyncio.gather(*workers) + except BaseException: + for worker in workers: + worker.cancel() + await asyncio.gather(*workers, return_exceptions=True) + raise + + # A dropped peer can re-queue work after the others went idle, so run another pass. while pending: alive = [peer for peer in self.peers() if peer.peer_id not in dropped] waiting, dropped_before = len(pending), len(dropped) - await asyncio.gather(*(peer_loop(peer) for peer in alive)) + await gather_workers([peer_loop(peer) for peer in alive]) if len(pending) == waiting and len(dropped) == dropped_before: break @@ -192,6 +260,8 @@ async def local(index: int) -> None: on_task(index, "running", LOCAL, None) started = time.perf_counter() output = await asyncio.to_thread(run_locally, inputs[index]) + if group_of[index] in done_groups: + return outcomes[index] = TaskOutcome( output=output, executed_by=LOCAL, @@ -200,8 +270,10 @@ async def local(index: int) -> None: ) on_task(index, "done", LOCAL, None) finish_group(index) + if journal is not None: + await journal.save(outcomes) - await asyncio.gather(*(local(i) for i in leftovers)) + await gather_workers([local(i) for i in leftovers]) return outcomes async def _send(self, peer: Any, kind: str, task_input: dict[str, Any]) -> dict[str, Any]: diff --git a/backend/src/quantum_backend_v2/artifacts/filecoin.py b/backend/src/quantum_backend_v2/artifacts/filecoin.py new file mode 100644 index 00000000..32fdf809 --- /dev/null +++ b/backend/src/quantum_backend_v2/artifacts/filecoin.py @@ -0,0 +1,302 @@ +"""Calibration-only storage and portable research packages. + +The Node bridge owns SDK interaction. Python independently checks bytes before +they become inputs, checkpoints, or externally verifiable provenance. +""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import json +import os +import re +import uuid +from pathlib import Path, PurePosixPath +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +MAX_BYTES = 16 * 1024 * 1024 + + +class FilecoinBridgeError(OSError): + """Only validated public reconciliation metadata crosses the subprocess boundary.""" + + def __init__(self, error: dict[str, Any]) -> None: + code = error.get("code", "bridge_failed") + self.code = ( + code + if isinstance(code, str) and re.fullmatch(r"[a-z_]{1,64}", code) + else "bridge_failed" + ) + hashes = error.get("transaction_hashes", []) + self.transaction_hashes = ( + [ + value + for value in hashes + if isinstance(value, str) and re.fullmatch(r"0x[0-9a-fA-F]{64}", value) + ] + if isinstance(hashes, list) + else [] + ) + self.uncertain = error.get("uncertain") is not False or bool(self.transaction_hashes) + super().__init__(f"Filecoin operation failed ({self.code}); check its journal") + + +class FilecoinCopy(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + provider_id: str + data_set_id: str + piece_id: str + transaction_hash: str | None = None + provider_address: str | None = None + retrieval_url: str | None = None + + +class FilecoinReceipt(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + network: Literal["calibration"] = "calibration" + chain_id: Literal[314159] = 314159 + piece_cid: str = Field(min_length=10, max_length=256, pattern=r"^b[a-z2-7]+$") + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + size_bytes: int = Field(ge=1, le=MAX_BYTES, strict=True) + complete: bool = True + copies: tuple[FilecoinCopy, ...] = () + transaction_hashes: tuple[str, ...] = () + payer_address: str | None = None + + +class PackageFile(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + content_base64: str = Field(max_length=MAX_BYTES * 4 // 3 + 4) + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + size_bytes: int = Field(ge=0, le=MAX_BYTES, strict=True) + + def data(self) -> bytes: + value = base64.b64decode(self.content_base64, validate=True) + if ( + len(value) != self.size_bytes + or hashlib.sha256(value).hexdigest() != self.sha256 + or base64.b64encode(value).decode() != self.content_base64 + ): + raise ValueError("Package file bytes do not match the manifest") + return value + + @model_validator(mode="after") + def verified(self) -> PackageFile: + self.data() + return self + + +class Package(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + schema_version: Literal["qmesh-filecoin-package-v1"] = "qmesh-filecoin-package-v1" + kind: Literal["experiment", "dataset", "checkpoint"] + identity: str = Field(min_length=1, max_length=256) + files: dict[str, PackageFile] = Field(min_length=1, max_length=256) + + @model_validator(mode="after") + def safe_manifest(self) -> Package: + for name in self.files: + path = PurePosixPath(name) + if ( + not name + or "\\" in name + or ":" in name + or path.is_absolute() + or ".." in path.parts + or str(path) != name + ): + raise ValueError("Package path must be a normalized relative path") + if sum(item.size_bytes for item in self.files.values()) > MAX_BYTES: + raise ValueError("Package exceeds the maximum size") + return self + + @classmethod + def create(cls, *, kind: str, identity: str, files: dict[str, bytes]) -> Package: + return cls( + kind=kind, + identity=identity, + files={ + name: PackageFile( + content_base64=base64.b64encode(data).decode(), + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + for name, data in files.items() + }, + ) + + def to_bytes(self) -> bytes: + from quantum_backend_v2.provenance.hashing import canonical_json + + data = canonical_json(self.model_dump(mode="json")) + if len(data) > MAX_BYTES: + raise ValueError("Encoded package exceeds the maximum size") + return data + + +class FilecoinStore: + """One-shot bridge; writes are never retried after a timeout or lost response.""" + + def __init__( + self, + *, + bridge_path: Path | None = None, + node: str = "node", + env: dict[str, str] | None = None, + timeout_seconds: float = 600, + retention_days: int = 7, + max_monthly_usdfc: str = "0.25", + max_lockup_usdfc: str = "1", + ) -> None: + self.bridge_path = ( + bridge_path or Path(__file__).resolve().parents[4] / "services/filecoin/bridge.mjs" + ) + self.node = node + self.env = env + self.timeout_seconds = timeout_seconds + self.retention_days = retention_days + self.max_monthly_usdfc = max_monthly_usdfc + self.max_lockup_usdfc = max_lockup_usdfc + + async def command(self, command: str, **arguments: Any) -> dict[str, Any]: + request = json.dumps({"command": command, **arguments}).encode() + if len(request) > MAX_BYTES * 2: + raise ValueError("Filecoin request exceeds size limit") + env = dict(self.env if self.env is not None else os.environ) + journal = None + if command == "upload" or (command == "payments" and arguments.get("action") != "balance"): + directory = Path( + env.get( + "QMESH_FILECOIN_JOURNAL_DIR", + str(self.bridge_path.parent.parent.parent / "tmp/filecoin-journal"), + ) + ) + directory.mkdir(parents=True, exist_ok=True) + journal = directory / f"{uuid.uuid4().hex}.jsonl" + env["QMESH_FILECOIN_JOURNAL_PATH"] = str(journal.resolve()) + else: + env.pop("QMESH_FILECOIN_JOURNAL_PATH", None) + + def submitted_hashes() -> list[str]: + if journal is None or not journal.exists() or journal.stat().st_size > 65536: + return [] + hashes = [] + for line in journal.read_text().splitlines(): + try: + event = json.loads(line) + value = event.get("transaction_hash") + if isinstance(value, str) and re.fullmatch(r"0x[0-9a-fA-F]{64}", value): + hashes.append(value) + except (ValueError, AttributeError): + continue + return list(dict.fromkeys(hashes)) + + process = await asyncio.create_subprocess_exec( + self.node, + str(self.bridge_path), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + limit=MAX_BYTES * 2, + ) + try: + stdout, _ = await asyncio.wait_for(process.communicate(request), self.timeout_seconds) + except (TimeoutError, asyncio.CancelledError) as exc: + if process.returncode is None: + process.kill() + await process.wait() + if isinstance(exc, asyncio.CancelledError): + exc.transaction_hashes = submitted_hashes() + raise + raise FilecoinBridgeError( + { + "code": "confirmation_timeout", + "uncertain": True, + "transaction_hashes": submitted_hashes(), + } + ) from None + if process.returncode != 0: + try: + body = json.loads(stdout) if len(stdout) < 65536 else {} + error = body.get("error", {}) if isinstance(body, dict) else {} + except (ValueError, UnicodeDecodeError): + error = {} + error = error if isinstance(error, dict) else {} + hashes = error.get("transaction_hashes", []) + error["transaction_hashes"] = [ + *(hashes if isinstance(hashes, list) else []), + *submitted_hashes(), + ] + raise FilecoinBridgeError(error) + if len(stdout) > MAX_BYTES * 2: + raise ValueError("Filecoin response exceeds size limit") + response = json.loads(stdout) + if not isinstance(response, dict): + raise ValueError("Filecoin bridge returned an invalid response") + return response + + async def upload(self, data: bytes) -> FilecoinReceipt: + if not 127 <= len(data) <= MAX_BYTES: + raise ValueError( + "Filecoin uploads require 127 bytes to 16 MiB; package smaller artifacts" + ) + raw = await self.command( + "upload", + data_base64=base64.b64encode(data).decode(), + retention_days=self.retention_days, + max_monthly_usdfc=self.max_monthly_usdfc, + max_lockup_usdfc=self.max_lockup_usdfc, + copies=1, + ) + receipt = FilecoinReceipt.model_validate(raw) + if ( + not receipt.complete + or receipt.size_bytes != len(data) + or receipt.sha256 != hashlib.sha256(data).hexdigest() + ): + raise ValueError("Filecoin receipt does not match the upload") + try: + await self.download(receipt) + except (OSError, ValueError, TimeoutError) as exc: + error = FilecoinBridgeError( + { + "code": "post_upload_retrieval_failed", + "uncertain": True, + "transaction_hashes": list(receipt.transaction_hashes), + } + ) + error.submitted_receipt = receipt.model_dump(mode="json") + raise error from exc + return receipt + + async def download(self, receipt: FilecoinReceipt) -> bytes: + response = await self.command( + "download", + piece_cid=receipt.piece_cid, + sha256=receipt.sha256, + size_bytes=receipt.size_bytes, + payer_address=receipt.payer_address, + provider_address=(receipt.copies[0].provider_address if receipt.copies else None), + ) + data = base64.b64decode(response["data_base64"], validate=True) + if len(data) != receipt.size_bytes or hashlib.sha256(data).hexdigest() != receipt.sha256: + raise ValueError("Filecoin bytes do not match the receipt") + return data + + async def status(self, receipt: FilecoinReceipt) -> dict[str, Any]: + proof = await self.command( + "status", + piece_cid=receipt.piece_cid, + copies=[item.model_dump() for item in receipt.copies], + ) + try: + await self.download(receipt) + retrieval = "verified" + except (OSError, ValueError, TimeoutError): + retrieval = "unavailable_or_mismatched" + return {"network": "calibration", "retrieval": retrieval, "possession": proof} diff --git a/backend/src/quantum_backend_v2/bootstrap/application.py b/backend/src/quantum_backend_v2/bootstrap/application.py index b280730d..8d9695c2 100644 --- a/backend/src/quantum_backend_v2/bootstrap/application.py +++ b/backend/src/quantum_backend_v2/bootstrap/application.py @@ -12,12 +12,15 @@ ProvenanceAnchorService, RegistryAnchorAdapter, ) +from quantum_backend_v2.application.filecoin_archive import FilecoinBundleStorage +from quantum_backend_v2.application.filecoin_service import FilecoinConfig, FilecoinService from quantum_backend_v2.application.parity import ( CircuitJobService, FinancialJobService, OptionsJobService, RiskJobService, ) +from quantum_backend_v2.application.peer_tasks import PeerTaskDispatcher from quantum_backend_v2.application.pharma_jobs import PharmaJobService from quantum_backend_v2.application.price_sources import ChainlinkPriceSource from quantum_backend_v2.application.provenance import ( @@ -58,6 +61,17 @@ def create_application(env: Mapping[str, str] | None = None) -> FastAPI: mongo_runtime=persistence_runtime.mongodb, enforce_enrollment=settings.auth_required, ) + filecoin_config = FilecoinConfig.from_env(dict(env)) + filecoin_service = None + if filecoin_config.enabled: + if persistence_runtime.mongodb is None: + raise ValueError("Filecoin requires MongoDB for durable operation journals") + filecoin_service = FilecoinService( + persistence_runtime.mongodb.database, + filecoin_config, + dict(env), + dispatcher=PeerTaskDispatcher(discovery_service=discovery_service), + ) circuit_job_service = CircuitJobService( discovery_service=discovery_service, libp2p_runtime=libp2p_runtime, @@ -69,6 +83,7 @@ def create_application(env: Mapping[str, str] | None = None) -> FastAPI: ), ) financial_job_service = FinancialJobService( + filecoin_store=filecoin_service.reader if filecoin_service else None, price_source=ChainlinkPriceSource(chain_registry, env), history_source=ChainlinkHistorySource(load_market_data(), env), discovery_service=discovery_service, @@ -80,6 +95,8 @@ def create_application(env: Mapping[str, str] | None = None) -> FastAPI: reservation_service = ReservationService() runtime_recovery_service = RuntimeRecoveryService() storage_adapters = {KUBO_BACKEND: GatewayBundleStorage(settings.gateway)} + if filecoin_service is not None: + storage_adapters["filecoin"] = FilecoinBundleStorage(filecoin_service.reader) anchor_adapters = {} if settings.near_registry.configured: anchor_adapters[NEAR_CHAIN_ID] = NearRegistryAdapter(settings.near_registry) @@ -107,7 +124,8 @@ def create_application(env: Mapping[str, str] | None = None) -> FastAPI: max_batch=anchor.max_batch, ) provenance_service = ProvenanceService( - MongoProvenanceRepository(), storage_adapters=storage_adapters, + MongoProvenanceRepository(), + storage_adapters=storage_adapters, anchor_adapters=anchor_adapters, ) return create_app( @@ -126,4 +144,5 @@ def create_application(env: Mapping[str, str] | None = None) -> FastAPI: runtime_recovery_service=runtime_recovery_service, provenance_service=provenance_service, anchor_service=anchor_service, + filecoin_service=filecoin_service, ) diff --git a/backend/src/quantum_backend_v2/provenance/models.py b/backend/src/quantum_backend_v2/provenance/models.py index 4d519423..7d878274 100644 --- a/backend/src/quantum_backend_v2/provenance/models.py +++ b/backend/src/quantum_backend_v2/provenance/models.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator +from quantum_backend_v2.artifacts.filecoin import FilecoinReceipt from quantum_backend_v2.provenance.merkle import proof_length @@ -69,6 +70,15 @@ class StorageReference(BaseModel): cid: str = Field(min_length=1) url: str | None = None expires_at: datetime | None = None + filecoin: FilecoinReceipt | None = None + + @model_validator(mode="after") + def validate_filecoin(self) -> StorageReference: + if self.backend == "filecoin" and (self.filecoin is None or self.cid != self.filecoin.piece_cid): + raise ValueError("Filecoin storage requires a matching typed receipt") + if self.backend != "filecoin" and self.filecoin is not None: + raise ValueError("Filecoin receipt cannot describe another backend") + return self class ChainAnchor(BaseModel): diff --git a/backend/tests/unit/test_filecoin.py b/backend/tests/unit/test_filecoin.py new file mode 100644 index 00000000..35bcad18 --- /dev/null +++ b/backend/tests/unit/test_filecoin.py @@ -0,0 +1,124 @@ +"""Filecoin manifests and trusted subprocess boundary; no network in unit tests.""" + +import base64 +import hashlib +import json +import os +import sys + +import pytest +from pydantic import ValidationError + +from quantum_backend_v2.artifacts.filecoin import ( + FilecoinBridgeError, + FilecoinReceipt, + FilecoinStore, + Package, + PackageFile, +) + + +def receipt(data: bytes) -> FilecoinReceipt: + return FilecoinReceipt( + piece_cid="bafkzcibfixture", + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + copies=[], + ) + + +def test_package_roundtrip_and_file_hash_validation(): + package = Package.create( + kind="experiment", + identity="run-12345", + files={"input.json": b'{"circuit":"h 0"}', "results/output.json": b'{"0":32}'}, + ) + assert package.files["input.json"].data() == b'{"circuit":"h 0"}' + assert Package.model_validate_json(package.to_bytes()) == package + payload = json.loads(package.to_bytes()) + payload["files"]["input.json"]["content_base64"] = base64.b64encode(b"tamper").decode() + with pytest.raises(ValueError): + Package.model_validate(payload) + + +@pytest.mark.parametrize("path", ["../secret", "/abs", "C:/secret", "a\\b", "a/../b", ""]) +def test_package_rejects_unsafe_paths(path): + with pytest.raises(ValueError): + Package.create(kind="experiment", identity="run-12345", files={path: b"x"}) + + +def test_receipt_rejects_mainnet_and_bad_digest(): + with pytest.raises(ValidationError): + FilecoinReceipt(chain_id=314, piece_cid="bafkzcibfixture", sha256="a" * 64, size_bytes=1) + with pytest.raises(ValidationError): + FilecoinReceipt(piece_cid="bafkzcibfixture", sha256="bad", size_bytes=1) + + +@pytest.mark.anyio +async def test_download_validates_bytes_not_bridge_claim(): + class Boundary(FilecoinStore): + async def command(self, command, **kwargs): + return {"data_base64": base64.b64encode(b"wrong").decode()} + + store = Boundary() + with pytest.raises(ValueError, match="match"): + await store.download(receipt(b"right")) + + +@pytest.mark.anyio +async def test_upload_requires_retrieval_before_returning_receipt(): + data = b"input" * 32 + + class Boundary(FilecoinStore): + async def command(self, command, **kwargs): + if command == "upload": + return receipt(data).model_dump() + raise OSError("provider unavailable") + + with pytest.raises(OSError): + await Boundary().upload(data) + + +def test_noncanonical_base64_and_oversize_are_rejected(): + with pytest.raises(ValueError): + PackageFile(content_base64="!!!", sha256="0" * 64, size_bytes=0) + + +@pytest.mark.anyio +async def test_killed_write_preserves_public_transaction_journal(tmp_path): + script = tmp_path / "bridge.py" + tx = "0x" + "a" * 64 + script.write_text( + "import os,json,time\n" + "with open(os.environ['QMESH_FILECOIN_JOURNAL_PATH'],'a') as f:\n" + f" f.write(json.dumps({{'transaction_hash':'{tx}'}})+'\\n')\n" + " f.flush(); os.fsync(f.fileno())\n" + "time.sleep(10)\n" + ) + store = FilecoinStore( + bridge_path=script, + node=sys.executable, + timeout_seconds=1, + env={**os.environ, "QMESH_FILECOIN_JOURNAL_DIR": str(tmp_path / "journals")}, + ) + with pytest.raises(FilecoinBridgeError) as result: + await store.command("payments", action="deposit", amount_usdfc="0.1") + assert result.value.uncertain + assert result.value.transaction_hashes == [tx] + + +@pytest.mark.anyio +async def test_real_node_bridge_and_python_agree_on_exclusive_journal(tmp_path): + store = FilecoinStore( + env={ + **os.environ, + "QMESH_FILECOIN_PRIVATE_KEY": "", + "QMESH_FILECOIN_JOURNAL_DIR": str(tmp_path), + } + ) + if not (store.bridge_path.parent / "node_modules/@filoz/synapse-sdk").exists(): + pytest.skip("Install services/filecoin dependencies for subprocess contract check") + with pytest.raises(FilecoinBridgeError) as error: + await store.command("payments", action="deposit", amount_usdfc="0.1") + assert error.value.code == "key_required" + assert not error.value.uncertain diff --git a/backend/tests/unit/test_filecoin_api.py b/backend/tests/unit/test_filecoin_api.py new file mode 100644 index 00000000..0f3557d1 --- /dev/null +++ b/backend/tests/unit/test_filecoin_api.py @@ -0,0 +1,50 @@ +import pytest +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient + +from quantum_backend_v2.api.deps.auth import configure_auth +from quantum_backend_v2.api.errors import register_exception_handlers +from quantum_backend_v2.api.routers.filecoin import build_filecoin_router +from quantum_backend_v2.application.filecoin_service import FilecoinConfig + + +def test_config_cannot_enable_mainnet_or_unbounded_budget(): + with pytest.raises(ValueError): + FilecoinConfig(chain_id=314) + with pytest.raises(ValueError): + FilecoinConfig( + agents={ + "agent": { + "owner_id": "alice", + "key_env": "KEY", + "address": "0x" + "1" * 40, + "max_monthly_usdfc": "999999", + } + } + ) + + +@pytest.mark.anyio +async def test_storage_writes_require_explicit_publication_and_auth(): + class Service: + calls = 0 + + async def upload(self, *args, **kwargs): + self.calls += 1 + + service = Service() + app = FastAPI() + register_exception_handlers(app) + app.include_router(build_filecoin_router(service=service)) + configure_auth(enabled=True, allow_dev_bearer_tokens=True) + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + payload = {"agent_id": "agent", "operation_id": "test-write", "data_base64": "eA=="} + assert (await client.post("/api/v1/filecoin/artifacts", json=payload)).status_code == 401 + assert ( + await client.post( + "/api/v1/filecoin/artifacts", + json=payload, + headers={"Authorization": "Bearer dev-alice"}, + ) + ).status_code == 422 + assert service.calls == 0 diff --git a/backend/tests/unit/test_filecoin_archive.py b/backend/tests/unit/test_filecoin_archive.py new file mode 100644 index 00000000..90e76f67 --- /dev/null +++ b/backend/tests/unit/test_filecoin_archive.py @@ -0,0 +1,267 @@ +import hashlib +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest +from pymongo.errors import DuplicateKeyError + +from quantum_backend_v2.application.filecoin_archive import ( + FilecoinBundleStorage, + FilecoinOperations, +) +from quantum_backend_v2.artifacts.filecoin import FilecoinReceipt, Package +from quantum_backend_v2.provenance.models import StorageReference + + +@pytest.mark.anyio +async def test_archive_accepts_real_completed_run_and_preserves_commitment(monkeypatch): + from quantum_backend_v2.application.filecoin_archive import export_run, verify_export + from quantum_backend_v2.persistence import mongodb + from quantum_backend_v2.provenance.hashing import bundle_hash + from quantum_backend_v2.provenance.verification import RunEvidence, build_result_bundle + + evidence = RunEvidence( + run_id="run-12345", + workflow_definition_id="circuit", + owner_user_id="alice", + input_snapshot={"qasm": "h 0"}, + plan_id="plan-1", + result={"fragment_results": []}, + ) + bundle = build_result_bundle(evidence, created_at=datetime.now(timezone.utc)) + + class Run: + @classmethod + async def get(cls, key): + return SimpleNamespace( + id=evidence.run_id, + status="COMPLETED", + owner_user_id="alice", + workflow_definition_id="circuit", + input_snapshot=evidence.input_snapshot, + artifact_bundle_id="plan-1", + output_snapshot={"result": evidence.result}, + ) + + class Plan: + @classmethod + async def get(cls, key): + return SimpleNamespace(payload={"plan_id": "plan-1"}) + + class Bundles: + @classmethod + async def find_one(cls, query): + return SimpleNamespace( + payload=bundle.model_dump(mode="json"), content_hash=bundle_hash(bundle) + ) + + @classmethod + def get_pymongo_collection(cls): + return cls + + @classmethod + async def update_one(cls, *args): + pass + + class Store: + uploads = [] + + async def upload(self, data): + self.uploads.append(data) + return FilecoinReceipt( + piece_cid="bafkzcibfixture", + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + + monkeypatch.setattr(mongodb, "WorkflowRunDocument", Run) + monkeypatch.setattr(mongodb, "ExecutionPlanDocument", Plan) + monkeypatch.setattr(mongodb, "ProvenanceBundleDocument", Bundles) + store = Store() + await export_run(evidence.run_id, owner_id="alice", store=store) + report = await verify_export(store.uploads[0]) + assert all(check.status == "passed" for check in report.checks if check.kind == "hash") + + +@pytest.mark.anyio +async def test_archive_operation_replay_does_not_publish_again(monkeypatch): + from quantum_backend_v2.application import filecoin_service + + calls = [] + receipt = FilecoinReceipt(piece_cid="bafkzcibfixture", sha256="a" * 64, size_bytes=200) + + async def export(*args, **kwargs): + calls.append(1) + return receipt + + class Service(filecoin_service.FilecoinService): + async def account(self, *args): + return None, object() + + database = {"filecoin_operations": Collection(), "filecoin_locks": Collection()} + service = Service(database, filecoin_service.FilecoinConfig(), {}, dispatcher=None) + monkeypatch.setattr(filecoin_service, "export_run", export) + + async def prepare(*args, **kwargs): + return None, b"{}" + + monkeypatch.setattr(filecoin_service, "prepare_run_export", prepare) + args = dict(agent_id="agent", owner_id="alice", operation_id="operation1", run_id="run-12345") + assert await service.export(**args) == receipt + assert await service.export(**args) == receipt + assert len(calls) == 1 + + +class Collection: + def __init__(self): + self.rows = {} + + async def find_one(self, query): + return self.rows.get(query["_id"]) + + async def insert_one(self, row): + if row["_id"] in self.rows: + raise DuplicateKeyError("duplicate") + self.rows[row["_id"]] = row.copy() + + async def update_one(self, query, update): + self.rows[query["_id"]].update(update["$set"]) + return SimpleNamespace(matched_count=1) + + async def delete_one(self, query): + self.rows.pop(query["_id"], None) + + +@pytest.mark.anyio +async def test_safe_rejection_releases_lock_but_lost_transaction_keeps_evidence(): + from quantum_backend_v2.artifacts.filecoin import FilecoinBridgeError + + rows, locks = Collection(), Collection() + ops = FilecoinOperations(rows, locks) + + async def fail_safely(): + raise FilecoinBridgeError({"code": "funding_required", "uncertain": False}) + + args = dict(owner_id="alice", agent_id="agent", operation_id="try1", payload={}) + with pytest.raises(FilecoinBridgeError): + await ops.execute(**args, action=fail_safely) + assert not locks.rows + + async def now_funded(): + return {"ok": True} + + assert await ops.execute(**args, action=now_funded) == {"ok": True} + tx = "0x" + "a" * 64 + + async def lost_transaction(): + raise FilecoinBridgeError( + {"code": "confirmation_timeout", "uncertain": True, "transaction_hashes": [tx]} + ) + + with pytest.raises(FilecoinBridgeError): + await ops.execute(**{**args, "operation_id": "try2"}, action=lost_transaction) + assert locks.rows + assert any(row.get("transaction_hashes") == [tx] for row in rows.rows.values()) + + +@pytest.mark.anyio +async def test_failed_read_after_confirmed_upload_is_not_a_preflight_rejection(): + from quantum_backend_v2.artifacts.filecoin import FilecoinBridgeError, FilecoinStore + + data = b"x" * 256 + tx = "0x" + "b" * 64 + receipt = FilecoinReceipt( + piece_cid="bafkzcibfixture", + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + transaction_hashes=(tx,), + ) + + class Store(FilecoinStore): + async def command(self, command, **kwargs): + if command == "upload": + return receipt.model_dump(mode="json") + raise FilecoinBridgeError({"code": "read_failed", "uncertain": False}) + + rows, locks = Collection(), Collection() + operations = FilecoinOperations(rows, locks) + + async def action(): + return (await Store().upload(data)).model_dump(mode="json") + + with pytest.raises(FilecoinBridgeError): + await operations.execute( + owner_id="alice", agent_id="agent", operation_id="upload", payload={}, action=action + ) + row = next(iter(rows.rows.values())) + assert row["state"] == "uncertain" + assert row["transaction_hashes"] == [tx] + assert row["submitted_receipt"] == receipt.model_dump(mode="json") + assert locks.rows + + +@pytest.mark.anyio +async def test_write_is_idempotent_and_unknown_write_blocks_retry(): + journal, locks = Collection(), Collection() + operations = FilecoinOperations(journal, locks) + called = [] + + async def action(): + called.append(True) + return {"tx": "confirmed"} + + args = dict(owner_id="alice", agent_id="agent", operation_id="operation1", payload={"x": 1}) + assert await operations.execute(**args, action=action) == {"tx": "confirmed"} + assert await operations.execute(**args, action=action) == {"tx": "confirmed"} + assert len(called) == 1 + with pytest.raises(ValueError, match="different"): + await operations.execute(**{**args, "payload": {"x": 2}}, action=action) + + async def uncertain(): + raise TimeoutError() + + with pytest.raises(TimeoutError): + await operations.execute(**{**args, "operation_id": "operation2"}, action=uncertain) + with pytest.raises(ValueError, match="reconcil"): + await operations.execute(**{**args, "operation_id": "operation3"}, action=action) + assert len(called) == 1 + + +@pytest.mark.anyio +async def test_reference_cannot_substitute_another_package(): + package = Package.create(kind="dataset", identity="dataset", files={"data.csv": b"a,b\n1,2\n"}) + data = package.to_bytes() + receipt = FilecoinReceipt( + piece_cid="bafkzcibfixture", sha256=hashlib.sha256(data).hexdigest(), size_bytes=len(data) + ) + + class Store: + async def download(self, ref): + return data + + with pytest.raises(ValueError): + await FilecoinBundleStorage(Store()).retrieve( + StorageReference(backend="filecoin", cid=receipt.piece_cid, filecoin=receipt) + ) + + +@pytest.mark.anyio +async def test_invalid_export_does_not_freeze_agent(monkeypatch): + from quantum_backend_v2.application import filecoin_service + + class Service(filecoin_service.FilecoinService): + async def account(self, *args): + return None, object() + + async def missing(*args, **kwargs): + raise LookupError("Run not found") + + database = {"filecoin_operations": Collection(), "filecoin_locks": Collection()} + service = Service(database, filecoin_service.FilecoinConfig(), {}, dispatcher=None) + monkeypatch.setattr(filecoin_service, "prepare_run_export", missing) + with pytest.raises(LookupError): + await service.export( + agent_id="agent", owner_id="alice", operation_id="missing-run", run_id="missing" + ) + assert not database["filecoin_locks"].rows + assert not database["filecoin_operations"].rows diff --git a/backend/tests/unit/test_filecoin_finance.py b/backend/tests/unit/test_filecoin_finance.py new file mode 100644 index 00000000..acc7e1ef --- /dev/null +++ b/backend/tests/unit/test_filecoin_finance.py @@ -0,0 +1,153 @@ +"""Immutable CSV datasets pass verified bytes into the existing finance pipeline.""" +from __future__ import annotations + +import base64 +import hashlib +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient + +from quantum_backend_v2.api.deps.auth import configure_auth +from quantum_backend_v2.api.errors.models import register_exception_handlers +from quantum_backend_v2.api.routers.financial import build_financial_router +from quantum_backend_v2.application import parity +from quantum_backend_v2.application.financial_portfolio import PortfolioOptimizationConfig +from quantum_backend_v2.artifacts.filecoin import FilecoinReceipt, FilecoinStore +from quantum_backend_v2.provenance.hashing import hash_json + +CSV = b"date,A,B\n2026-01-01,10,20\n2026-01-02,11,19\n" + + +def receipt(): + return FilecoinReceipt(piece_cid="bafkzcibfixture", sha256=hashlib.sha256(CSV).hexdigest(), + size_bytes=len(CSV)) + + +class MemoryJob: + jobs = {} + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + self.error = None + self.created_at = self.updated_at = datetime.now(timezone.utc) + + async def insert(self): + self.jobs[self.id] = self + + async def save(self): + self.jobs[self.id] = self + + @classmethod + async def get(cls, key): + return cls.jobs.get(key) + + +class ReadBoundary(FilecoinStore): + data = CSV + + async def command(self, command, **kwargs): + assert command == "download" + return {"data_base64": base64.b64encode(self.data).decode()} + + +@pytest.fixture +def setup(monkeypatch): + MemoryJob.jobs = {} + monkeypatch.setattr(parity, "FinancialJobDocument", MemoryJob) + + async def ensure_user(_): + pass + + monkeypatch.setattr(parity, "_ensure_platform_user", ensure_user) + builder_calls = [] + + def build(**kwargs): + builder_calls.append(kwargs) + return SimpleNamespace(payload={"row_count": 2, "col_count": 2, "result": "computed"}) + + monkeypatch.setattr(parity, "build_portfolio_optimization_artifacts", build) + return builder_calls + + +@pytest.mark.anyio +async def test_filecoin_csv_is_verified_persisted_and_committed(setup): + store = ReadBoundary() + service = parity.FinancialJobService(discovery_service=None, libp2p_runtime=None, + filecoin_store=store) + doc = await service.submit_filecoin(owner_user_id="alice", filename="dataset.csv", + receipt=receipt(), config=PortfolioOptimizationConfig()) + assert doc.owner_user_id == "alice" + assert doc.request_payload["filecoin_dataset"] == receipt().model_dump(mode="json") + assert "csv_text" not in doc.request_payload + await service.process(job_id=doc.id, csv_bytes=b"must never be used") + assert doc.status == "completed", doc.error + assert setup[0]["csv_bytes"] == CSV + assert doc.request_payload["csv_text"] == CSV.decode() + bundle = doc.result_payload["provenance"]["bundle"] + assert bundle["input_hash"] == hash_json(doc.request_payload) + assert bundle["oracle_refs"] == [] + assert "valuation" not in doc.result_payload + stranger = SimpleNamespace(user_id="bob", is_admin=lambda: False) + assert await service.get_job(doc.id, current_user=stranger) is None + + +@pytest.mark.anyio +@pytest.mark.parametrize("failure", ["tampered", "unavailable", "disabled"]) +async def test_unverified_or_disabled_dataset_never_reaches_builder(setup, failure): + store = ReadBoundary() + service = parity.FinancialJobService(discovery_service=None, libp2p_runtime=None, + filecoin_store=store) + doc = await service.submit_filecoin(owner_user_id="alice", filename="dataset.csv", + receipt=receipt(), config=PortfolioOptimizationConfig()) + if failure == "tampered": + store.data = b"tampered" + elif failure == "unavailable": + async def unavailable(*args, **kwargs): + raise OSError("sensitive-provider-url") + store.command = unavailable + else: + service._filecoin_store = None + await service.process(job_id=doc.id, csv_bytes=CSV) + assert doc.status == "failed" + assert "sensitive-provider-url" not in doc.error + assert not setup + + +@pytest.mark.anyio +async def test_disabled_filecoin_rejects_submission(setup): + service = parity.FinancialJobService(discovery_service=None, libp2p_runtime=None) + with pytest.raises(ValueError, match="not configured"): + await service.submit_filecoin(owner_user_id="alice", filename="dataset.csv", + receipt=receipt(), config=PortfolioOptimizationConfig()) + assert not MemoryJob.jobs + + +@pytest.mark.anyio +async def test_filecoin_endpoint_auth_validation_and_owner(setup): + service = parity.FinancialJobService(discovery_service=None, libp2p_runtime=None, + filecoin_store=ReadBoundary()) + app = FastAPI() + register_exception_handlers(app) + app.include_router(build_financial_router(financial_job_service=service)) + configure_auth(enabled=True, allow_dev_bearer_tokens=True) + body = {"receipt": receipt().model_dump(mode="json"), "qaoa_reps": 2, "budget": 1} + try: + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + no_auth = await client.post("/api/v1/finance/filecoin-submit", json=body) + assert no_auth.status_code == 401 + headers = {"Authorization": "Bearer dev-alice"} + invalid = await client.post("/api/v1/finance/filecoin-submit", headers=headers, + json={**body, "qaoa_reps": 99}) + assert invalid.status_code == 422 + response = await client.post( + "/api/v1/finance/filecoin-submit", headers=headers, json=body + ) + assert response.status_code == 201, response.text + doc = MemoryJob.jobs[response.json()["job_id"]] + assert doc.owner_user_id == "alice" + assert setup[0]["config"].qaoa_reps == 2 + finally: + configure_auth(enabled=True, allow_dev_bearer_tokens=False) diff --git a/backend/tests/unit/test_filecoin_mongo.py b/backend/tests/unit/test_filecoin_mongo.py new file mode 100644 index 00000000..3ff9dcb2 --- /dev/null +++ b/backend/tests/unit/test_filecoin_mongo.py @@ -0,0 +1,44 @@ +"""Exercise durable atomic claims against MongoDB in CI, without network storage.""" + +import asyncio +import os +import uuid + +import pytest +from pymongo import AsyncMongoClient + +from quantum_backend_v2.application.filecoin_archive import FilecoinOperations + + +@pytest.mark.anyio +async def test_real_mongo_serializes_agent_writes_and_recovers_confirmed_lock(): + uri = os.getenv("QMESH_TEST_MONGODB_URI") + if not uri: + pytest.skip("Set QMESH_TEST_MONGODB_URI for real Mongo integration") + client = AsyncMongoClient(uri, serverSelectionTimeoutMS=5000) + name = "qmesh_filecoin_test_" + uuid.uuid4().hex + database = client[name] + try: + operations = FilecoinOperations(database.operations, database.locks) + started, finish = asyncio.Event(), asyncio.Event() + + async def first(): + started.set() + await finish.wait() + return {"confirmed": "receipt"} + + arguments = dict(owner_id="alice", agent_id="agent", operation_id="one", payload={}) + task = asyncio.create_task(operations.execute(**arguments, action=first)) + await started.wait() + with pytest.raises(ValueError, match="reconciliation"): + await operations.execute(**{**arguments, "operation_id": "two"}, action=first) + finish.set() + assert await task == {"confirmed": "receipt"} + row = await database.operations.find_one({"state": "confirmed"}) + await database.locks.insert_one({"_id": "agent", "operation_key": row["_id"]}) + await operations.recover_confirmed_lock("agent", "alice") + assert await database.locks.count_documents({}) == 0 + assert await operations.execute(**arguments, action=first) == {"confirmed": "receipt"} + finally: + await client.drop_database(name) + await client.close() diff --git a/backend/tests/unit/test_filecoin_task_service.py b/backend/tests/unit/test_filecoin_task_service.py new file mode 100644 index 00000000..58b77ed8 --- /dev/null +++ b/backend/tests/unit/test_filecoin_task_service.py @@ -0,0 +1,325 @@ +"""Task recovery uses durable receipts and fenced leases across coordinators.""" + +from __future__ import annotations + +import asyncio +import copy +import hashlib +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +import pytest +from pymongo.errors import DuplicateKeyError + +from quantum_backend_v2.application import compute_tasks, filecoin_service +from quantum_backend_v2.application.filecoin_service import FilecoinConfig, FilecoinService +from quantum_backend_v2.application.filecoin_tasks import CheckpointPersistenceError, create_dataset +from quantum_backend_v2.application.peer_tasks import PeerTaskDispatcher +from quantum_backend_v2.artifacts.filecoin import FilecoinReceipt + + +def matches(row, query): + for key, expected in query.items(): + if key == "$or": + if not any(matches(row, branch) for branch in expected): + return False + elif isinstance(expected, dict): + value = row.get(key) + for operator, argument in expected.items(): + if operator == "$in" and value not in argument: + return False + if operator == "$ne" and value == argument: + return False + if operator == "$lte" and (value is None or value > argument): + return False + if operator == "$gt" and (value is None or value <= argument): + return False + elif row.get(key) != expected: + return False + return True + + +class Collection: + def __init__(self): + self.rows = {} + self.crash_checkpoint = False + + async def find_one(self, query, *, sort=None): + rows = [copy.deepcopy(row) for row in self.rows.values() if matches(row, query)] + for key, direction in reversed(sort or []): + rows.sort(key=lambda row: row.get(key, ""), reverse=direction < 0) + return rows[0] if rows else None + + async def insert_one(self, row): + if row["_id"] in self.rows: + raise DuplicateKeyError("duplicate") + self.rows[row["_id"]] = copy.deepcopy(row) + + async def update_one(self, query, update): + row = next((row for row in self.rows.values() if matches(row, query)), None) + if row is None: + return SimpleNamespace(modified_count=0, matched_count=0) + if self.crash_checkpoint and "checkpoint" in update.get("$set", {}): + self.crash_checkpoint = False + raise asyncio.CancelledError() + row.update(copy.deepcopy(update.get("$set", {}))) + return SimpleNamespace(modified_count=1, matched_count=1) + + async def update_many(self, query, update): + count = 0 + for row in self.rows.values(): + if matches(row, query): + row.update(copy.deepcopy(update.get("$set", {}))) + count += 1 + return SimpleNamespace(modified_count=count) + + async def delete_one(self, query): + for key, row in list(self.rows.items()): + if matches(row, query): + self.rows.pop(key) + + +class Store: + def __init__(self): + self.data = {} + self.uploads = 0 + + def keep(self, data): + receipt = FilecoinReceipt( + piece_cid="bafkzcibfixture", + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + self.data[receipt.sha256] = data + return receipt + + async def upload(self, data): + self.uploads += 1 + return self.keep(data) + + async def download(self, receipt): + data = self.data[receipt.sha256] + assert hashlib.sha256(data).hexdigest() == receipt.sha256 + return data + + +class Service(FilecoinService): + async def account(self, *args): + return None, self.store + + +@pytest.fixture +def fixture(monkeypatch): + database = { + name: Collection() + for name in ("filecoin_task_runs", "filecoin_operations", "filecoin_locks") + } + store, calls = Store(), [] + + def compute(value): + calls.append(value) + return {"energy": -1.0} + + monkeypatch.setitem(compute_tasks.TASK_KINDS, "pharma.vqe", compute) + service = Service( + database, FilecoinConfig(), {}, dispatcher=PeerTaskDispatcher(discovery_service=None) + ) + service.store = store + inputs = [ + {"problem": {"n_qubits": 2, "n_electrons": 1, "terms": [["ZZ", 1.0]]}, "max_iterations": 3} + ] + dataset = store.keep(create_dataset("pharma.vqe", inputs)) + return service, database, dataset, calls + + +@pytest.mark.anyio +async def test_confirmed_checkpoint_survives_crash_before_task_callback(fixture): + service, database, dataset, calls = fixture + claim = await service.prepare_tasks( + run_id="restart", owner_id="alice", agent_id="agent", dataset=dataset + ) + database["filecoin_task_runs"].crash_checkpoint = True + with pytest.raises(asyncio.CancelledError): + await service.execute_tasks(claim) + row = next(iter(database["filecoin_task_runs"].rows.values())) + assert row["checkpoint"] is None + row["lease_until"] = datetime.now(timezone.utc) - timedelta(seconds=1) + await service.recover_tasks() + second = Service( + database, FilecoinConfig(), {}, dispatcher=PeerTaskDispatcher(discovery_service=None) + ) + second.store = service.store + claim = await second.prepare_tasks( + run_id="restart", owner_id="alice", agent_id="agent", dataset=dataset + ) + await second.execute_tasks(claim) + restored = await second.task("restart", "alice") + assert restored["state"] == "completed", restored + assert len(calls) == 1 + assert service.store.uploads == 1 + assert restored["checkpoint"] is not None + + +@pytest.mark.anyio +async def test_active_other_coordinator_is_not_reclaimed(fixture): + service, database, dataset, calls = fixture + claim = await service.prepare_tasks( + run_id="active", owner_id="alice", agent_id="agent", dataset=dataset + ) + await service.recover_tasks() + assert (await service.task("active", "alice"))["state"] == "running" + with pytest.raises(ValueError, match="running"): + await service.prepare_tasks( + run_id="active", owner_id="alice", agent_id="agent", dataset=dataset + ) + await service.execute_tasks(claim) + assert len(calls) == 1 + + +@pytest.mark.anyio +async def test_lease_lost_during_compute_cannot_pay_for_checkpoint(fixture, monkeypatch): + service, database, dataset, calls = fixture + claim = await service.prepare_tasks( + run_id="lost", owner_id="alice", agent_id="agent", dataset=dataset + ) + row = next(iter(database["filecoin_task_runs"].rows.values())) + + def loses_lease(value): + row["lease_token"] = "another-coordinator" + return {"energy": -1.0} + + monkeypatch.setitem(compute_tasks.TASK_KINDS, "pharma.vqe", loses_lease) + await service.execute_tasks(claim) + assert service.store.uploads == 0 + assert not database["filecoin_operations"].rows + + +@pytest.mark.anyio +async def test_safe_rejected_checkpoint_does_not_permanently_block_resume(fixture): + service, database, dataset, calls = fixture + await database["filecoin_operations"].insert_one( + { + "_id": "safe-rejected", + "state": "rejected", + "owner_id": "alice", + "agent_id": "agent", + "scope": "tasks:alice:funded", + } + ) + claim = await service.prepare_tasks( + run_id="funded", owner_id="alice", agent_id="agent", dataset=dataset + ) + await service.execute_tasks(claim) + assert (await service.task("funded", "alice"))["state"] == "completed" + assert len(calls) == 1 + + +@pytest.mark.anyio +async def test_expired_generation_cannot_run_after_new_claim(fixture): + service, database, dataset, calls = fixture + old_claim = await service.prepare_tasks( + run_id="expired", owner_id="alice", agent_id="agent", dataset=dataset + ) + row = next(iter(database["filecoin_task_runs"].rows.values())) + row["lease_until"] = datetime.now(timezone.utc) - timedelta(seconds=1) + new_claim = await service.prepare_tasks( + run_id="expired", owner_id="alice", agent_id="agent", dataset=dataset + ) + await service.execute_tasks(old_claim) + assert calls == [] + await service.execute_tasks(new_claim) + assert len(calls) == 1 + + +@pytest.mark.anyio +async def test_unknown_checkpoint_operation_blocks_resume_before_compute(fixture): + service, database, dataset, calls = fixture + await database["filecoin_operations"].insert_one( + { + "_id": "unknown", + "state": "started", + "owner_id": "alice", + "agent_id": "agent", + "scope": "tasks:alice:uncertain", + } + ) + claim = await service.prepare_tasks( + run_id="uncertain", owner_id="alice", agent_id="agent", dataset=dataset + ) + await service.execute_tasks(claim) + row = await service.task("uncertain", "alice") + assert row["state"] == "failed" + assert calls == [] + assert service.store.uploads == 0 + + +@pytest.mark.anyio +async def test_persistence_error_retains_last_confirmed_receipt(fixture): + service, database, dataset, calls = fixture + saved = service.store.keep(b"saved checkpoint receipt") + + class FailedDispatcher: + async def run(self, *args, **kwargs): + raise CheckpointPersistenceError(saved) + + service.dispatcher = FailedDispatcher() + claim = await service.prepare_tasks( + run_id="callback", owner_id="alice", agent_id="agent", dataset=dataset + ) + await service.execute_tasks(claim) + row = await service.task("callback", "alice") + assert row["state"] == "failed" + assert row["checkpoint"] == saved.model_dump(mode="json") + assert row["error"] == "CheckpointPersistenceError" + + +@pytest.mark.anyio +async def test_lease_loss_cancels_work_and_does_not_overwrite_new_coordinator(fixture, monkeypatch): + service, database, dataset, calls = fixture + ready = asyncio.Event() + original_sleep = asyncio.sleep + + async def fast_heartbeat(_seconds): + await ready.wait() + await original_sleep(0.01) + + monkeypatch.setattr(filecoin_service.asyncio, "sleep", fast_heartbeat) + + class ReplacedDispatcher: + async def run(self, *args, **kwargs): + row = next(iter(database["filecoin_task_runs"].rows.values())) + row["lease_token"] = "another-coordinator" + ready.set() + await asyncio.Event().wait() + + service.dispatcher = ReplacedDispatcher() + claim = await service.prepare_tasks( + run_id="lease", owner_id="alice", agent_id="agent", dataset=dataset + ) + with pytest.raises(asyncio.CancelledError): + await service.execute_tasks(claim) + row = next(iter(database["filecoin_task_runs"].rows.values())) + assert row["state"] == "running" + assert row["lease_token"] == "another-coordinator" + assert calls == [] + + +@pytest.mark.anyio +async def test_another_owners_checkpoint_is_never_restored(fixture): + service, database, dataset, calls = fixture + await database["filecoin_operations"].insert_one( + { + "_id": "foreign", + "state": "confirmed", + "owner_id": "bob", + "agent_id": "agent", + "scope": "tasks:alice:scoped", + "result": {"invalid": True}, + } + ) + claim = await service.prepare_tasks( + run_id="scoped", owner_id="alice", agent_id="agent", dataset=dataset + ) + await service.execute_tasks(claim) + assert (await service.task("scoped", "alice"))["state"] == "completed" + assert len(calls) == 1 diff --git a/backend/tests/unit/test_filecoin_tasks.py b/backend/tests/unit/test_filecoin_tasks.py new file mode 100644 index 00000000..0d9a6545 --- /dev/null +++ b/backend/tests/unit/test_filecoin_tasks.py @@ -0,0 +1,381 @@ +"""Durable task inputs and explicit task-boundary restart, without paid storage.""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import json +from types import SimpleNamespace + +import pytest + +from quantum_backend_v2.application.peer_tasks import PeerTaskDispatcher +from quantum_backend_v2.artifacts.filecoin import FilecoinReceipt, FilecoinStore +from quantum_backend_v2.protocols.compute import ComputeTaskRequest, ComputeTaskResponse + +pytestmark = pytest.mark.anyio + + +class MemoryStore(FilecoinStore): + def __init__(self): + self.data = {} + self.uploads = 0 + self.corrupt = False + self.fail_upload = False + + async def upload(self, data): + self.uploads += 1 + if self.fail_upload: + raise OSError("storage outcome unknown") + receipt = FilecoinReceipt( + piece_cid="bafkzcibfixture", + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + self.data[receipt.sha256] = data + return receipt + + async def command(self, command, **arguments): + assert command == "download" + data = b"corrupt" if self.corrupt else self.data[arguments["sha256"]] + return {"data_base64": base64.b64encode(data).decode()} + + +class Network: + def __init__(self, peer="peer-a"): + self.peer = SimpleNamespace(peer_id=peer, health_status="healthy", network_addresses=()) + self.registry = SimpleNamespace(list_peers=lambda include_stale=False: [self.peer]) + self.calls = [] + + async def request_peer_rpc(self, **kwargs): + request = ComputeTaskRequest.model_validate_json(kwargs["payload"]) + self.calls.append(request.input["n"]) + return ( + ComputeTaskResponse( + task_id=request.task_id, + executing_peer_id=self.peer.peer_id, + ok=True, + output={"double": request.input["n"] * 2}, + ) + .model_dump_json() + .encode() + ) + + def dispatcher(self): + return PeerTaskDispatcher( + discovery_service=self, protocol_id="/test", local_peer_id="coordinator" + ) + + +def no_local(_): + raise AssertionError("a checkpoint error must not rerun the task locally") + + +async def test_checkpoint_resume_skips_completed_inputs_on_a_replacement_peer(): + store = MemoryStore() + original = Network() + published = [] + + async def interrupt_after_saved(receipt): + published.append(receipt) + raise OSError("coordinator stopped after saving") + + with pytest.raises(RuntimeError, match="checkpoint"): + await original.dispatcher().run( + "test.double", + [{"n": 1}, {"n": 2}], + run_locally=no_local, + filecoin_store=store, + checkpoint_namespace="user/run-one", + on_checkpoint=interrupt_after_saved, + ) + assert original.calls == [1] + assert store.uploads == 1 + + replacement = Network("peer-replacement") + + async def save(receipt): + published.append(receipt) + + outcomes = await replacement.dispatcher().run( + "test.double", + [{"n": 1}, {"n": 2}], + run_locally=no_local, + filecoin_store=store, + checkpoint_namespace="user/run-one", + checkpoint_receipt=published[-1], + on_checkpoint=save, + ) + assert replacement.calls == [2] + assert [o.output["double"] for o in outcomes] == [2, 4] + assert [o.executed_by for o in outcomes] == ["peer-a", "peer-replacement"] + assert store.uploads == 2 + + +async def test_verified_dataset_supplies_the_actual_dispatched_inputs(): + from quantum_backend_v2.application.filecoin_tasks import create_dataset + + store, network = MemoryStore(), Network() + receipt = await store.upload(create_dataset("test.double", [{"n": 3}])) + result = await network.dispatcher().run( + "test.double", + None, + run_locally=no_local, + filecoin_store=store, + dataset_receipt=receipt, + ) + assert network.calls == [3] + assert result[0].output == {"double": 6} + assert store.uploads == 1 # reading a dataset does not enable checkpoint writes + + +@pytest.mark.parametrize("mismatch", ["input", "kind", "namespace", "groups"]) +async def test_checkpoint_identity_mismatch_prevents_any_execution(mismatch): + store, network = MemoryStore(), Network() + published = [] + + async def save(receipt): + published.append(receipt) + + await network.dispatcher().run( + "test.double", + [{"n": 1}, {"n": 2}], + run_locally=no_local, + filecoin_store=store, + checkpoint_namespace="user/run-one", + on_checkpoint=save, + ) + new_network = Network("new-peer") + with pytest.raises(ValueError, match="checkpoint"): + await new_network.dispatcher().run( + "test.other" if mismatch == "kind" else "test.double", + [{"n": 9}, {"n": 2}] if mismatch == "input" else [{"n": 1}, {"n": 2}], + run_locally=no_local, + filecoin_store=store, + checkpoint_namespace="user/run-two" if mismatch == "namespace" else "user/run-one", + checkpoint_receipt=published[-1], + on_checkpoint=save, + groups=["same", "same"] if mismatch == "groups" else None, + ) + assert new_network.calls == [] + + +async def test_corrupt_dataset_bytes_are_rejected_before_computation(): + from quantum_backend_v2.application.filecoin_tasks import create_dataset + + store, network = MemoryStore(), Network() + receipt = await store.upload(create_dataset("test.double", [{"n": 3}])) + store.corrupt = True + with pytest.raises(ValueError, match="match"): + await network.dispatcher().run( + "test.double", + None, + run_locally=no_local, + filecoin_store=store, + dataset_receipt=receipt, + ) + assert network.calls == [] + + +async def test_resume_keeps_group_winners_and_does_not_run_sibling_starts(): + store, network = MemoryStore(), Network() + published = [] + + async def interrupt(receipt): + published.append(receipt) + raise OSError("stop") + + task_inputs = [{"n": 1}, {"n": 2}, {"n": 3}] + with pytest.raises(RuntimeError, match="checkpoint"): + await network.dispatcher().run( + "test.double", + task_inputs, + run_locally=no_local, + groups=["a", "a", "b"], + filecoin_store=store, + checkpoint_namespace="user/group-run", + on_checkpoint=interrupt, + ) + + async def save(receipt): + published.append(receipt) + + replacement = Network("new-peer") + results = await replacement.dispatcher().run( + "test.double", + task_inputs, + run_locally=no_local, + groups=["a", "a", "b"], + filecoin_store=store, + checkpoint_namespace="user/group-run", + checkpoint_receipt=published[-1], + on_checkpoint=save, + ) + assert replacement.calls == [3] + assert results[0].output == {"double": 2} + assert results[1] is None + assert results[2].output == {"double": 6} + + +async def test_storage_failure_does_not_repeat_computation_or_upload(): + store, network = MemoryStore(), Network() + store.fail_upload = True + + async def save(_): + raise AssertionError("upload failed, so there is no receipt") + + with pytest.raises(RuntimeError, match="checkpoint"): + await network.dispatcher().run( + "test.double", + [{"n": 1}, {"n": 2}], + run_locally=no_local, + filecoin_store=store, + checkpoint_namespace="user/run-fails", + on_checkpoint=save, + ) + assert network.calls == [1] + assert store.uploads == 1 + + +async def test_local_fallback_receives_the_verified_dataset(): + from quantum_backend_v2.application.filecoin_tasks import create_dataset + + store = MemoryStore() + receipt = await store.upload(create_dataset("test.double", [{"n": 4}])) + calls = [] + + def run_local(value): + calls.append(value) + return {"double": value["n"] * 2} + + results = await PeerTaskDispatcher(discovery_service=None).run( + "test.double", + None, + run_locally=run_local, + filecoin_store=store, + dataset_receipt=receipt, + ) + assert calls == [{"n": 4}] + assert results[0].output == {"double": 8} + + +async def test_completed_resume_neither_computes_nor_uploads_again(): + store, network = MemoryStore(), Network() + published = [] + + async def save(receipt): + published.append(receipt) + + options = dict( + filecoin_store=store, + checkpoint_namespace="owner/completed", + on_checkpoint=save, + ) + await network.dispatcher().run("test.double", [{"n": 1}], run_locally=no_local, **options) + store.fail_upload = True + fresh = Network("new-peer") + outcomes = await fresh.dispatcher().run( + "test.double", + [{"n": 1}], + run_locally=no_local, + checkpoint_receipt=published[-1], + **options, + ) + assert outcomes[0].output == {"double": 2} + assert fresh.calls == [] + assert store.uploads == 1 + + +@pytest.mark.parametrize("invalid", ["index", "duplicate_group", "nonfinite_output"]) +async def test_hash_valid_but_invalid_checkpoint_is_rejected(invalid): + store, network = MemoryStore(), Network() + published = [] + + async def save(receipt): + published.append(receipt) + + options = dict( + filecoin_store=store, + checkpoint_namespace="owner/structure", + on_checkpoint=save, + groups=["one", "one"], + ) + await network.dispatcher().run( + "test.double", + [{"n": 1}, {"n": 2}], + run_locally=no_local, + **options, + ) + payload = json.loads(store.data[published[-1].sha256]) + if invalid == "index": + payload["completed"]["2"] = payload["completed"].pop("0") + elif invalid == "duplicate_group": + payload["completed"]["1"] = payload["completed"]["0"] + else: + payload["completed"]["0"]["output"] = {"energy": float("nan")} + malformed = await store.upload(json.dumps(payload).encode()) + fresh = Network() + with pytest.raises(ValueError): + await fresh.dispatcher().run( + "test.double", + [{"n": 1}, {"n": 2}], + run_locally=no_local, + checkpoint_receipt=malformed, + **options, + ) + assert fresh.calls == [] + + +async def test_receipt_callback_failure_exposes_uploaded_checkpoint_for_recovery(): + from quantum_backend_v2.application.filecoin_tasks import CheckpointPersistenceError + + store, network = MemoryStore(), Network() + + async def failed_callback(_): + raise OSError("database unavailable") + + with pytest.raises(CheckpointPersistenceError) as failure: + await network.dispatcher().run( + "test.double", + [{"n": 1}], + run_locally=no_local, + filecoin_store=store, + checkpoint_namespace="owner/callback", + on_checkpoint=failed_callback, + ) + assert failure.value.receipt is not None + assert json.loads(await store.download(failure.value.receipt))["completed"]["0"]["output"] == { + "double": 2 + } + assert store.uploads == 1 + + +async def test_concurrent_storage_failure_stops_other_checkpoint_uploads(): + class UncertainStore(MemoryStore): + async def upload(self, data): + self.uploads += 1 + await asyncio.sleep(0.01) + raise OSError("transaction outcome unknown") + + store, network = UncertainStore(), Network() + peers = [ + SimpleNamespace(peer_id=f"peer-{n}", health_status="healthy", network_addresses=()) + for n in range(3) + ] + network.registry.list_peers = lambda include_stale=False: peers + + async def save(_): + raise AssertionError("no confirmed upload") + + with pytest.raises(RuntimeError, match="checkpoint"): + await network.dispatcher().run( + "test.double", + [{"n": n} for n in range(6)], + run_locally=no_local, + filecoin_store=store, + checkpoint_namespace="owner/parallel", + on_checkpoint=save, + ) + assert sorted(network.calls) == [0, 1, 2] + assert store.uploads == 1 diff --git a/frontend/package.json b/frontend/package.json index 39e1e790..04df0f3c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,6 +18,7 @@ "test:vault-cid": "bun test tests/vault-cid-resolve.test.mjs", "test:vault-run-link": "bun test tests/vault-run-circuit-link.test.mjs", "test:vault-circuit": "bun test tests/vault-publish-circuit.test.mjs", + "test:filecoin": "bun test tests/filecoin.test.mjs tests/filecoin-workspace.test.mjs", "analyze": "ANALYZE=true bun run build" }, "dependencies": { diff --git a/frontend/src/app/(main)/vault/filecoin/page.tsx b/frontend/src/app/(main)/vault/filecoin/page.tsx new file mode 100644 index 00000000..6d8b56fa --- /dev/null +++ b/frontend/src/app/(main)/vault/filecoin/page.tsx @@ -0,0 +1,5 @@ +import { FilecoinVault } from "@/features/filecoin/filecoin-vault"; + +export default function FilecoinPage() { + return ; +} diff --git a/frontend/src/app/api/filecoin/[...path]/route.ts b/frontend/src/app/api/filecoin/[...path]/route.ts new file mode 100644 index 00000000..69deed99 --- /dev/null +++ b/frontend/src/app/api/filecoin/[...path]/route.ts @@ -0,0 +1,12 @@ +import { agentBackendFetch } from "@/features/agentkit/server/backend-fetch"; +import { proxyFilecoin } from "@/features/filecoin/server/proxy"; + +export const maxDuration = 600; +async function handle( + request: Request, + context: { params: Promise<{ path: string[] }> }, +) { + const { path } = await context.params; + return proxyFilecoin(request, path, agentBackendFetch); +} +export { handle as GET, handle as POST }; diff --git a/frontend/src/constants/navigation.ts b/frontend/src/constants/navigation.ts index 42f83359..91b3d159 100644 --- a/frontend/src/constants/navigation.ts +++ b/frontend/src/constants/navigation.ts @@ -176,6 +176,7 @@ export const NAV_CONFIG: RailItem[] = [ links: [ { label: "My Circuits", href: "/vault/my/circuits" }, { label: "My Runs", href: "/vault/my/runs" }, + { label: "Filecoin Storage", href: "/vault/filecoin" }, ], }, ], diff --git a/frontend/src/features/filecoin/filecoin-vault.tsx b/frontend/src/features/filecoin/filecoin-vault.tsx new file mode 100644 index 00000000..01435c5d --- /dev/null +++ b/frontend/src/features/filecoin/filecoin-vault.tsx @@ -0,0 +1,1088 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import Link from "next/link"; +import { + Archive, + ArrowUpRight, + Check, + FileUp, + Loader2, + RefreshCw, + Wallet, +} from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { PageHeader } from "@/shared/components/layout/page-header"; +import { useAuth } from "@/features/auth/hooks/use-auth"; +import { + FilecoinRequestError, + formatToken, + parseReceipt, + pendingKey, + readPending, + requestFilecoin, + transactionUrl, +} from "./model"; +import type { + AccountBalance, + FilecoinReceipt, + PendingWrite, + StorageAccount, + StoredArtifact, +} from "./model"; +import { ReceiptPanel } from "./receipt-panel"; + +type Intent = { + label: string; + detail: string; + path: string; + body: Record; +}; +type TaskRun = { + run_id: string; + state: string; + checkpoint?: FilecoinReceipt; + outcomes?: unknown[]; + error?: string; +}; +const selectClass = + "h-9 w-full rounded-lg border border-input bg-background px-3 text-sm outline-none focus-visible:ring-2 focus-visible:ring-ring"; +const panelClass = "rounded-xl border border-border bg-card p-5"; +const tabClass = + "min-h-8 data-[state=active]:bg-background data-[state=active]:text-foreground data-[state=active]:shadow-sm"; + +export function FilecoinVault() { + const { user, loading } = useAuth(); + return ( + + ); +} + +export function FilecoinWorkspace({ + userId, + authLoading = false, +}: { + userId: string | null; + authLoading?: boolean; +}) { + const [accounts, setAccounts] = useState([]); + const [artifacts, setArtifacts] = useState([]); + const [agentId, setAgentId] = useState(""); + const [balanceResult, setBalanceResult] = useState<{ + agentId: string; + value: AccountBalance; + } | null>(null); + const [loading, setLoading] = useState(!!userId); + const [available, setAvailable] = useState(false); + const [error, setError] = useState(""); + const [notice, setNotice] = useState(""); + const [pending, setPending] = useState(null); + const [storageReady, setStorageReady] = useState(false); + const submitting = useRef(false); + const [busy, setBusy] = useState(""); + const [intent, setIntent] = useState(null); + const [tab, setTab] = useState("archive"); + const [selected, setSelected] = useState(null); + const [runId, setRunId] = useState(""); + const [file, setFile] = useState(null); + const [consent, setConsent] = useState(false); + const [amount, setAmount] = useState("1"); + const [datasetId, setDatasetId] = useState(""); + const [taskId, setTaskId] = useState(""); + const [task, setTask] = useState(null); + const [taskConsent, setTaskConsent] = useState(false); + const [financeJob, setFinanceJob] = useState(""); + const [portfolioBudget, setPortfolioBudget] = useState("2"); + const [lastTx, setLastTx] = useState(""); + const account = accounts.find((item) => item.agent_id === agentId); + const balance = + available && balanceResult?.agentId === agentId + ? balanceResult.value + : null; + const canWrite = available && storageReady && !!account && !pending && !busy; + + const refresh = useCallback(() => { + if (!userId) return Promise.resolve(); + return Promise.all([ + requestFilecoin<{ accounts: StorageAccount[] }>("accounts"), + requestFilecoin<{ artifacts: StoredArtifact[] }>("artifacts"), + ]) + .then(([owned, history]) => { + setAccounts(owned.accounts); + setArtifacts(history.artifacts); + setAvailable(true); + setAgentId((current) => + owned.accounts.some((item) => item.agent_id === current) + ? current + : (owned.accounts[0]?.agent_id ?? ""), + ); + setError(""); + }) + .catch((e) => { + setAvailable(false); + setError( + e instanceof FilecoinRequestError && e.status === 404 + ? "Filecoin storage is not enabled on this deployment yet." + : e instanceof Error + ? e.message + : "Storage is unavailable.", + ); + }) + .finally(() => { + setLoading(false); + }); + }, [userId]); + + useEffect(() => { + void refresh(); + }, [refresh]); + useEffect(() => { + if (!userId) return; + const read = () => { + try { + setPending(readPending(localStorage, userId)); + setStorageReady(true); + } catch { + setStorageReady(false); + setError( + "Browser storage is unavailable. Enable it before submitting a storage transaction.", + ); + } + }; + read(); + window.addEventListener("storage", read); + return () => window.removeEventListener("storage", read); + }, [userId]); + useEffect(() => { + let live = true; + if (agentId && available) + void requestFilecoin( + `agents/${encodeURIComponent(agentId)}/balance`, + ) + .then((value) => { + if (live) setBalanceResult({ agentId, value }); + }) + .catch((e) => { + if (live) setError(e.message); + }); + return () => { + live = false; + }; + }, [agentId, available]); + + function clearPending() { + if (userId) localStorage.removeItem(pendingKey(userId)); + setPending(null); + } + async function sendWrite() { + if ( + !intent || + !account || + !userId || + pending || + submitting.current || + !storageReady + ) + return; + submitting.current = true; + const operation: PendingWrite = { + operation_id: crypto.randomUUID(), + agent_id: agentId, + label: intent.label, + started_at: new Date().toISOString(), + }; + setBusy(intent.label); + setError(""); + setNotice(""); + try { + // Persist the ID before sending; a closed tab must never turn a lost reply into a new payment. + const existing = readPending(localStorage, userId); + if (existing) { + setPending(existing); + setIntent(null); + return; + } + localStorage.setItem(pendingKey(userId), JSON.stringify(operation)); + setPending(operation); + const result = await requestFilecoin>( + intent.path, + { + ...intent.body, + agent_id: agentId, + operation_id: operation.operation_id, + }, + ); + if (result.piece_cid) { + setSelected(parseReceipt(result)); + setTab("proofs"); + } + if (typeof result.transaction_hash === "string") + setLastTx(result.transaction_hash); + clearPending(); + setIntent(null); + setNotice(`${intent.label} confirmed.`); + await refresh(); + setBalanceResult({ + agentId, + value: await requestFilecoin( + `agents/${encodeURIComponent(agentId)}/balance`, + ), + }); + } catch (e) { + if ( + e instanceof FilecoinRequestError && + [400, 401, 403, 404, 413, 415, 422].includes(e.status) + ) + clearPending(); + if (e instanceof FilecoinRequestError && e.status === 409) { + // A completed conflict response plus no journal entry means validation stopped before a write. + try { + await requestFilecoin( + `agents/${encodeURIComponent(agentId)}/operations/${operation.operation_id}`, + ); + } catch (lookup) { + if (lookup instanceof FilecoinRequestError && lookup.status === 404) + clearPending(); + } + } + setError( + e instanceof Error + ? e.message + : "The reply was lost. Check this operation before sending another.", + ); + setIntent(null); + } finally { + submitting.current = false; + setBusy(""); + } + } + async function checkPending() { + if (!pending) return; + setBusy("Checking operation"); + setError(""); + try { + const result = await requestFilecoin<{ + state: string; + result?: Record; + submitted_receipt?: FilecoinReceipt; + }>( + `agents/${encodeURIComponent(pending.agent_id)}/operations/${encodeURIComponent(pending.operation_id)}`, + ); + if (result.result?.piece_cid) setSelected(parseReceipt(result.result)); + if (result.submitted_receipt) + setSelected(parseReceipt(result.submitted_receipt)); + if (result.state === "confirmed" || result.state === "rejected") { + clearPending(); + setNotice( + result.state === "confirmed" + ? "The operation is confirmed." + : "The operation was rejected before storage was charged. You can correct it and try again.", + ); + await refresh(); + } else + setNotice( + "The operation still needs reconciliation. Its saved ID is unchanged; no new transaction was sent.", + ); + } catch (e) { + setError( + e instanceof Error ? e.message : "Operation status unavailable.", + ); + } finally { + setBusy(""); + } + } + async function prepareUpload() { + if (!file || !consent) return; + if (file.size < 127 || file.size > 16 * 1024 * 1024) { + setError( + "Choose a file between 127 bytes and 16 MiB. Package smaller files before publishing.", + ); + return; + } + setBusy("Reading file"); + setError(""); + try { + const data = await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onerror = () => + reject(new Error("The selected file could not be read.")); + reader.onload = () => resolve(String(reader.result).split(",")[1]); + reader.readAsDataURL(file); + }); + setIntent({ + label: "Publish artifact", + detail: `${file.name} (${file.size.toLocaleString()} bytes) will be publicly retrievable. Storage uses ${account?.agent_name ?? "this agent"}’s test-token allowance.`, + path: "artifacts", + body: { data_base64: data, publish_publicly: true }, + }); + } catch (e) { + setError(e instanceof Error ? e.message : "Unable to read file."); + } finally { + setBusy(""); + } + } + async function importReceipt(value?: File) { + if (!value) return; + try { + if (value.size > 64 * 1024) + throw new Error("Receipt JSON must be smaller than 64 KiB."); + setSelected(parseReceipt(JSON.parse(await value.text()))); + setError(""); + } catch (e) { + setError(e instanceof Error ? e.message : "Invalid receipt."); + } + } + function payment(action: string) { + if (!account) return; + if ( + ["deposit", "withdraw"].includes(action) && + (!/^\d+(\.\d{1,18})?$/.test(amount) || + Number(amount) <= 0 || + Number(amount) > Number(account.max_deposit_usdfc)) + ) { + setError( + `Enter an amount above zero and at most ${account.max_deposit_usdfc} test USDFC.`, + ); + return; + } + if (action === "terminate" && !/^\d+$/.test(datasetId)) { + setError("Enter the dataset ID whose storage should end."); + return; + } + const labels: Record = { + deposit: "Deposit test USDFC", + withdraw: "Withdraw test USDFC", + approve: "Authorize storage", + revoke: "Revoke storage allowance", + terminate: "End dataset storage", + }; + const detail = + action === "approve" + ? `Authorize up to ${account.max_monthly_usdfc} test USDFC per month and ${account.max_lockup_usdfc} locked test USDFC, with a 30-day lockup ceiling.` + : action === "terminate" + ? `End storage for dataset ${datasetId}. This affects every piece in that dataset.` + : action === "revoke" + ? "Prevent new storage commitments from using this allowance. Existing obligations may remain." + : `${action === "deposit" ? "Add" : "Withdraw"} ${amount} test USDFC ${action === "deposit" ? "to" : "from"} this agent’s Filecoin storage balance.`; + setIntent({ + label: labels[action], + detail, + path: "payments", + body: { + action, + ...(["deposit", "withdraw"].includes(action) + ? { amount_usdfc: amount } + : {}), + ...(action === "terminate" ? { data_set_id: datasetId } : {}), + }, + }); + } + async function runTasks() { + if (!selected || !taskId || !taskConsent) return; + setBusy("Submitting tasks"); + setError(""); + try { + const value = await requestFilecoin( + `tasks/${encodeURIComponent(taskId)}/run`, + { agent_id: agentId, dataset: selected, publish_publicly: true }, + ); + setTask(value); + setNotice( + "Task batch accepted. Refresh its status to see saved checkpoints.", + ); + } catch (e) { + setError( + e instanceof Error + ? e.message + : "Submission unavailable. Check the same run ID before resuming.", + ); + } finally { + setBusy(""); + } + } + async function taskStatus() { + setBusy("Checking tasks"); + setError(""); + try { + setTask( + await requestFilecoin(`tasks/${encodeURIComponent(taskId)}`), + ); + } catch (e) { + setError(e instanceof Error ? e.message : "Task status unavailable."); + } finally { + setBusy(""); + } + } + async function finance() { + if (!selected) return; + setBusy("Submitting finance"); + setError(""); + try { + const result = await requestFilecoin<{ job_id: string }>( + "finance-submit", + { receipt: selected, budget: Number(portfolioBudget) }, + ); + setFinanceJob(result.job_id); + } catch (e) { + setError( + e instanceof Error + ? e.message + : "Check finance history before submitting again.", + ); + } finally { + setBusy(""); + } + } + + return ( +
+ + + Calibration · test tokens + + +
+
+
+ + +
+ +
+ {!authLoading && !userId && ( +

+ + Sign in + {" "} + to view your Filecoin artifacts. +

+ )} + {!loading && available && !accounts.length && ( +
+

Connect a storage agent

+

+ Your account has no configured Filecoin wallet yet. Ask the + project operator to connect a Calibration wallet to one of your + agents. +

+ + Open agents + +
+ )} + {pending && ( +
+

+ {busy ? "Operation in progress" : "Check the previous operation"} +

+

+ {pending.label}. A reply may arrive after the transaction is + submitted; another write is paused until this is resolved. +

+

+ {pending.operation_id} +

+ +
+ )} + {error && ( +

+ {error} +

+ )} + {notice && ( +

+ + {notice} +

+ )} + + + + Publish + + + Saved evidence + + + Agent funding + + + Use a dataset + + + +
+
+

Archive a completed run

+

+ Preserve its inputs, results, execution plan and NEAR + provenance together. +

+ + setRunId(e.target.value)} + placeholder="Run ID from your lab" + /> + +
+
+

Publish a file or dataset

+

+ Store an experiment package, a finance CSV, or a pharma task + dataset. +

+ + setFile(e.target.files?.[0] ?? null)} + /> + +
+
+ +
+ +
+
+
+

Your stored research

+

+ Choose an artifact or open a receipt shared with you. +

+
+
+ + void importReceipt(e.target.files?.[0])} + /> +
+
+
+ {artifacts.length === 0 ? ( +

+ No storage operations yet. Publish a file to keep its + evidence here. +

+ ) : ( + artifacts.map((item) => { + const receipt = item.result?.piece_cid + ? item.result + : item.submitted_receipt; + return ( + + ); + }) + )} +
+
+ {selected && ( + + )} +
+ +
+
+ +

Agent storage wallet

+
+

+ {account?.address ?? "Select a configured agent"} +

+
+ + + + +
+ {account && ( +

+ Limits: {account.max_monthly_usdfc} test USDFC/month,{" "} + {account.max_lockup_usdfc} locked. Funding targets{" "} + {account.retention_days} days of runway; storage does not + expire automatically. +

+ )} +
+
+ + setAmount(e.target.value)} + /> +
+ + +
+ {lastTx && transactionUrl(lastTx) && ( + + View confirmed transaction + + )} +
+ + Withdraw or end storage + +
+ + +
+
+
+ + setDatasetId(e.target.value)} + placeholder="From its storage receipt" + /> +
+ +
+
+
+
+ +
+

Choose the saved inputs

+

+ Select an artifact in Saved evidence, or import its receipt + below. The input bytes are verified before execution. +

+ + void importReceipt(e.target.files?.[0])} + /> + {selected && ( +

+ {selected.piece_cid} +

+ )} +
+
+
+

Continue pharma tasks

+

+ Use a pharma task dataset. Reusing the same run ID restores + completed tasks; unfinished tasks are sent to available + workers. +

+ + setTaskId(e.target.value)} + placeholder="A name you can reuse to resume" + /> + +
+ + +
+ {task && ( +
+

+ {task.state.replaceAll("_", " ")} +

+ {task.error && ( +

{task.error}

+ )} +

+ {task.checkpoint + ? "A checkpoint is saved." + : "No checkpoint saved yet."} +

+ {task.checkpoint && ( + + )} +
+ )} +
+
+

Model a saved finance dataset

+

+ Run portfolio analysis on the exact CSV retained on Filecoin. + Storage preserves the data; it does not establish its market + source. +

+ + setPortfolioBudget(e.target.value)} + /> + + {financeJob && ( + + Open finance job + + )} + + Finance history + +
+
+
+
+ {!!busy && ( +

+ + {busy}… Keep this operation ID while confirmation is pending. +

+ )} +
+ { + if (!open && !busy) setIntent(null); + }} + > + + + {intent?.label} + {intent?.detail} + +
+

Filecoin Calibration · no real tokens

+

+ {account?.address} +

+
+ + + + +
+
+
+ ); +} + +function Amount({ + label, + value, + unit, +}: { + label: string; + value: string; + unit: string; +}) { + return ( +
+

{label}

+

{value}

+

{unit}

+
+ ); +} diff --git a/frontend/src/features/filecoin/model.ts b/frontend/src/features/filecoin/model.ts new file mode 100644 index 00000000..20cf7001 --- /dev/null +++ b/frontend/src/features/filecoin/model.ts @@ -0,0 +1,190 @@ +import { z } from "zod"; + +export const receiptSchema = z.object({ + network: z.literal("calibration"), + chain_id: z.literal(314159), + piece_cid: z.string().regex(/^b[a-z2-7]{9,255}$/), + sha256: z.string().regex(/^[0-9a-f]{64}$/), + size_bytes: z + .number() + .int() + .min(1) + .max(16 * 1024 * 1024), + complete: z.boolean(), + copies: z + .array( + z.object({ + provider_id: z.string(), + data_set_id: z.string(), + piece_id: z.string(), + transaction_hash: z.string().nullable().optional(), + provider_address: z.string().nullable().optional(), + retrieval_url: z.string().nullable().optional(), + }), + ) + .max(2), + transaction_hashes: z.array(z.string()), + payer_address: z.string().nullable().optional(), +}); +export type FilecoinReceipt = z.infer; +export function parseReceipt(value: unknown): FilecoinReceipt { + const parsed = receiptSchema.safeParse(value); + if (!parsed.success) + throw new Error("Choose a valid Filecoin Calibration receipt JSON file."); + return parsed.data; +} + +export type StorageAccount = { + agent_id: string; + agent_name?: string; + address: string; + max_monthly_usdfc: string; + max_lockup_usdfc: string; + max_deposit_usdfc: string; + retention_days: number; +}; +export type StoredArtifact = { + operation_id: string; + agent_id: string; + state: string; + created_at?: string; + result?: FilecoinReceipt; + submitted_receipt?: FilecoinReceipt | null; +}; +export type ProofCopy = { + registered?: boolean; + live?: boolean; + proof_status?: string; + provider_id?: string; + data_set_id?: string; +}; +export type StorageStatus = { + retrieval: string; + possession: { observed_at?: string; copies?: ProofCopy[] }; +}; +export type VerificationReport = { + status: string; + checks: { kind: string; name: string; status: string; message: string }[]; +}; +export type AccountBalance = { + tfil_base_units: string; + usdfc_base_units: string; + account: { + funds: string; + availableFunds: string; + lockupRatePerMonth: string; + }; + approval: { isApproved: boolean }; +}; +export type PendingWrite = { + operation_id: string; + agent_id: string; + label: string; + started_at: string; +}; + +export function pendingKey(userId: string) { + return `qmesh-filecoin-pending:${userId}`; +} +export function readPending( + storage: Pick, + userId: string, +): PendingWrite | null { + const value = storage.getItem(pendingKey(userId)); + if (!value) return null; + const parsed = z + .object({ + operation_id: z.string().min(8).max(128), + agent_id: z.string().min(1).max(128), + label: z.string(), + started_at: z.string(), + }) + .safeParse(JSON.parse(value)); + if (!parsed.success) + throw new Error( + "The saved storage operation is unreadable. Reconcile it before submitting another write.", + ); + return parsed.data; +} +export function formatToken(value?: string): string { + if (!value || !/^\d+$/.test(value)) return "—"; + const units = BigInt(value); + const scale = BigInt("1000000000000000000"); + if (units > BigInt(0) && units < BigInt("1000000000000")) return "<0.000001"; + const fraction = (units % scale) + .toString() + .padStart(18, "0") + .slice(0, 6) + .replace(/0+$/, ""); + return `${units / scale}${fraction ? `.${fraction}` : ""}`; +} +export function proofLabel(copy: ProofCopy): string { + if (!copy.registered) return "Piece not registered"; + if (!copy.live) return "Dataset inactive"; + if (copy.proof_status === "proven_this_period") return "Proven this period"; + if (copy.proof_status === "pending_this_period") + return "Awaiting this period’s proof"; + return "Proof unavailable"; +} +export function anchorLabel(report: VerificationReport): string { + const checks = report.checks.filter((check) => check.kind === "anchor"); + if ( + !checks.length || + checks.every((check) => check.status === "not_applicable") + ) + return "No NEAR anchor recorded"; + if (checks.some((check) => check.status === "failed")) + return "Anchor verification failed"; + if (checks.every((check) => check.status === "passed")) + return "Recorded anchors verified"; + return "Anchor verification unavailable"; +} +export function transactionUrl(hash: string): string | null { + return /^0x[0-9a-fA-F]{64}$/.test(hash) + ? `https://beryx.io/fil/calibration/txs/${hash}` + : null; +} + +export class FilecoinRequestError extends Error { + constructor( + message: string, + readonly status: number, + ) { + super(message); + } +} +export async function requestFilecoin( + path: string, + body?: unknown, +): Promise { + const response = await fetch(`/api/filecoin/${path}`, { + method: body === undefined ? "GET" : "POST", + cache: "no-store", + ...(body === undefined + ? {} + : { + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }), + }); + const result = await response.json().catch(() => null); + if (!response.ok) { + const detail = + typeof result?.detail === "string" ? result.detail : result?.error; + throw new FilecoinRequestError( + typeof detail === "string" + ? detail + : "The storage service could not complete this request.", + response.status, + ); + } + return result as T; +} +export function downloadBlob(blob: Blob, name: string) { + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = name; + anchor.click(); + setTimeout(() => URL.revokeObjectURL(url), 1000); +} diff --git a/frontend/src/features/filecoin/receipt-panel.tsx b/frontend/src/features/filecoin/receipt-panel.tsx new file mode 100644 index 00000000..795d2346 --- /dev/null +++ b/frontend/src/features/filecoin/receipt-panel.tsx @@ -0,0 +1,222 @@ +"use client"; + +import { useState } from "react"; +import { Download, ExternalLink, ShieldCheck } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { + anchorLabel, + downloadBlob, + proofLabel, + requestFilecoin, + transactionUrl, +} from "./model"; +import type { + FilecoinReceipt, + StorageStatus, + VerificationReport, +} from "./model"; + +export function ReceiptPanel({ receipt }: { receipt: FilecoinReceipt }) { + const [status, setStatus] = useState(null); + const [near, setNear] = useState(null); + const [busy, setBusy] = useState(""); + const [error, setError] = useState(""); + async function check(kind: "status" | "verify") { + setBusy(kind); + setError(""); + try { + if (kind === "status") + setStatus(await requestFilecoin(kind, receipt)); + else setNear(await requestFilecoin(kind, receipt)); + } catch (e) { + setError(e instanceof Error ? e.message : "Verification unavailable."); + } finally { + setBusy(""); + } + } + async function download() { + setBusy("download"); + setError(""); + try { + const response = await fetch("/api/filecoin/retrieve", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(receipt), + }); + if (!response.ok) + throw new Error( + "The file could not be retrieved and verified. No download was saved.", + ); + downloadBlob( + await response.blob(), + `qmesh-${receipt.sha256.slice(0, 12)}.bin`, + ); + } catch (e) { + setError(e instanceof Error ? e.message : "Download unavailable."); + } finally { + setBusy(""); + } + } + return ( +
+
+
+

+ Artifact evidence +

+

+ {receipt.size_bytes.toLocaleString()} bytes ·{" "} + {receipt.copies.length} storage{" "} + {receipt.copies.length === 1 ? "copy" : "copies"} +

+
+ +
+
+
+
Piece CID
+
+ {receipt.piece_cid} +
+
+
+
SHA256
+
+ {receipt.sha256} +
+
+
+
+ copy.registered) && + status.possession.copies.length > 0 + ? "Registered on Calibration" + : "Registration not verified" + : "Not checked" + } + /> + + + `Provider ${copy.provider_id}: ${proofLabel(copy)}`, + ) + .join(" · ") + : "Not checked" + } + /> + +
+ {status?.possession.observed_at && ( +

+ Observed {new Date(status.possession.observed_at).toLocaleString()} +

+ )} + {near && ( +
    + {near.checks.map((check, index) => ( +
  • + + {check.status.replaceAll("_", " ")} + {" "} + · {check.name.replaceAll("_", " ")}: {check.message} +
  • + ))} +
+ )} +
+ + + +
+ {error && ( +

+ {error} +

+ )} +
+ {receipt.transaction_hashes.map((hash) => { + const url = transactionUrl(hash); + return ( + url && ( + + View storage transaction + + ) + ); + })} +
+
+ ); +} + +function EvidenceLine({ title, detail }: { title: string; detail: string }) { + return ( +
+ {title} + {detail} +
+ ); +} diff --git a/frontend/src/features/filecoin/server/proxy.ts b/frontend/src/features/filecoin/server/proxy.ts new file mode 100644 index 00000000..9232fabe --- /dev/null +++ b/frontend/src/features/filecoin/server/proxy.ts @@ -0,0 +1,106 @@ +import { BACKEND } from "@/constants/backend"; + +const IDENTIFIER = "[a-zA-Z0-9_-]{1,128}"; +const READ = [ + /^accounts$/, + /^artifacts$/, + new RegExp(`^agents/${IDENTIFIER}/balance$`), + new RegExp(`^agents/${IDENTIFIER}/operations/${IDENTIFIER}$`), + new RegExp(`^tasks/${IDENTIFIER}$`), +]; +const WRITE = [ + /^(artifacts|retrieve|status|verify|payments|finance-submit)$/, + new RegExp(`^runs/${IDENTIFIER}/archive$`), + new RegExp(`^tasks/${IDENTIFIER}/run$`), +]; +const MAX_BODY = Math.ceil((16 * 1024 * 1024) / 3) * 4 + 16384; + +export async function proxyFilecoin( + request: Request, + segments: string[], + send: (url: string, init?: RequestInit) => Promise, +): Promise { + const path = segments.join("/"); + const allowed = + request.method === "GET" ? READ : request.method === "POST" ? WRITE : []; + if (!allowed.some((pattern) => pattern.test(path))) + return Response.json( + { error: "Unknown Filecoin operation." }, + { status: 404 }, + ); + if ( + request.method === "POST" && + request.headers.get("origin") !== new URL(request.url).origin + ) + return Response.json( + { error: "Use the signed-in application to submit this operation." }, + { status: 403 }, + ); + if ( + request.method === "POST" && + !request.headers.get("content-type")?.startsWith("application/json") + ) + return Response.json({ error: "JSON is required." }, { status: 415 }); + let body: string | undefined; + if (request.method === "POST") { + if (Number(request.headers.get("content-length")) > MAX_BODY) + return Response.json({ error: "File exceeds 16 MiB." }, { status: 413 }); + const reader = request.body?.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + if (reader) + while (true) { + const { value, done } = await reader.read(); + if (done) break; + size += value.length; + if (size > MAX_BODY) { + await reader.cancel(); + return Response.json( + { error: "File exceeds 16 MiB." }, + { status: 413 }, + ); + } + chunks.push(value); + } + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + body = new TextDecoder().decode(bytes); + try { + JSON.parse(body); + } catch { + return Response.json({ error: "Invalid JSON." }, { status: 400 }); + } + } + try { + const endpoint = + path === "finance-submit" + ? "finance/filecoin-submit" + : `filecoin/${path}`; + const upstream = await send(`${BACKEND.BASE_URL}/api/v1/${endpoint}`, { + method: request.method, + ...(body === undefined + ? {} + : { body, headers: { "Content-Type": "application/json" } }), + }); + return new Response(upstream.body, { + status: upstream.status, + headers: { + "Content-Type": + upstream.headers.get("content-type") ?? "application/json", + "Cache-Control": "private, no-store", + }, + }); + } catch { + return Response.json( + { + error: + "Filecoin service unavailable. Check the operation before submitting another write.", + }, + { status: 502 }, + ); + } +} diff --git a/frontend/src/features/finance/components/finance-page-client.tsx b/frontend/src/features/finance/components/finance-page-client.tsx index 091a0a5b..0c23cb3a 100644 --- a/frontend/src/features/finance/components/finance-page-client.tsx +++ b/frontend/src/features/finance/components/finance-page-client.tsx @@ -2,6 +2,7 @@ import { PortfolioUploadPanel } from "./portfolio-upload-panel"; import { TokenizedPortfolioPanel } from "./tokenized-portfolio-panel"; +import Link from "next/link"; export function FinancePageClient() { return ( @@ -15,6 +16,7 @@ export function FinancePageClient() {

+

Have a dataset on Filecoin? Use its saved inputs from the vault.

); } diff --git a/frontend/src/proxy.ts b/frontend/src/proxy.ts index ae24a5b7..f57826d6 100644 --- a/frontend/src/proxy.ts +++ b/frontend/src/proxy.ts @@ -10,6 +10,8 @@ const PUBLIC_PREFIX_PATHS = [ "/api/network/node-script", "/api/network/coordinator-info", "/api/agentkit", + // This API validates the full session before forwarding and returns JSON 401s. + "/api/filecoin", API.X402.ROOT, ]; diff --git a/frontend/tests/filecoin-workspace.test.mjs b/frontend/tests/filecoin-workspace.test.mjs new file mode 100644 index 00000000..c807d5f1 --- /dev/null +++ b/frontend/tests/filecoin-workspace.test.mjs @@ -0,0 +1,49 @@ +import { expect, mock, test } from "bun:test"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; + +// Only the test's Next navigation and session boundary are substituted. +// Production FilecoinVault still obtains its user from useAuth, and the proxy enforces it. +mock.module("next/link", () => ({ + default: ({ href, children, ...props }) => + createElement("a", { href, ...props }, children), +})); +mock.module("../src/features/auth/hooks/use-auth", () => ({ + useAuth: () => ({ user: null, loading: false }), +})); +const { FilecoinVault } = await import( + "../src/features/filecoin/filecoin-vault.tsx" +); +const { ReceiptPanel } = await import( + "../src/features/filecoin/receipt-panel.tsx" +); + +test("signed-out vault keeps storage mutations disabled and offers sign in", () => { + const html = renderToStaticMarkup(createElement(FilecoinVault)); + expect(html).toContain('href="/signin"'); + const publish = [ + ...html.matchAll(/]*)>([\s\S]*?)<\/button>/g), + ].find((button) => button[2].endsWith("Publish file")); + expect(publish?.[1]).toContain('disabled=""'); + expect(html).toContain("Calibration"); +}); + +test("a receipt alone is not rendered as a verified possession or NEAR proof", () => { + const html = renderToStaticMarkup( + createElement(ReceiptPanel, { + receipt: { + network: "calibration", + chain_id: 314159, + piece_cid: "bafkzcibfixture", + sha256: "a".repeat(64), + size_bytes: 256, + complete: true, + copies: [], + transaction_hashes: [], + }, + }), + ); + expect(html).toContain("Not checked"); + expect(html).not.toContain("Proven this period"); + expect(html).not.toContain("Recorded anchors verified"); +}); diff --git a/frontend/tests/filecoin.test.mjs b/frontend/tests/filecoin.test.mjs new file mode 100644 index 00000000..bf13b061 --- /dev/null +++ b/frontend/tests/filecoin.test.mjs @@ -0,0 +1,182 @@ +import { describe, expect, test } from "bun:test"; +import { + anchorLabel, + parseReceipt, + proofLabel, + formatToken, + pendingKey, + readPending, +} from "../src/features/filecoin/model.ts"; +import { proxyFilecoin } from "../src/features/filecoin/server/proxy.ts"; + +const receipt = { + network: "calibration", + chain_id: 314159, + piece_cid: "bafkzcibfixture", + sha256: "a".repeat(64), + size_bytes: 256, + complete: true, + copies: [], + transaction_hashes: [], +}; + +describe("Filecoin evidence", () => { + test("rejects mainnet and malformed evidence instead of displaying verified", () => { + expect(parseReceipt(receipt)).toEqual(receipt); + expect(() => parseReceipt({ ...receipt, chain_id: 314 })).toThrow(); + expect(() => parseReceipt({ ...receipt, sha256: "tampered" })).toThrow(); + }); + test("pending or unregistered possession never displays as proven", () => { + expect( + proofLabel({ + registered: true, + live: true, + proof_status: "proven_this_period", + }), + ).toBe("Proven this period"); + expect( + proofLabel({ + registered: true, + live: true, + proof_status: "pending_this_period", + }), + ).toBe("Awaiting this period’s proof"); + expect( + proofLabel({ + registered: false, + live: true, + proof_status: "proven_this_period", + }), + ).toBe("Piece not registered"); + }); + test("formats token units without precision loss and scopes pending writes to the account", () => { + expect(formatToken("1000000000000000000")).toBe("1"); + expect(formatToken("1234560000000000000")).toBe("1.23456"); + expect(formatToken("123456")).toBe("<0.000001"); + expect(formatToken(undefined)).toBe("—"); + expect(pendingKey("alice")).not.toBe(pendingKey("bob")); + }); + test("restores an uncertain operation after reload without replacing its ID", () => { + const pending = { + operation_id: "persisted-id", + agent_id: "a", + label: "Deposit", + started_at: "2026-11-11T00:00:00Z", + }; + const storage = { + getItem: (key) => + key === pendingKey("alice") ? JSON.stringify(pending) : null, + }; + expect(readPending(storage, "alice")).toEqual(pending); + expect(readPending(storage, "bob")).toBeNull(); + expect(() => + readPending({ getItem: () => '{"operation_id":"lost"}' }, "alice"), + ).toThrow(); + }); + test("an intact package with an unavailable anchor does not claim NEAR verification", () => { + expect( + anchorLabel({ + status: "partially_verified", + checks: [ + { + kind: "hash", + name: "content", + status: "passed", + message: "Intact", + }, + { + kind: "anchor", + name: "near", + status: "unavailable", + message: "RPC unavailable", + }, + ], + }), + ).toBe("Anchor verification unavailable"); + }); +}); + +describe("authenticated Filecoin proxy", () => { + test("rejects path traversal and unlisted operations without forwarding", async () => { + let calls = 0; + const send = async () => { + calls++; + return Response.json({}); + }; + expect( + ( + await proxyFilecoin( + new Request("https://app.test/api/filecoin/accounts"), + ["..", "wallet"], + send, + ) + ).status, + ).toBe(404); + expect( + ( + await proxyFilecoin( + new Request("https://app.test/api/filecoin/accounts"), + ["payments"], + send, + ) + ).status, + ).toBe(404); + expect(calls).toBe(0); + }); + test("rejects cross-origin writes and forwards the exact same operation ID", async () => { + const body = { + operation_id: "stable-id", + agent_id: "a", + action: "deposit", + amount_usdfc: "1", + }; + let calls = 0; + const send = async (url, init) => { + calls++; + expect(JSON.parse(init.body)).toEqual(body); + return Response.json({ transaction_hash: "tx" }); + }; + const request = (origin) => + new Request("https://app.test/api/filecoin/payments", { + method: "POST", + headers: { Origin: origin, "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + expect( + (await proxyFilecoin(request("https://evil.test"), ["payments"], send)) + .status, + ).toBe(403); + expect( + (await proxyFilecoin(request("https://app.test"), ["payments"], send)) + .status, + ).toBe(200); + expect(calls).toBe(1); + }); + test("preserves auth failures and binary retrieval", async () => { + const request = new Request("https://app.test/api/filecoin/retrieve", { + method: "POST", + headers: { + Origin: "https://app.test", + "Content-Type": "application/json", + }, + body: JSON.stringify(receipt), + }); + const binary = await proxyFilecoin( + request, + ["retrieve"], + async () => + new Response(new Uint8Array([0, 255, 42]), { + headers: { "Content-Type": "application/octet-stream" }, + }), + ); + expect([...new Uint8Array(await binary.arrayBuffer())]).toEqual([ + 0, 255, 42, + ]); + const denied = await proxyFilecoin( + new Request("https://app.test/api/filecoin/accounts"), + ["accounts"], + async () => Response.json({ error: "Sign in" }, { status: 401 }), + ); + expect(denied.status).toBe(401); + }); +}); diff --git a/frontend/tests/proxy-public-root.test.mjs b/frontend/tests/proxy-public-root.test.mjs index 7ac443e0..e4202600 100644 --- a/frontend/tests/proxy-public-root.test.mjs +++ b/frontend/tests/proxy-public-root.test.mjs @@ -29,4 +29,10 @@ describe("authentication proxy", () => { expect(response.status).toBe(307); }); + + test("lets the Filecoin API enforce its session without redirecting JSON requests to HTML", () => { + expect(proxy(request("/api/filecoin/accounts")).status).toBe(200); + expect(proxy(request("/api/filecoin-malicious/accounts")).status).toBe(307); + expect(proxy(request("/vault/filecoin")).status).toBe(307); + }); }); diff --git a/services/filecoin/README.md b/services/filecoin/README.md new file mode 100644 index 00000000..f7eb4eea --- /dev/null +++ b/services/filecoin/README.md @@ -0,0 +1,171 @@ +# QMeshPy Filecoin bridge + +This private Node CLI is the Python backend's single Filecoin SDK boundary. It uses +the official Synapse SDK 2.0.2 and Synapse Core 0.10.3, pinned by `package-lock.json`. +Install with `npm ci --ignore-scripts`, then run `node bridge.mjs`. Node 22 or newer +is required. Send one JSON object on stdin; receive one JSON object on stdout. +`npm test` checks validation, budget rejection, receipt handling and SDK imports. + +Only **Calibration (314159)** is supported. The bridge checks the RPC chain before +every operation, signs for Calibration, disables CDN, and never buys tokens or +executes SDK `prepare()` funding transactions (those grant unlimited approvals). +Use faucet tFIL and test USDFC. No deployed custom contract is required. + +Environment: + +- `QMESH_FILECOIN_PRIVATE_KEY`: dedicated Calibration EVM private key, supplied by + the operator's secret store; required for writes, never included in output. +- `QMESH_FILECOIN_EXPECTED_ADDRESS`: optional expected agent wallet; writes fail + if the configured signing key differs. The backend sets this for agent actions. +- `QMESH_FILECOIN_RPC_URL`: HTTPS Calibration RPC, default + `https://api.calibration.node.glif.io/rpc/v1`. +- `QMESH_FILECOIN_NETWORK`: if supplied, must be `calibration`. +- `QMESH_FILECOIN_JOURNAL_PATH`: optional unique, new file for this command's + public transaction events. Its parent directory must already exist. The + backend sets this inside its configured persistent journal directory. + +Do not expose this CLI directly to a browser. Authorization, agent policy, +idempotency and durable operation journaling belong to the Python caller. Use a +different signing key for each funded agent. A Base AgentKit wallet does not +automatically become a Filecoin account. + +## Upload and retrieve + +Upload requires already deposited test USDFC and a finite operator allowance: + +```json +{"command":"upload","data_base64":"...","copies":1,"retention_days":7,"max_monthly_usdfc":"0.25","max_lockup_usdfc":"1"} +``` + +Bytes must be canonical base64, between 127 bytes and 16 MiB. Package small +artifacts in a manifest envelope. `copies` is 1 or 2; `retention_days` is 1–30 and +sets a **funding runway target**, not an automatic deletion or termination date. +Storage remains subject to provider health, available funds and service status. +The bridge reads current upload costs, requires sufficient existing funding, +and rejects cost or existing operator allowances above the supplied limits. +It also reads the contract's current `defaultLockupPeriod` and rejects shorter +operator approvals before storing bytes or signing a commitment. At the current +Calibration deployment, this period is 86,400 epochs (30 days), independently of +a seven-day funding runway target. Small uploads also incur a per-dataset +monthly fee and lifecycle reserve, so their costs are not just per-byte charges. +It stores and commits once per selected provider, without retrying writes or +silently changing providers. Two copies can incur partial success: preserve an +error's transaction hashes and reconcile before attempting any new upload. + +Successful receipt: + +```json +{"network":"calibration","chain_id":314159,"piece_cid":"bafk...","payer_address":"0x...","sha256":"64 lowercase hex digits","size_bytes":1024,"complete":true,"copies":[{"provider_id":"1","provider_address":"0x...","data_set_id":"123","piece_id":"0","transaction_hash":"0x...","retrieval_url":"https://..."}],"transaction_hashes":["0x..."]} +``` + +Identifiers are decimal strings, transaction hashes EVM hex strings. The +transaction hash is the SDK's confirmed hash when available, and its receipt +must succeed before the copy is returned. The receipt's PieceCID is a Filecoin +piece commitment, **not an ordinary IPFS CID**. + +Key-free download: + +```json +{"command":"download","piece_cid":"bafk...","sha256":"...","size_bytes":1024,"provider_address":"0x...","payer_address":"0x..."} +``` + +Supply at least `provider_address` or `payer_address` from the receipt. Resolution +uses the SDK's registered provider lookup, never a caller-provided retrieval URL. +The response stream is capped at the expected size (maximum 16 MiB) and cancelled +as soon as it exceeds that limit. Retrieval requires HTTPS with a DNS hostname; +literal IP addresses, local hostnames and URL credentials are rejected. Up to +three redirects are followed within the registered provider's same HTTPS origin, +with every target validated; cross-origin redirects and loops are rejected. +The SDK computes the received bytes' PieceCID, and independent SHA256/size checks +must also pass. Output adds `data_base64`, `sha256`, `size_bytes`, `network`, and +`chain_id`. Deployments should apply network egress controls as DNS answers are +outside this URL validation boundary. + +## Storage status + +```json +{"command":"status","piece_cid":"bafk...","copies":[{"provider_id":"1","data_set_id":"123","piece_id":"0"}]} +``` + +Returns `observed_block`, `observed_at`, and `copies` with the supplied identifiers +plus `registered`, `live`, `service_end_epoch`, `next_challenge_epoch`, +`payer_address`, and `proof_status`. `registered` checks the exact PieceCID and +piece ID, provider binding and scheduled removals. `proof_status` is +`proven_this_period`, `pending_this_period`, or `unknown`; a pending current +period does not prove that a past proof deadline was missed. RPC/read failures +are errors, never an invented positive status. Reads observe current chain state; +they are not an atomic historical snapshot. PDP possession and successful +application retrieval are separate checks. + +## Explicit agent payments + +Every monetary value below is a **decimal string in test USDFC**, 18 decimals. +The hard bridge maximum is 10 test USDFC per request and total account funding; +monthly and lockup allowances may not exceed 10 test USDFC. The caller may impose +lower daily/agent budgets. Balance output uses integer base-unit strings. + +```json +{"command":"payments","action":"balance","payer_address":"0x..."} +{"command":"payments","action":"deposit","amount_usdfc":"1"} +{"command":"payments","action":"approve","max_monthly_usdfc":"0.25","max_lockup_usdfc":"1","max_lockup_days":30} +{"command":"payments","action":"withdraw","amount_usdfc":"0.01"} +{"command":"payments","action":"revoke"} +{"command":"payments","action":"terminate","data_set_id":"123"} +``` + +`balance` without a payer uses the configured signer address if present; otherwise +it reports the zero address, so callers should always pass the intended wallet. +`deposit` approves only the exact ERC20 amount needed. `approve` computes a finite +per-epoch rate from the monthly cap and sets a finite lockup amount/period. +`withdraw` can only withdraw unlocked funds. `revoke` prevents new operator +activity; it does not cancel existing rails. `terminate` uses the SDK's direct +wallet path and returns the actual service `end_epoch`; storage can continue +through its lockup period. Renew retention by explicitly depositing more test +USDFC before deficit; no scheduled refill or hidden real-money fallback exists. + +## Errors and uncertain writes + +Exit code 1 returns: + +```json +{"error":{"code":"operation_failed","message":"Filecoin operation failed; inspect recorded transaction hashes before retrying a write","uncertain":true,"transaction_hashes":["0x..."]}} +``` + +Errors deliberately omit SDK diagnostic strings, keys and signed payloads. +Safe diagnostics include `operation_stage` (`preflight`, `store`, `commit`, +`receipt`, or `payment`), a bounded list of local error class names in +`error_types`, and numeric `http_statuses` when present in the error's cause +chain. These fields never include error messages or request/response bodies. +`funding_required` additionally includes `deposit_needed_base_units`. +An approval with insufficient duration includes `required_lockup_epochs`. +`uncertain:true` can occur with no known hash if a send failed after submission; +**never automatically retry such an operation**. Save the request ID and result +durably and reconcile wallet/provider/on-chain state. Process termination or a +caller timeout during a write must also be journaled as uncertain. +Preflight validation, funding/allowance rejection and RPC failures before any +write-capable SDK call return `uncertain:false` with no transaction hashes. +After provider storage starts or a payment call can submit, failures remain +uncertain even when the provider or RPC returned no hash. +An empty on-chain dataset list does not exclude a signed commitment still +pending at a provider. Do not increase approval limits to retry an uncertain +commitment without reconciling that provider operation or isolating the signer. + +When a journal path is configured, the bridge creates it exclusively and appends +NDJSON events with `fsync` before continuing. `{"type":"write_started"}` is +flushed before each write attempt. Every available transaction hash is flushed +as `{"type":"transaction_submitted","transaction_hash":"0x..."}` before +its receipt wait, including the intermediate ERC20 approval within a deposit. +No private keys, signed payloads or request data enter the journal. Existing +files are never reused or truncated. Journal failures before the first write +stop submission; failures after submission remain uncertain. The final stdout +protocol stays one JSON object. Preserve this journal after timeout/process +termination so transaction reconciliation does not depend on process memory. + +Depositing can require two transactions: the SDK's ERC20 approval wait defaults +to 180 seconds, followed by a separate deposit with a 120-second receipt wait. +These waits are sequential, plus RPC/simulation time. A slow command is not +evidence that no transaction was sent; use its journal and fresh chain reads. + +References: [SDK](https://docs.filecoin.cloud/developer-guides/synapse/), +[upload lifecycle](https://docs.filecoin.cloud/developer-guides/storage/upload-pipeline/), +[payments](https://docs.filecoin.cloud/developer-guides/payments/payment-operations/). diff --git a/services/filecoin/bridge.mjs b/services/filecoin/bridge.mjs new file mode 100644 index 00000000..072235dc --- /dev/null +++ b/services/filecoin/bridge.mjs @@ -0,0 +1,393 @@ +import { createHash } from 'node:crypto'; +import { openSync, writeSync, fsyncSync, closeSync } from 'node:fs'; +import { isIP } from 'node:net'; +import { pathToFileURL } from 'node:url'; +import { Synapse, calibration, TOKENS } from '@filoz/synapse-sdk'; +import * as Piece from '@filoz/synapse-core/piece'; +import * as PDP from '@filoz/synapse-core/pdp-verifier'; +import * as WarmStorage from '@filoz/synapse-core/warm-storage'; +import { http, isAddress, parseUnits, zeroAddress } from 'viem'; +import { privateKeyToAccount } from 'viem/accounts'; + +export const MAX_BYTES = 16 * 1024 * 1024; +const MAX_USDFC = 10n * 10n ** 18n; +const EPOCHS_DAY = 2880n; +const EPOCHS_MONTH = 86400n; +const NETWORK = {network:'calibration', chain_id:314159}; +const MUTATIONS = new Set(['upload','deposit','approve','withdraw','revoke','terminate']); +const sha256 = bytes => createHash('sha256').update(bytes).digest('hex'); + +function fail(code, message, details={}) { + const error = new Error(message); + Object.assign(error, {code, details, safe:true}); + throw error; +} +function integer(value, min, max, name) { + if (!Number.isSafeInteger(value) || value < min || value > max) fail('invalid_request',`Invalid ${name}`); + return value; +} +function id(value) { + if (typeof value !== 'string' || !/^(0|[1-9][0-9]{0,76})$/.test(value)) fail('invalid_request','Invalid decimal identifier'); + return BigInt(value); +} +function amount(value) { + if (typeof value !== 'string' || !/^(0|[1-9][0-9]*)(\.[0-9]{1,18})?$/.test(value)) fail('invalid_request','Use a decimal USDFC string'); + const parsed=parseUnits(value,18); + if (parsed <= 0n || parsed > MAX_USDFC) fail('invalid_request','Amount must be greater than zero and at most 10 test USDFC'); + return parsed; +} + +export function parseRequest(input) { + if (input == null || Array.isArray(input) || typeof input !== 'object') fail('invalid_request','Expected a JSON object'); + if ((input.network != null && input.network !== 'calibration') || (input.chain_id != null && input.chain_id !== 314159)) { + fail('wrong_network','Only Filecoin Calibration is supported'); + } + const request={...input}; + if (request.payer_address != null && !isAddress(request.payer_address)) fail('invalid_request','Invalid payer address'); + if (request.command === 'upload') { + if (typeof request.data_base64 !== 'string' || request.data_base64.length > Math.ceil(MAX_BYTES/3)*4 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(request.data_base64)) { + fail('invalid_request','Invalid or oversized base64 payload'); + } + request.bytes=Buffer.from(request.data_base64,'base64'); + integer(request.bytes.length,127,MAX_BYTES,'payload size'); + if (request.bytes.toString('base64') !== request.data_base64) fail('invalid_request','Non-canonical base64 payload'); + request.copies=integer(request.copies ?? 2,1,2,'copies'); + request.retention_days=integer(request.retention_days,1,30,'retention_days'); + request.monthly=amount(request.max_monthly_usdfc); + request.lockup=amount(request.max_lockup_usdfc); + } else if (request.command === 'download') { + integer(request.size_bytes,127,MAX_BYTES,'size_bytes'); + if (!/^[0-9a-f]{64}$/.test(request.sha256 ?? '')) fail('invalid_request','Expected SHA256 digest'); + const piece=Piece.tryFrom(request.piece_cid); + if (!piece || piece.size !== request.size_bytes) fail('invalid_request','PieceCID and size disagree'); + if (request.provider_address != null && !isAddress(request.provider_address)) fail('invalid_request','Invalid provider address'); + if (!request.provider_address && !request.payer_address) fail('invalid_request','Supply provider_address or payer_address'); + } else if (request.command === 'status') { + if (!Array.isArray(request.copies) || request.copies.length < 1 || request.copies.length > 2) fail('invalid_request','Expected one or two copies'); + if (!Piece.tryFrom(request.piece_cid)) fail('invalid_request','Invalid PieceCID'); + for (const copy of request.copies) {id(copy.provider_id);id(copy.data_set_id);id(copy.piece_id);} + } else if (request.command === 'payments') { + if (request.action === 'approve') { + request.monthly=amount(request.max_monthly_usdfc); + request.lockup=amount(request.max_lockup_usdfc); + request.days=integer(request.max_lockup_days,1,30,'max_lockup_days'); + } else if (request.action === 'deposit' || request.action === 'withdraw') request.amount=amount(request.amount_usdfc); + else if (request.action === 'terminate') request.dataset=id(request.data_set_id); + else if (!['balance','revoke'].includes(request.action)) fail('invalid_request','Unknown payment action'); + } else fail('invalid_request','Unknown command'); + return request; +} + +export function makeSynapse(request, env=process.env) { + if (env.QMESH_FILECOIN_NETWORK && env.QMESH_FILECOIN_NETWORK !== 'calibration') fail('wrong_network','Only Filecoin Calibration is supported'); + const needsKey=MUTATIONS.has(request.command) || MUTATIONS.has(request.action); + const privateKey=env.QMESH_FILECOIN_PRIVATE_KEY; + if (needsKey && !/^0x[0-9a-fA-F]{64}$/.test(privateKey ?? '')) fail('key_required','Configure a Calibration signing key'); + const rpc=env.QMESH_FILECOIN_RPC_URL ?? 'https://api.calibration.node.glif.io/rpc/v1'; + let url; + try {url=new URL(rpc);} catch {fail('invalid_config','Invalid RPC URL');} + if (url.protocol !== 'https:' || url.username || url.password) fail('invalid_config','RPC must use HTTPS without URL credentials'); + const keyAccount=/^0x[0-9a-fA-F]{64}$/.test(privateKey ?? '') ? privateKeyToAccount(privateKey) : null; + if (needsKey && env.QMESH_FILECOIN_EXPECTED_ADDRESS && + keyAccount.address.toLowerCase() !== env.QMESH_FILECOIN_EXPECTED_ADDRESS.toLowerCase()) { + fail('wrong_signer','Configured signer does not match the expected agent wallet'); + } + if (needsKey && request.payer_address && keyAccount.address.toLowerCase() !== request.payer_address.toLowerCase()) { + fail('wrong_signer','Requested payer does not match the configured signing key'); + } + const account=needsKey ? keyAccount : (request.payer_address ?? keyAccount?.address ?? zeroAddress); + return Synapse.create({account,chain:calibration,transport:http(rpc,{retryCount:0,timeout:30000}), + source:'qmeshpy',withCDN:false,pieceBatching:false}); +} + +export function createJournal(path) { + const descriptor=openSync(path,'ax',0o600); + return { + append(event) { + const bytes=Buffer.from(JSON.stringify(event)+'\n'); + let offset=0; + while(offsetcloseSync(descriptor), + }; +} + +async function confirmed(sdk, hash, recordHash) { + recordHash(hash); + const receipt=await sdk.readClient.waitForTransactionReceipt({hash,confirmations:1,timeout:120000}); + if (receipt.status !== 'success') fail('transaction_reverted','Calibration transaction reverted',{transaction_hash:hash}); + return {...NETWORK,transaction_hash:hash,block_number:receipt.blockNumber.toString()}; +} + +async function upload(request,sdk,hashes,startWrite,recordHash,setStage) { + const contexts=await sdk.storage.createContexts({copies:request.copies,metadata:{application:'qmeshpy'}}); + const costs=await sdk.storage.calculateMultiContextCosts(contexts,{pieceSizes:[BigInt(request.bytes.length)], + extraRunwayEpochs:BigInt(request.retention_days)*EPOCHS_DAY}); + if (costs.depositNeeded > 0n) fail('funding_required','Deposit test USDFC explicitly before upload', + {deposit_needed_base_units:costs.depositNeeded.toString()}); + if (costs.rates.perMonth > request.monthly || costs.lockups.total > request.lockup) { + fail('budget_exceeded','Quoted storage cost exceeds the requested budget'); + } + const priceList=await sdk.readClient.readContract({ + address:calibration.contracts.fwssView.address,abi:calibration.contracts.fwssView.abi, + functionName:'getPriceList'}); + const requiredLockup=priceList.lockups.defaultLockupPeriod; + const approval=await sdk.payments.serviceApproval(); + if (!approval.isApproved || approval.rateAllowance > MAX_USDFC/EPOCHS_MONTH || approval.lockupAllowance > MAX_USDFC || + approval.rateAllowance > request.monthly/EPOCHS_MONTH || approval.lockupAllowance > request.lockup || + approval.rateUsage+costs.lockups.rateDeltaPerEpoch > approval.rateAllowance || + approval.lockupUsage+costs.lockups.total > approval.lockupAllowance || + approval.maxLockupPeriod > 30n*EPOCHS_DAY) { + fail('approval_required','Configure a sufficient finite operator approval within the upload budget'); + } + if (approval.maxLockupPeriod < requiredLockup) { + fail('approval_required','Operator lockup period is shorter than the current storage contract requirement', + {required_lockup_epochs:requiredLockup.toString()}); + } + const copies=[]; + let pieceCid; + for (const context of contexts) { + // Explicit contexts avoid the high-level SDK's replacement-provider retries. + setStage('store'); + startWrite(); + const stored=await context.store(request.bytes); + if (stored.size !== request.bytes.length || (pieceCid && stored.pieceCid.toString() !== pieceCid)) { + fail('integrity_error','Provider returned an inconsistent piece'); + } + pieceCid=stored.pieceCid.toString(); + setStage('commit'); + const commit=await context.commit({pieces:[{pieceCid:stored.pieceCid}],onSubmitted:recordHash}); + const hash=commit.confirmedTxHash ?? commit.txHash; + setStage('receipt'); + await confirmed(sdk,hash,recordHash); + copies.push({provider_id:context.provider.id.toString(),provider_address:context.provider.serviceProvider, + data_set_id:commit.dataSetId.toString(),piece_id:commit.pieceIds[0].toString(),transaction_hash:hash, + retrieval_url:context.getPieceUrl(stored.pieceCid)}); + } + return {...NETWORK,piece_cid:pieceCid,payer_address:sdk.client.account.address, + sha256:sha256(request.bytes),size_bytes:request.bytes.length,complete:copies.length===request.copies, + copies,transaction_hashes:[...new Set(hashes)]}; +} + +function retrievalUrl(serviceURL,pieceCid) { + let url; + try {url=new URL(Piece.createPieceUrlPDP({serviceURL,cid:pieceCid}));} + catch {fail('unsafe_provider_url','Invalid registered provider URL');} + return validateRetrievalUrl(url); +} + +function validateRetrievalUrl(url) { + const hostname=url.hostname.toLowerCase().replace(/\.$/,''); + if (url.protocol!=='https:' || url.username || url.password || isIP(hostname) || hostname.includes(':') || + !hostname.includes('.') || /\.(localhost|local|internal|lan|home)$/.test(hostname)) { + fail('unsafe_provider_url','Provider retrieval requires an HTTPS public DNS name without credentials'); + } + return url; +} + +async function downloadBytes(url,expectedSize) { + // The SDK's convenience download buffers an unbounded response before checking its PieceCID. + const origin=url.origin; + const visited=new Set([url.href]); + const signal=AbortSignal.timeout(60000); + let response; + for (let redirects=0;;redirects++) { + response=await fetch(url,{redirect:'manual',signal}); + if (![301,302,303,307,308].includes(response.status)) break; + await response.body?.cancel(); + const location=response.headers.get('location'); + if (!location || redirects>=3) fail('retrieval_unavailable','Provider redirect limit reached or target missing'); + let target; + try {target=new URL(location,url);} + catch {fail('unsafe_provider_url','Invalid provider redirect URL');} + validateRetrievalUrl(target); + if (target.origin!==origin) fail('unsafe_provider_url','Provider redirect must stay on its registered HTTPS origin'); + if (visited.has(target.href)) fail('retrieval_unavailable','Provider redirect loop detected'); + visited.add(target.href); + url=target; + } + if (!response.ok || !response.body) { + await response.body?.cancel(); + fail('retrieval_unavailable','Registered provider did not return the piece'); + } + const length=response.headers.get('content-length'); + if (length!==null && (!/^[0-9]+$/.test(length) || BigInt(length)>BigInt(expectedSize))) { + await response.body.cancel(); + fail('integrity_error','Provider response exceeds the expected piece size'); + } + const reader=response.body.getReader(); + const chunks=[]; + let size=0; + try { + while (true) { + const {done,value}=await reader.read(); + if (done) break; + size+=value.length; + if (size>expectedSize) { + await reader.cancel(); + fail('integrity_error','Provider response exceeds the expected piece size'); + } + chunks.push(value); + } + } finally {reader.releaseLock();} + return Buffer.concat(chunks,size); +} + +async function download(request,sdk) { + const addresses=request.provider_address ? [request.provider_address] : [...new Set( + (await sdk.storage.findDataSets({address:request.payer_address})) + .filter(dataset=>dataset.isLive && dataset.isManaged && dataset.pdpEndEpoch===0n) + .map(dataset=>dataset.serviceProvider))]; + for (const address of addresses) { + const provider=await sdk.getProviderInfo(address); + const url=retrievalUrl(provider.pdp.serviceURL,request.piece_cid); + let bytes; + try {bytes=await downloadBytes(url,request.size_bytes);} + catch(error) { + if (error.code==='integrity_error' || request.provider_address) throw error; + continue; + } + if (bytes.length !== request.size_bytes || sha256(bytes) !== request.sha256 || + (await Piece.calculate(bytes)).toString() !== request.piece_cid) { + fail('integrity_error','Retrieved bytes do not match the expected PieceCID, size and SHA256'); + } + return {...NETWORK,data_base64:bytes.toString('base64'),sha256:request.sha256,size_bytes:bytes.length}; + } + fail('retrieval_unavailable','No registered provider returned the requested piece'); +} + +async function status(request,sdk) { + const block=await sdk.readClient.getBlockNumber({cacheTime:0}); + const copies=[]; + for (const copy of request.copies) { + const dataSetId=id(copy.data_set_id); + const [dataset,live,pieces,nextChallenge,removals]=await Promise.all([ + WarmStorage.getDataSet(sdk.readClient,{dataSetId}),PDP.dataSetLive(sdk.readClient,{dataSetId}), + PDP.getActivePiecesByCursor(sdk.readClient,{dataSetId,cursor:id(copy.piece_id),limit:1n}), + PDP.getNextChallengeEpoch(sdk.readClient,{dataSetId}),PDP.getScheduledRemovals(sdk.readClient,{dataSetId}), + ]); + const registered=dataset != null && dataset.providerId===id(copy.provider_id) && + pieces.items.some(piece=>piece.id===id(copy.piece_id) && piece.cid.toString()===request.piece_cid) && + !removals.includes(id(copy.piece_id)); + // A current period without a proof is pending, not a claim of a failed or missing historic proof. + const proof=dataset == null ? null : await sdk.readClient.readContract({ + address:calibration.contracts.fwssView.address,abi:calibration.contracts.fwssView.abi, + functionName:'provenThisPeriod',args:[dataSetId]}); + copies.push({...copy,registered,live,service_end_epoch:dataset?.pdpEndEpoch.toString() ?? null, + next_challenge_epoch:nextChallenge?.toString() ?? null, + proof_status:proof===true?'proven_this_period':proof===false?'pending_this_period':'unknown', + payer_address:dataset?.payer ?? null}); + } + return {...NETWORK,observed_block:block.toString(),observed_at:new Date().toISOString(),copies}; +} + +async function payments(request,sdk,startWrite,recordHash) { + const service=sdk.payments; + if (request.action==='balance') { + const [fil,usdfc,account,approval]=await Promise.all([service.walletBalance({token:TOKENS.FIL}), + service.walletBalance({token:TOKENS.USDFC}),service.accountSummary(),service.serviceApproval()]); + return {...NETWORK,address:sdk.client.account.address,tfil_base_units:fil.toString(), + usdfc_base_units:usdfc.toString(),account,approval}; + } + if (request.action==='terminate') { + startWrite(); + const result=await sdk.storage.terminateService({dataSetId:request.dataset,skipProvider:true,onSubmitted:recordHash}); + const hash=result.confirmedTxHash ?? result.txHash; + if (hash) await confirmed(sdk,hash,recordHash); + return {...NETWORK,data_set_id:request.data_set_id,end_epoch:result.endEpoch.toString(),transaction_hash:hash ?? null}; + } + let hash; + if (request.action==='deposit') { + const account=await service.accountInfo(); + if (account.funds+request.amount > MAX_USDFC) fail('budget_exceeded','Total account funding may not exceed 10 test USDFC'); + if (await service.walletBalance({token:TOKENS.USDFC}) < request.amount) { + fail('funding_required','Wallet has insufficient test USDFC for the requested deposit'); + } + startWrite(); + hash=await service.deposit({amount:request.amount,onApprovalTransaction:recordHash}); + } else if (request.action==='approve') { + startWrite(); + hash=await service.approveService({rateAllowance:request.monthly/EPOCHS_MONTH,lockupAllowance:request.lockup, + maxLockupPeriod:BigInt(request.days)*EPOCHS_DAY}); + } else if (request.action==='withdraw') { + const account=await service.accountInfo(); + if (account.availableFunds{}) { + const hashes=[]; + let started=false; + let stage='preflight'; + const setStage=value=>{stage=value;}; + const startWrite=()=>{ + recordEvent({type:'write_started'}); + started=true; + }; + const recordHash=hash=>{ + if (!/^0x[0-9a-fA-F]{64}$/.test(hash)) fail('invalid_receipt','Invalid transaction hash returned by Filecoin service'); + hashes.push(hash); + recordEvent({type:'transaction_submitted',transaction_hash:hash}); + }; + try { + if (await sdk.readClient.getChainId() !== 314159) fail('wrong_network','RPC is not Filecoin Calibration'); + if (request.command==='download') return await download(request,sdk); + if (request.command==='status') return await status(request,sdk); + if (request.command==='upload') return await upload(request,sdk,hashes,startWrite,recordHash,setStage); + setStage('payment'); + return await payments(request,sdk,startWrite,recordHash); + } catch(error) { + error.transaction_hashes=[...new Set(hashes)]; + error.uncertain=started || hashes.length>0; + error.operation_stage=stage; + throw error; + } +} + +export function safeError(error) { + const errorTypes=[]; + const httpStatuses=[]; + const seen=new Set(); + for (let cause=error;cause instanceof Error && !seen.has(cause) && seen.size<5;cause=cause.cause) { + seen.add(cause); + // Class names are local diagnostics; do not serialize messages, details or request/response bodies. + const name=cause.constructor?.name; + if (typeof name==='string' && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(name)) errorTypes.push(name); + for (const status of [cause.status,cause.statusCode,cause.response?.status]) { + if (Number.isInteger(status) && status>=100 && status<=599) httpStatuses.push(status); + } + } + return {error:{code:error.safe?error.code:'operation_failed', + message:error.safe?error.message:'Filecoin operation failed; inspect recorded transaction hashes before retrying a write', + uncertain:error.uncertain ?? false,transaction_hashes:error.transaction_hashes ?? [], + ...(['preflight','store','commit','receipt','payment'].includes(error.operation_stage)?{operation_stage:error.operation_stage}:{}), + error_types:errorTypes,http_statuses:[...new Set(httpStatuses)],...(error.safe?error.details:{})}}; +} +const json=value=>JSON.stringify(value,(_,item)=>typeof item==='bigint'?item.toString():item); + +if (process.argv[1] && import.meta.url===pathToFileURL(process.argv[1]).href) { + // Keep SDK diagnostics (which can include signed payloads) out of the wire protocol and logs. + console.log=console.warn=console.error=()=>{}; + let journal; + try { + let input=''; + for await (const chunk of process.stdin) { + input+=chunk; + if (input.length>Math.ceil(MAX_BYTES/3)*4+16384) fail('invalid_request','Request exceeds maximum size'); + } + const request=parseRequest(JSON.parse(input)); + if (process.env.QMESH_FILECOIN_JOURNAL_PATH) journal=createJournal(process.env.QMESH_FILECOIN_JOURNAL_PATH); + process.stdout.write(json(await execute(request,makeSynapse(request),journal?.append))+'\n'); + } catch(error) { + process.stdout.write(json(safeError(error))+'\n'); + process.exitCode=1; + } finally {journal?.close();} +} diff --git a/services/filecoin/bridge.test.mjs b/services/filecoin/bridge.test.mjs new file mode 100644 index 00000000..96389b39 --- /dev/null +++ b/services/filecoin/bridge.test.mjs @@ -0,0 +1,347 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtempSync, readFileSync, rmSync, rmdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { calibration } from '@filoz/synapse-sdk'; +import { calculate as pieceCalculate } from '@filoz/synapse-core/piece'; +import { parseRequest, execute, safeError, makeSynapse, createJournal, MAX_BYTES } from './bridge.mjs'; + +const data = Buffer.alloc(256, 7); +const digest = createHash('sha256').update(data).digest('hex'); +const upload = {command: 'upload', data_base64: data.toString('base64'), copies: 1, + retention_days: 7, max_monthly_usdfc: '1', max_lockup_usdfc: '1'}; +const address = '0x1111111111111111111111111111111111111111'; +const hash = `0x${'a'.repeat(64)}`; + +test('reject unsupported network, malformed bytes and unbounded budgets before RPC', () => { + for (const request of [ + {...upload, network: 'mainnet'}, {...upload, chain_id: 314}, + {...upload, data_base64: '!!!'}, {...upload, data_base64: 'YQ=='}, + {...upload, copies: 3}, {...upload, retention_days: 31}, + {...upload, max_monthly_usdfc: '11'}, {...upload, max_lockup_usdfc: 'Infinity'}, + ]) assert.throws(() => parseRequest(request)); + assert.equal(parseRequest(upload).bytes.length, 256); +}); + +test('real SDK import uses Calibration contract addresses', () => { + assert.equal(calibration.id, 314159); + const sdk = makeSynapse({command: 'status'}, {}); + assert.equal(sdk.chain.id, 314159); + assert.equal(sdk.storage.withCDN, false); + assert.throws(() => makeSynapse(upload, {}), /signing key/); + assert.throws(() => makeSynapse({command:'status'}, {QMESH_FILECOIN_NETWORK:'mainnet'})); +}); + +function fake() { + const calls = []; + const ctx = {provider: {id: 3n, serviceProvider: address}, + store: async bytes => {calls.push('store'); return {pieceCid: {toString:()=> 'piece'}, size: bytes.length};}, + commit: async opts => { calls.push('commit'); opts.onSubmitted(hash); return { + txHash: hash, confirmedTxHash: hash, dataSetId: 5n, pieceIds:[0n]}; }, + getPieceUrl: () => 'https://provider.example/piece/piece'}; + const sdk = {client: {account: {address}}, readClient: { + getChainId: async()=>314159, waitForTransactionReceipt: async()=>({status:'success',blockNumber:10n}), + readContract: async()=>({lockups:{defaultLockupPeriod:86400n}}), + }, storage: {createContexts: async()=>[ctx], calculateMultiContextCosts: async()=>({ + rates:{perEpoch:1n,perMonth:86400n}, lockups:{total:1n,rateDeltaPerEpoch:1n}, depositNeeded:0n}), + }, payments: {serviceApproval: async()=>({isApproved:true,rateAllowance:100000n,rateUsage:0n, + lockupAllowance:100n,lockupUsage:0n,maxLockupPeriod:86400n})}}; + return {sdk,calls}; +} + +test('upload uses funded finite allowance and records confirmed copy receipt', async()=> { + const {sdk,calls} = fake(); + const result = await execute(parseRequest(upload), sdk); + assert.deepEqual(calls,['store','commit']); + assert.equal(result.sha256,digest); + assert.equal(result.payer_address,address); + assert.equal(result.copies[0].data_set_id,'5'); + assert.equal(result.copies[0].transaction_hash,hash); + assert.equal(result.complete,true); +}); + +test('wrong RPC chain and insufficient funding cause no storage mutation', async()=> { + const a = fake(); a.sdk.readClient.getChainId=async()=>314; + await assert.rejects(execute(parseRequest(upload),a.sdk),/Calibration/); + assert.deepEqual(a.calls,[]); + const b = fake(); b.sdk.storage.calculateMultiContextCosts=async()=>({depositNeeded:1n,rates:{perMonth:1n}}); + await assert.rejects(execute(parseRequest(upload),b.sdk), e=>e.code==='funding_required'); + assert.deepEqual(b.calls,[]); +}); + +test('unbounded approval and over-budget estimate never upload', async()=> { + const a=fake(); a.sdk.payments.serviceApproval=async()=>({isApproved:true,rateAllowance:2n**256n-1n, + lockupAllowance:2n**256n-1n}); + await assert.rejects(execute(parseRequest(upload),a.sdk),e=>e.code==='approval_required'); + assert.deepEqual(a.calls,[]); + const b=fake(); b.sdk.storage.calculateMultiContextCosts=async()=>({depositNeeded:0n,rates:{perMonth:2n*10n**18n}}); + await assert.rejects(execute(parseRequest(upload),b.sdk),e=>e.code==='budget_exceeded'); + assert.deepEqual(b.calls,[]); +}); + +test('operator lockup shorter than live contract requirement is rejected before store',async()=>{ + const {sdk,calls}=fake(); + sdk.payments.serviceApproval=async()=>({isApproved:true,rateAllowance:100000n,rateUsage:0n, + lockupAllowance:100n,lockupUsage:0n,maxLockupPeriod:20160n}); + await assert.rejects(execute(parseRequest(upload),sdk),e=>e.code==='approval_required' && + e.uncertain===false && e.transaction_hashes.length===0 && e.details.required_lockup_epochs==='86400'); + assert.deepEqual(calls,[]); +}); + +test('diagnostics retain stage and error classes without leaking SDK messages',async()=>{ + const {sdk}=fake(); + sdk.storage.createContexts=async()=>[{ + provider:{id:3n,serviceProvider:address},store:async()=>{ + const cause=new TypeError('signedpayload and secret must remain private'); + cause.response={status:403}; + throw new Error('private SDK message',{cause}); + }, + }]; + try {await execute(parseRequest(upload),sdk);assert.fail('expected failure');} + catch(error){ + const serialized=safeError(error); + assert.equal(serialized.error.operation_stage,'store'); + assert.deepEqual(serialized.error.error_types,['Error','TypeError']); + assert.deepEqual(serialized.error.http_statuses,[403]); + assert.ok(!JSON.stringify(serialized).includes('private')); + assert.equal(serialized.error.uncertain,true); + } +}); + +test('uncertain commit preserves submitted hashes and never retries', async()=> { + const a=fake(); a.sdk.storage.createContexts=async()=>[{ + provider:{id:3n,serviceProvider:address}, store:async()=>({pieceCid:{toString:()=> 'piece'},size:256}), + commit:async opts=>{a.calls.push('commit');opts.onSubmitted(hash);throw Error('secret=never echo');} + }]; + try {await execute(parseRequest(upload),a.sdk); assert.fail('expected failure');} + catch(error) { + const result=safeError(error); assert.equal(result.error.uncertain,true); + assert.deepEqual(result.error.transaction_hashes,[hash]); + assert.ok(!JSON.stringify(result).includes('secret')); + } + assert.deepEqual(a.calls,['commit']); +}); + +test('payment approval uses finite supplied limits', async()=> { + const {sdk}=fake(); let options; + sdk.payments.approveService=async p=>{options=p;return hash;}; + const result=await execute(parseRequest({command:'payments',action:'approve',max_monthly_usdfc:'1', + max_lockup_usdfc:'2',max_lockup_days:30}),sdk); + assert.equal(options.rateAllowance,10n**18n/86400n); + assert.equal(options.lockupAllowance,2n*10n**18n); + assert.equal(options.maxLockupPeriod,86400n); + assert.equal(result.transaction_hash,hash); +}); + +test('explicit deposit rejects total account funding over test-token cap', async()=> { + const {sdk}=fake(); sdk.payments.accountInfo=async()=>({funds:10n**19n}); + sdk.payments.deposit=async()=>assert.fail('deposit must not run'); + await assert.rejects(execute(parseRequest({command:'payments',action:'deposit',amount_usdfc:'1'}),sdk), + e=>e.code==='budget_exceeded'); +}); + +test('download requires expected size and digest',()=>{ + assert.throws(()=>parseRequest({command:'download',piece_cid:'bad',sha256:digest,size_bytes:MAX_BYTES+1})); + assert.throws(()=>parseRequest({command:'download',piece_cid:'bad',size_bytes:256})); +}); + +test('agent wallet binding rejects a different signer before RPC',()=>{ + const env={QMESH_FILECOIN_PRIVATE_KEY:`0x${'1'.repeat(64)}`,QMESH_FILECOIN_EXPECTED_ADDRESS:address}; + assert.throws(()=>makeSynapse(upload,env),e=>e.code==='wrong_signer'); +}); + +test('fresh key-free download validates real piece shape and rejects mismatched bytes',async t=>{ + const cid=await pieceCalculate(data); + const request=parseRequest({command:'download',piece_cid:cid.toString(),sha256:digest,size_bytes:data.length, + provider_address:address}); + const {sdk}=fake(); + sdk.getProviderInfo=async()=>({pdp:{serviceURL:'https://provider.example/'}}); + t.mock.method(globalThis,'fetch',async()=>new Response(data)); + const result=await execute(request,sdk); + assert.equal(result.data_base64,data.toString('base64')); + t.mock.method(globalThis,'fetch',async()=>new Response(Buffer.alloc(data.length,8))); + await assert.rejects(execute(request,sdk),e=>e.code==='integrity_error'); +}); + +test('provider piece redirect to same-origin public path is followed with integrity checks',async t=>{ + const cid=await pieceCalculate(data); + const request=parseRequest({command:'download',piece_cid:cid.toString(),sha256:digest,size_bytes:data.length, + provider_address:address}); + const {sdk}=fake(); sdk.getProviderInfo=async()=>({pdp:{serviceURL:'https://provider.example/'}}); + const paths=[]; + t.mock.method(globalThis,'fetch',async(url,options)=>{ + assert.equal(options.redirect,'manual'); + paths.push(url.pathname); + return paths.length===1 ? new Response(null,{status:301,headers:{location:`/public/${cid}`}}) : new Response(data); + }); + assert.equal((await execute(request,sdk)).data_base64,data.toString('base64')); + assert.deepEqual(paths,[`/piece/${cid}`,`/public/${cid}`]); +}); + +test('cross-origin and unsafe redirects are rejected before fetching their target',async t=>{ + const cid=await pieceCalculate(data); + const request=parseRequest({command:'download',piece_cid:cid.toString(),sha256:digest,size_bytes:data.length, + provider_address:address}); + const {sdk}=fake(); sdk.getProviderInfo=async()=>({pdp:{serviceURL:'https://provider.example/'}}); + for (const location of ['https://other.example/public/piece','http://provider.example/public/piece', + 'https://user:secret@provider.example/public/piece','https://127.0.0.1/piece','https://localhost/piece']) { + let requests=0; + t.mock.method(globalThis,'fetch',async()=>{ + requests++; + return new Response(null,{status:302,headers:{location}}); + }); + await assert.rejects(execute(request,sdk),e=>e.code==='unsafe_provider_url'); + assert.equal(requests,1); + } +}); + +test('redirect loops and more than three same-origin redirects are bounded',async t=>{ + const cid=await pieceCalculate(data); + const request=parseRequest({command:'download',piece_cid:cid.toString(),sha256:digest,size_bytes:data.length, + provider_address:address}); + const {sdk}=fake(); sdk.getProviderInfo=async()=>({pdp:{serviceURL:'https://provider.example/'}}); + let requests=0; + t.mock.method(globalThis,'fetch',async url=>{ + requests++; + return new Response(null,{status:307,headers:{location:url.href}}); + }); + await assert.rejects(execute(request,sdk),e=>e.code==='retrieval_unavailable'); + assert.equal(requests,1); + requests=0; + t.mock.method(globalThis,'fetch',async()=>{ + requests++; + return new Response(null,{status:308,headers:{location:`/hop/${requests}`}}); + }); + await assert.rejects(execute(request,sdk),e=>e.code==='retrieval_unavailable'); + assert.equal(requests,4); +}); + +test('oversized provider stream is cancelled before buffering its remaining chunks',async t=>{ + const cid=await pieceCalculate(data); + const request=parseRequest({command:'download',piece_cid:cid.toString(),sha256:digest,size_bytes:data.length, + provider_address:address,retrieval_url:'http://localhost:1234/untrusted'}); + const {sdk}=fake(); + sdk.getProviderInfo=async()=>({pdp:{serviceURL:'https://provider.example/'}}); + let pulled=0; + let cancelled=false; + const stream=new ReadableStream({ + pull(controller) { + pulled++; + controller.enqueue(new Uint8Array(128)); + if (pulled===100) controller.close(); + }, + cancel(){cancelled=true;}, + },{highWaterMark:0}); + t.mock.method(globalThis,'fetch',async(url,options)=>{ + assert.ok(url.toString().startsWith('https://provider.example/')); + assert.equal(options.redirect,'manual'); + return new Response(stream); + }); + await assert.rejects(execute(request,sdk),e=>e.code==='integrity_error' && e.uncertain===false); + assert.equal(cancelled,true); + assert.equal(pulled,3); +}); + +test('PieceCID is checked independently when size and SHA256 match supplied bytes',async t=>{ + const cid=await pieceCalculate(data); + const different=Buffer.alloc(data.length,8); + const request=parseRequest({command:'download',piece_cid:cid.toString(),sha256:createHash('sha256').update(different).digest('hex'), + size_bytes:data.length,provider_address:address}); + const {sdk}=fake(); sdk.getProviderInfo=async()=>({pdp:{serviceURL:'https://provider.example/'}}); + t.mock.method(globalThis,'fetch',async()=>new Response(different)); + await assert.rejects(execute(request,sdk),e=>e.code==='integrity_error'); +}); + +test('payer-only discovery uses registered providers and tries another provider after 404',async t=>{ + const cid=await pieceCalculate(data); + const request=parseRequest({command:'download',piece_cid:cid.toString(),sha256:digest,size_bytes:data.length, + payer_address:address}); + const {sdk}=fake(); + sdk.storage.findDataSets=async()=>[ + {isLive:true,isManaged:true,pdpEndEpoch:0n,serviceProvider:address}, + {isLive:true,isManaged:true,pdpEndEpoch:0n,serviceProvider:'0x2222222222222222222222222222222222222222'}, + ]; + sdk.getProviderInfo=async address=>({pdp:{serviceURL:address.startsWith('0x1')?'https://first.example/':'https://second.example/'}}); + t.mock.method(globalThis,'fetch',async url=>url.hostname==='first.example'?new Response('',{status:404}):new Response(data)); + assert.equal((await execute(request,sdk)).data_base64,data.toString('base64')); +}); + +test('untrusted registry retrieval URLs are rejected before HTTP',async t=>{ + const cid=await pieceCalculate(data); + const request=parseRequest({command:'download',piece_cid:cid.toString(),sha256:digest,size_bytes:data.length, + provider_address:address}); + const {sdk}=fake(); + t.mock.method(globalThis,'fetch',async()=>assert.fail('must not fetch rejected URL')); + for (const serviceURL of ['http://provider.example/','https://127.0.0.1/','https://[::1]/', + 'https://10.0.0.1/','https://169.254.169.254/','https://localhost/','https://storage.local/', + 'https://user:password@provider.example/']) { + sdk.getProviderInfo=async()=>({pdp:{serviceURL}}); + await assert.rejects(execute(request,sdk),e=>e.code==='unsafe_provider_url'); + } +}); + +test('preflight RPC errors release write journal but store failures remain uncertain',async()=>{ + const a=fake(); a.sdk.storage.calculateMultiContextCosts=async()=>{throw Error('read failed');}; + await assert.rejects(execute(parseRequest(upload),a.sdk),e=>e.uncertain===false && e.transaction_hashes.length===0); + assert.deepEqual(a.calls,[]); + const b=fake(); b.sdk.storage.createContexts=async()=>[{ + provider:{id:3n,serviceProvider:address},store:async()=>{throw Error('provider connection lost');}, + }]; + await assert.rejects(execute(parseRequest(upload),b.sdk),e=>e.uncertain===true && e.transaction_hashes.length===0); +}); + +test('deterministic insufficient wallet balance never invokes deposit',async()=>{ + const {sdk}=fake(); sdk.payments.accountInfo=async()=>({funds:0n}); + sdk.payments.walletBalance=async()=>0n; + sdk.payments.deposit=async()=>assert.fail('deposit must not run'); + await assert.rejects(execute(parseRequest({command:'payments',action:'deposit',amount_usdfc:'1'}),sdk), + e=>e.code==='funding_required' && e.uncertain===false && e.transaction_hashes.length===0); +}); + +test('broadcast failure before a hash is returned remains uncertain',async()=>{ + const {sdk}=fake(); sdk.payments.approveService=async()=>{throw Error('send disconnected');}; + await assert.rejects(execute(parseRequest({command:'payments',action:'approve',max_monthly_usdfc:'1', + max_lockup_usdfc:'1',max_lockup_days:30}),sdk),e=>e.uncertain===true && e.transaction_hashes.length===0); +}); + +test('transaction events are durable before the receipt wait starts',async t=>{ + const dir=mkdtempSync(join(tmpdir(),'qmesh-filecoin-journal-')); + const path=join(dir,'operation.ndjson'); + const journal=createJournal(path); + t.after(()=>{journal.close();rmSync(path);rmdirSync(dir);}); + const {sdk}=fake(); + const read=()=>readFileSync(path,'utf8').trim().split('\n').map(line=>JSON.parse(line)); + sdk.payments.approveService=async()=>{ + assert.deepEqual(read(),[{type:'write_started'}]); + return hash; + }; + sdk.readClient.waitForTransactionReceipt=async()=>{ + assert.deepEqual(read(),[{type:'write_started'},{type:'transaction_submitted',transaction_hash:hash}]); + return {status:'success',blockNumber:10n}; + }; + await execute(parseRequest({command:'payments',action:'approve',max_monthly_usdfc:'1', + max_lockup_usdfc:'1',max_lockup_days:30}),sdk,journal.append); + assert.throws(()=>createJournal(path)); +}); + +test('journal failure before a write prevents submission',async()=>{ + const {sdk}=fake(); sdk.payments.approveService=async()=>assert.fail('must not submit without journal'); + await assert.rejects(execute(parseRequest({command:'payments',action:'approve',max_monthly_usdfc:'1', + max_lockup_usdfc:'1',max_lockup_days:30}),sdk,()=>{throw Error('fsync failed');}), + e=>e.uncertain===false && e.transaction_hashes.length===0); +}); + +test('journal failure after broadcast preserves uncertain hash in final error',async()=>{ + const {sdk}=fake(); sdk.payments.approveService=async()=>hash; + await assert.rejects(execute(parseRequest({command:'payments',action:'approve',max_monthly_usdfc:'1', + max_lockup_usdfc:'1',max_lockup_days:30}),sdk,event=>{ + if(event.type==='transaction_submitted') throw Error('fsync failed'); + }),e=>e.uncertain===true && e.transaction_hashes[0]===hash); +}); + +test('reverted transaction is never returned as a successful upload',async()=>{ + const {sdk}=fake(); sdk.readClient.waitForTransactionReceipt=async()=>({status:'reverted'}); + await assert.rejects(execute(parseRequest(upload),sdk),e=>e.code==='transaction_reverted' && e.uncertain); +}); diff --git a/services/filecoin/package-lock.json b/services/filecoin/package-lock.json new file mode 100644 index 00000000..2e222f36 --- /dev/null +++ b/services/filecoin/package-lock.json @@ -0,0 +1,778 @@ +{ + "name": "@qmeshpy/filecoin-bridge", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@qmeshpy/filecoin-bridge", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@filoz/synapse-core": "0.10.3", + "@filoz/synapse-sdk": "2.0.2", + "viem": "2.57.17" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@adraffy/ens-normalize": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@adraffy/ens-normalize/-/ens-normalize-1.11.1.tgz", + "integrity": "sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==", + "license": "MIT" + }, + "node_modules/@filoz/synapse-core": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@filoz/synapse-core/-/synapse-core-0.10.3.tgz", + "integrity": "sha512-awWRsQZlW5N55+CmzX14raUsEm2Y/tqeGAKxvZhW2GUZZLaMabnAQ570loco/PM9Em0efSQh4A8AmNKcmgzelg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@hugomrdias/commp-wasm": "0.1.4", + "dnum": "2.17.0", + "iso-web": "^3.1.2", + "multiformats": "14.0.5", + "ox": "0.14.34", + "p-locate": "7.0.0", + "p-some": "7.0.0", + "sync-multihash-sha2": "1.0.0", + "zod": "4.4.3" + }, + "peerDependencies": { + "viem": "2.x" + } + }, + "node_modules/@filoz/synapse-sdk": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@filoz/synapse-sdk/-/synapse-sdk-2.0.2.tgz", + "integrity": "sha512-PiYybc4FHq/YWHouZKiS9otp3e32agE2u0C26L04uPklfYMdVeagJV38LlppxDkSTmymHrxJ292i6kGZe9hG4A==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@filoz/synapse-core": "^0.10.0", + "multiformats": "14.0.5" + }, + "peerDependencies": { + "viem": "2.x" + } + }, + "node_modules/@hugomrdias/commp-wasm": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@hugomrdias/commp-wasm/-/commp-wasm-0.1.4.tgz", + "integrity": "sha512-e2oiHAmjtFZ3cxalwqJQgxW8FHlFqj0cXN6PM4NPEhEH/Mf5wjLBzlslA11hUsyOMmMK9+WtWPPNycg5RdiBKg==", + "license": "MIT" + }, + "node_modules/@noble/ciphers": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.3.0.tgz", + "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/curves": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.1.tgz", + "integrity": "sha512-k11yZxZg+t+gWvBbIswW0yoJlu8cHOC7dhunwOzoWH/mXGBiYyR4YY6hAEK/3EUs4UpB8la1RfdRpeGsFHkWsA==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.8.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/base": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.2.6.tgz", + "integrity": "sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip32": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.7.0.tgz", + "integrity": "sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==", + "license": "MIT", + "dependencies": { + "@noble/curves": "~1.9.0", + "@noble/hashes": "~1.8.0", + "@scure/base": "~1.2.5" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.6.0.tgz", + "integrity": "sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "~1.8.0", + "@scure/base": "~1.2.5" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/abitype": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/abitype/-/abitype-1.3.0.tgz", + "integrity": "sha512-fk6Te+bojIFrMvMZrnOO+SxCB+RUksTGOzq/60ZRvs1L+BVzvi2bqt9L3W/17ZLdZsyM1FuYf65P5nlmoiH1Bg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/wevm" + }, + "peerDependencies": { + "typescript": ">=5.0.4", + "zod": "^3.22.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/atomically": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/atomically/-/atomically-2.1.1.tgz", + "integrity": "sha512-P4w9o2dqARji6P7MHprklbfiArZAWvo07yW7qs3pdljb3BWr12FIB7W+p0zJiuiVsUpRO0iZn1kFFcpPegg0tQ==", + "license": "MIT", + "dependencies": { + "stubborn-fs": "^2.0.0", + "when-exit": "^2.1.4" + } + }, + "node_modules/bigint-mod-arith": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/bigint-mod-arith/-/bigint-mod-arith-3.3.1.tgz", + "integrity": "sha512-pX/cYW3dCa87Jrzv6DAr8ivbbJRzEX5yGhdt8IutnX/PCIXfpx+mabWNK/M8qqh+zQ0J3thftUBHW0ByuUlG0w==", + "license": "MIT", + "engines": { + "node": ">=10.4.0" + } + }, + "node_modules/conf": { + "version": "15.1.0", + "resolved": "https://registry.npmjs.org/conf/-/conf-15.1.0.tgz", + "integrity": "sha512-Uy5YN9KEu0WWDaZAVJ5FAmZoaJt9rdK6kH+utItPyGsCqCgaTKkrmZx3zoE0/3q6S3bcp3Ihkk+ZqPxWxFK5og==", + "license": "MIT", + "dependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "atomically": "^2.0.3", + "debounce-fn": "^6.0.0", + "dot-prop": "^10.0.0", + "env-paths": "^3.0.0", + "json-schema-typed": "^8.0.1", + "semver": "^7.7.2", + "uint8array-extras": "^1.5.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/debounce-fn": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/debounce-fn/-/debounce-fn-6.0.0.tgz", + "integrity": "sha512-rBMW+F2TXryBwB54Q0d8drNEI+TfoS9JpNTAoVpukbWEhjXQq4rySFYLaqXMFXwdv61Zb2OHtj5bviSoimqxRQ==", + "license": "MIT", + "dependencies": { + "mimic-function": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/delay": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/delay/-/delay-7.0.0.tgz", + "integrity": "sha512-C3vaGs818qzZjCvVJ98GQUMVyWeg7dr5w2Nwwb2t5K8G98jOyyVO2ti2bKYk5yoYElqH3F2yA53ykuEnwD6MCg==", + "license": "MIT", + "dependencies": { + "random-int": "^3.1.0", + "unlimited-timeout": "^0.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/dnum": { + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/dnum/-/dnum-2.17.0.tgz", + "integrity": "sha512-Abo8RU2ZoABVO2R051XlJEgDIXAlA8/ZjOT2F1uAWvm6Vb8TphmN4k7qgu5nWKSv/JUGLVty6QPEeLTvaxNRYQ==", + "license": "MIT", + "dependencies": { + "from-exponential": "^1.1.1" + } + }, + "node_modules/dot-prop": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-10.2.0.tgz", + "integrity": "sha512-BTJ9aZYL3vCfZlZOBLy9v8TUqWGQ0pzFnygKwFZt5udj6viBoFIBviKPUoZLDCPn1FoXffv6McQFDenrm5Krfw==", + "license": "MIT", + "dependencies": { + "type-fest": "^5.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/env-paths": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", + "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz", + "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/from-exponential": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/from-exponential/-/from-exponential-1.1.1.tgz", + "integrity": "sha512-VBE7f5OVnYwdgB3LHa+Qo29h8qVpxhVO9Trlc+AWm+/XNAgks1tAwMFHb33mjeiof77GglsJzeYF7OqXrROP/A==", + "license": "MIT" + }, + "node_modules/idb-keyval": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.3.0.tgz", + "integrity": "sha512-um+2dgAWmYsu615EXpWVwSmapJhON0G43t3Ka/EVaohzPQXSMqKEqeDK/oIW3Ow+BXaF2PvSc+oBTFp793A5Ow==", + "license": "Apache-2.0" + }, + "node_modules/is-network-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/iso-base": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/iso-base/-/iso-base-4.4.0.tgz", + "integrity": "sha512-W4BJbRDBi66wcBFJ23ZlGnFOZ874CIut4y9PlsScMSYu7ckCX+MWNAaEoE0efTSYDoVEn1qy0FDVSjE8q0ieHw==", + "license": "MIT", + "dependencies": { + "bigint-mod-arith": "^3.3.1" + } + }, + "node_modules/iso-kv": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/iso-kv/-/iso-kv-3.2.0.tgz", + "integrity": "sha512-rMVXO7zDEecXOJEiDnbDqjZ7gSe7VqDn2FTkUFWeqkcYMmgMdmB7pDgUaZxiEyiL1+gO7RnbqEI2Ce3xIW4UBQ==", + "license": "MIT", + "dependencies": { + "conf": "^15.1.0", + "idb-keyval": "^6.2.1", + "iso-base": "^4.4.0", + "kysely": "^0.29.2" + } + }, + "node_modules/iso-web": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/iso-web/-/iso-web-3.1.2.tgz", + "integrity": "sha512-OpWz+KNH4vYNVvpc6T1z/zkZtT6iTQXYjc0sIbkPDwYEM5Qurg0LYESrEHloxb6xj5T/2rrnsTtaYlUSIzgicg==", + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "delay": "^7.0.0", + "is-network-error": "^1.3.1", + "iso-kv": "^3.2.0", + "p-retry": "^8.0.0" + } + }, + "node_modules/isows": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/isows/-/isows-1.0.7.tgz", + "integrity": "sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "peerDependencies": { + "ws": "*" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, + "node_modules/kysely": { + "version": "0.29.8", + "resolved": "https://registry.npmjs.org/kysely/-/kysely-0.29.8.tgz", + "integrity": "sha512-ixHU1H4R+LeC5Y0nAhYgiGRFpFRHm2DXKEmJSs/vbes9sKXyssyzxzE/NYiOJX5fY0BL28biCre3GqlCqUwyXQ==", + "license": "MIT", + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/mimic-function": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", + "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/multiformats": { + "version": "14.0.5", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-14.0.5.tgz", + "integrity": "sha512-vbIm83F2yZ1pWJGS0yl0ysracIvv56LtbrIyiIQHoLdYDJOMoLfVFsXhh9DUH4SFdkdkFhucyWniihsNzVEjkQ==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/ox": { + "version": "0.14.34", + "resolved": "https://registry.npmjs.org/ox/-/ox-0.14.34.tgz", + "integrity": "sha512-12seOIk7dv8eAoGQhcWaeKZxNz304IVcDvb9U5Y7JZAEVe21Nm1YMxLjhWah+su5BD4Omx4Zz0z5x3ij9M4GYQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "dependencies": { + "@adraffy/ens-normalize": "^1.11.0", + "@noble/ciphers": "^1.3.0", + "@noble/curves": "1.9.1", + "@noble/hashes": "^1.8.0", + "@scure/bip32": "^1.7.0", + "@scure/bip39": "^1.6.0", + "abitype": "^1.2.3", + "eventemitter3": "5.0.1" + }, + "peerDependencies": { + "typescript": ">=5.4.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/p-limit": { + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-7.3.3.tgz", + "integrity": "sha512-SKgVvDU5TNIxQ4r30U75BNLC6a+6GOEjngrk/ysM5+Zu3oJuk1SC5ApzIzY/7PjCEjoVPMWJ0zho/4QUiEB7TQ==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^1.2.1" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-7.0.0.tgz", + "integrity": "sha512-FRPW2lT1b/B8/CNkCOZ/Xl4mz52CWzwb+/dLa0GcCrH7u7djFf36VftuRJ5w/eCr1YXtbTGPuGoEDVSk14EwNQ==", + "license": "MIT", + "dependencies": { + "p-limit": "^7.2.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-8.0.1.tgz", + "integrity": "sha512-NAigyu8hfvJe7MsS18mnc4is0MZtau3QMdBklaJKK23oBQb6occO3UPM3vHmTckW8KhfyjMZaUOqdTFEYliHgg==", + "license": "MIT", + "dependencies": { + "is-network-error": "^1.3.0" + }, + "engines": { + "node": ">=22" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-some": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/p-some/-/p-some-7.0.0.tgz", + "integrity": "sha512-9ldWF6puBzuchsUq7M1THjwwmoiXesqRdpB4WH0D7urKXdGkIaDqVhQS2BSfRRYZ970j9gm3U4/h9hHQx2G1Ug==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/random-int": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/random-int/-/random-int-3.1.0.tgz", + "integrity": "sha512-h8CRz8cpvzj0hC/iH/1Gapgcl2TQ6xtnCpyOI5WvWfXf/yrDx2DOU+tD9rX23j36IF11xg1KqB9W11Z18JPMdw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/semver": { + "version": "7.8.6", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.6.tgz", + "integrity": "sha512-9lpSSU4Mly3mrtruwLs1eLK6+gzbzqeBFwRNjIjE+9LuDpWdYctrQlQ3ej1Ilgi7gknpsTRnPoLP8DpCHqHIJw==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/stubborn-fs": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-2.0.0.tgz", + "integrity": "sha512-Y0AvSwDw8y+nlSNFXMm2g6L51rBGdAQT20J3YSOqxC53Lo3bjWRtr2BKcfYoAf352WYpsZSTURrA0tqhfgudPA==", + "license": "MIT", + "dependencies": { + "stubborn-utils": "^1.0.1" + } + }, + "node_modules/stubborn-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/stubborn-utils/-/stubborn-utils-1.0.2.tgz", + "integrity": "sha512-zOh9jPYI+xrNOyisSelgym4tolKTJCQd5GBhK0+0xJvcYDcwlOoxF/rnFKQ2KRZknXSG9jWAp66fwP6AxN9STg==", + "license": "MIT" + }, + "node_modules/sync-multihash-sha2": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/sync-multihash-sha2/-/sync-multihash-sha2-1.0.0.tgz", + "integrity": "sha512-A5gVpmtKF0ov+/XID0M0QRJqF2QxAsj3x/LlDC8yivzgoYCoWkV+XaZPfVu7Vj1T/hYzYS1tfjwboSbXjqocug==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@noble/hashes": "^1.3.1" + } + }, + "node_modules/tagged-tag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/tagged-tag/-/tagged-tag-1.0.0.tgz", + "integrity": "sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-fest": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.10.0.tgz", + "integrity": "sha512-NoSdpq/WEiAg5sjmBkmV/hfxv6HJH4NqPNrqjtSO5CwRmpsDfaf4begxW34KdJykH/l1yHtwBWQkCRdoXO8mPA==", + "license": "(MIT OR CC0-1.0)", + "dependencies": { + "tagged-tag": "^1.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/uint8array-extras": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.6.0.tgz", + "integrity": "sha512-8iAasVS4wUx0gPLjH8Xtz2PeDzTSiy8QiLGu2DT3X5GU+egFEQhpnbtkehbAwjvDcxIERmCYcysiODFmE3Ud0Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/unlimited-timeout": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/unlimited-timeout/-/unlimited-timeout-0.1.0.tgz", + "integrity": "sha512-D4g+mxFeQGQHzCfnvij+R35ukJ0658Zzudw7j16p4tBBbNasKkKM4SocYxqhwT5xA7a9JYWDzKkEFyMlRi5sng==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/viem": { + "version": "2.57.17", + "resolved": "https://registry.npmjs.org/viem/-/viem-2.57.17.tgz", + "integrity": "sha512-yIMqNoKoVAMwX+zVbqkxfqq7qbGHmKXZidh/xfVpiPTJlAbbDDdDIMH5dDQWxhZcj56L8tPOwsIIxF618P+TOg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "dependencies": { + "@noble/curves": "1.9.1", + "@noble/hashes": "1.8.0", + "@scure/bip32": "1.7.0", + "@scure/bip39": "1.6.0", + "abitype": "1.2.3", + "isows": "1.0.7", + "ox": "0.14.63", + "ws": "8.21.0" + }, + "peerDependencies": { + "mppx": ">=0.17.0", + "typescript": ">=5.0.4" + }, + "peerDependenciesMeta": { + "mppx": { + "optional": true + }, + "typescript": { + "optional": true + } + } + }, + "node_modules/viem/node_modules/abitype": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/abitype/-/abitype-1.2.3.tgz", + "integrity": "sha512-Ofer5QUnuUdTFsBRwARMoWKOH1ND5ehwYhJ3OJ/BQO+StkwQjHw0XyVh4vDttzHB7QOFhPHa/o413PJ82gU/Tg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/wevm" + }, + "peerDependencies": { + "typescript": ">=5.0.4", + "zod": "^3.22.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/viem/node_modules/ox": { + "version": "0.14.63", + "resolved": "https://registry.npmjs.org/ox/-/ox-0.14.63.tgz", + "integrity": "sha512-TSKYN151boL1Aq/vsFrd8Buvd7v+oTQr8V7z4f4QQTSMB+ohD8cbNbHCtKRLI0Gs8LuYKirf5qvElTKVIFe2jw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "dependencies": { + "@adraffy/ens-normalize": "^1.11.0", + "@noble/ciphers": "^1.3.0", + "@noble/curves": "1.9.1", + "@noble/hashes": "^1.8.0", + "@scure/bip32": "^1.7.0", + "@scure/bip39": "^1.6.0", + "abitype": "^1.2.3", + "eventemitter3": "5.0.1" + }, + "peerDependencies": { + "typescript": ">=5.4.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/when-exit": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/when-exit/-/when-exit-2.1.5.tgz", + "integrity": "sha512-VGkKJ564kzt6Ms1dbgPP/yuIoQCrsFAnRbptpC5wOEsDaNsbCB2bnfnaA8i/vRs5tjUSEOtIuvl9/MyVsvQZCg==", + "license": "MIT" + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yocto-queue": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", + "license": "MIT", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.6.tgz", + "integrity": "sha512-4UmzQUF5fxvfcYHlbjBmejU+w+hbDnWFx4aS8kBx7nPBHcO2SqPbpAG8snbdg+Xse4pRefR98hgjFgIMCI/UhA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/services/filecoin/package.json b/services/filecoin/package.json new file mode 100644 index 00000000..0ed881bd --- /dev/null +++ b/services/filecoin/package.json @@ -0,0 +1,23 @@ +{ + "name": "@qmeshpy/filecoin-bridge", + "version": "1.0.0", + "description": "Calibration-only Filecoin storage bridge for QMeshPy", + "scripts": { + "test": "node --test bridge.test.mjs", + "check": "node --check bridge.mjs" + }, + "license": "Apache-2.0", + "type": "module", + "dependencies": { + "@filoz/synapse-core": "0.10.3", + "@filoz/synapse-sdk": "2.0.2", + "viem": "2.57.17" + }, + "private": true, + "engines": { + "node": ">=22" + }, + "overrides": { + "zod": "4.6.6" + } +}