diff --git a/ardrive-gql-verify/.gitignore b/ardrive-gql-verify/.gitignore new file mode 100644 index 0000000000..f8f7bcfc25 --- /dev/null +++ b/ardrive-gql-verify/.gitignore @@ -0,0 +1,57 @@ +# Dependencies +node_modules/ +yarn.lock + +# Build outputs +dist/ +build/ + +# Logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Coverage directory used by tools like istanbul +coverage/ +*.lcov + +# nyc test coverage +.nyc_output + +# Dependency directories +jspm_packages/ + +# Optional npm cache directory +.npm + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test +.env.local + +# IDEs +.vscode/ +.idea/ + +# OS generated files +.DS_Store +Thumbs.db + +# default output directory +output diff --git a/ardrive-gql-verify/.yarnrc b/ardrive-gql-verify/.yarnrc new file mode 100644 index 0000000000..123ac74a0a --- /dev/null +++ b/ardrive-gql-verify/.yarnrc @@ -0,0 +1 @@ +ignore-engines true diff --git a/ardrive-gql-verify/package.json b/ardrive-gql-verify/package.json new file mode 100644 index 0000000000..8889ec057f --- /dev/null +++ b/ardrive-gql-verify/package.json @@ -0,0 +1,29 @@ +{ + "name": "ardrive-gql", + "version": "1.0.0", + "description": "ArDrive GraphQL verification and comparison tool", + "author": "stevenyi@gmail.com", + "license": "MIT", + "private": true, + "main": "dist/index.js", + "bin": { + "ardrive-gql": "dist/index.js" + }, + "scripts": { + "build": "tsc", + "start": "yarn build && node dist/index.js", + "verify": "yarn build && node dist/index.js" + }, + "dependencies": { + "ardrive-core-js": "^3.0.3", + "arweave": "^1.15.7", + "commander": "^11.0.0", + "diff": "^8.0.2", + "graphql": "^16.8.1", + "graphql-request": "^7.2.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.0.0" + } +} diff --git a/ardrive-gql-verify/src/commands/drive-compare.ts b/ardrive-gql-verify/src/commands/drive-compare.ts new file mode 100644 index 0000000000..d31af0bd1b --- /dev/null +++ b/ardrive-gql-verify/src/commands/drive-compare.ts @@ -0,0 +1,141 @@ +import { arDriveAnonymousFactory, EntityID } from 'ardrive-core-js'; +import Arweave from 'arweave'; +import { Command } from 'commander'; +import { createPatch } from 'diff'; +import fs from 'fs'; +import path from 'path'; + +export const createDriveCompareCommand = () => { + const command = new Command('drive-compare'); + + command + .description('Compare drives between two gateways') + .requiredOption('--drive-id ', 'Drive ID to compare') + .requiredOption('--reference-gateway ', 'Reference gateway URL', 'https://arweave.net') + .requiredOption('--target-gateway ', 'Target gateway URL') + .option('--debug', 'Always show reference and target responses') + .option('--out-dir ', 'Output directory for JSON files', 'output') + .action(async (options) => { + const { driveId, referenceGateway, targetGateway, debug, outDir } = options; + + console.log('Drive Compare Command'); + console.log(`Drive ID: ${driveId}`); + console.log(`Reference Gateway: ${referenceGateway}`); + console.log(`Target Gateway: ${targetGateway}`); + console.log(`Output Directory: ${outDir}`); + + // Ensure output directory exists + if (!fs.existsSync(outDir)) { + fs.mkdirSync(outDir, { recursive: true }); + console.log(`Created output directory: ${outDir}`); + } + + try { + const parseGateway = (url: string) => { + const u = new URL(url); + return { + host: u.hostname, + port: u.port, + protocol: u.protocol.slice(0, -1) as 'https' | 'http', + timeout: 600000 + }; + }; + + const refConfig = parseGateway(referenceGateway); + const targetConfig = parseGateway(targetGateway); + + const arweaveRef = Arweave.init(refConfig); + const arweaveTarget = Arweave.init(targetConfig); + + const arDriveRef = arDriveAnonymousFactory({ arweave: arweaveRef }); + const arDriveTarget = arDriveAnonymousFactory({ arweave: arweaveTarget }); + + const entityDriveId = new EntityID(driveId); + + const driveRef = await arDriveRef.getPublicDrive({ driveId: entityDriveId }); + const driveTarget = await arDriveTarget.getPublicDrive({ driveId: entityDriveId }); + + const contentsRef = await arDriveRef.listPublicFolder({ folderId: driveRef.rootFolderId, maxDepth: 100, includeRoot: true }); + const contentsTarget = await arDriveTarget.listPublicFolder({ folderId: driveTarget.rootFolderId, maxDepth: 100, includeRoot: true }); + + // Sort contents by name + contentsRef.sort((a, b) => a.entityId.toString().localeCompare(b.entityId.toString())); + contentsTarget.sort((a, b) => a.entityId.toString().localeCompare(b.entityId.toString())); + + if (debug) { + console.log('Reference drive:', driveRef); + console.log('Target drive:', driveTarget); + console.log('Reference contents:', contentsRef); + console.log('Target contents:', contentsTarget); + } + + // check items in each collection for duplicates as well as missing items + const [refIds, duplicates] = contentsRef.reduce((acc, item) => { + const [ids, dupes] = acc; + const entityId = item.entityId.toString(); + if (ids.has(entityId)) { + dupes.add(entityId); + } else { + ids.add(entityId); + } + return acc; + }, [new Set(), new Set()]); + + const [targetIds, targetDupes] = contentsTarget.reduce((acc, item) => { + const [ids, dupes] = acc; + const entityId = item.entityId.toString(); + if (ids.has(entityId)) { + dupes.add(entityId); + } else { + ids.add(entityId); + } + return acc; + }, [new Set(), new Set()]); + + const missingIds = new Set([...refIds].filter((id) => !targetIds.has(id))); + const extraIds = new Set([...targetIds].filter((id) => !refIds.has(id))); + + fs.writeFileSync(path.join(outDir, 'missing-ids.json'), JSON.stringify([...missingIds], null, 2)); + fs.writeFileSync(path.join(outDir, 'extra-ids.json'), JSON.stringify([...extraIds], null, 2)); + fs.writeFileSync(path.join(outDir, 'duplicate-ids.json'), JSON.stringify([...duplicates], null, 2)); + fs.writeFileSync(path.join(outDir, 'target-duplicate-ids.json'), JSON.stringify([...targetDupes], null, 2)); + + // console.log('Missing IDs:', missingIds); + // console.log('Extra IDs:', extraIds); + // console.log('Duplicate IDs:', duplicates); + // console.log('Target duplicate IDs:', targetDupes); + + // JSON check + const refJson = JSON.stringify(contentsRef, null, 2); + const targetJson = JSON.stringify(contentsTarget, null, 2); + + console.log(`Reference JSON length: ${refJson.length}`); + console.log(`Target JSON length: ${targetJson.length}`); + + // write reference.json and target.json files + fs.writeFileSync(path.join(outDir, 'reference.json'), refJson); + fs.writeFileSync(path.join(outDir, 'target.json'), targetJson); + + const MAX_DIFF_SIZE = 1000000; // 1MB limit for diff computation + + if (refJson.length > MAX_DIFF_SIZE || targetJson.length > MAX_DIFF_SIZE) { + console.log('JSON sizes too large for detailed diff. Contents likely differ.'); + return; + } + + if (refJson === targetJson) { + console.log('Drive contents match.'); + } else { + console.log('Drive contents differ:'); + console.log('Starting diff computation...'); + const patch = createPatch('drive-contents', refJson, targetJson); + console.log('Diff computation completed.'); + console.log(patch); + } + } catch (error) { + console.error('Error comparing drives:', (error as Error).message, error); + } + }); + + return command; +} diff --git a/ardrive-gql-verify/src/commands/verify.ts b/ardrive-gql-verify/src/commands/verify.ts new file mode 100644 index 0000000000..f33a3b763f --- /dev/null +++ b/ardrive-gql-verify/src/commands/verify.ts @@ -0,0 +1,137 @@ +import { Command } from 'commander'; +import { diffLines } from 'diff'; +import * as fs from 'fs'; +import { GraphQLClient, gql } from 'graphql-request'; +import * as path from 'path'; + + +// const owner = 'N4h8M9A9hasa3tF47qQyNvcKjm4APBKuFs7vqUVm-SI'; +// const fileId = 'e4ef007d-d44b-430a-9a20-6f55e560aeea'; +// const driveId = '87082af3-741d-4620-a556-06b3e41c3d88'; +// const privateDriveId = 'caae3a63-6adb-4266-85bc-0e072b631cf4'; +// const dataTxId = 'QsSJfMlY92kg3lFNEudnaQcZbrEubpXEIRAVlal4L7U'; +// const dataTxId2 = 'LdZMi_m0dWbj3te2GFgrLkBoPnj9Zm4eEzA99FrM-yQ'; + +const owner = 'PgNXwu6EfxxwDlbOo83Eralb_j4VKPQ0SVoZa14vpy0' +const fileId = '6cc33df4-ad76-46df-b960-0d1a3c5e3fce' +const driveId = '17520b74-e487-46eb-890a-d76c4f70b0fc' +const dataTxId = 'wMi9eNXRTxJXNJSCkKzU936Kt0lELw1FMVb4wxmWvaY'; + +// keys are file names, values are parameters to pass in for each query +const queries: Record = { + "AllFileEntitiesWithId": { fileId, owner, after: null, lastBlockHeight: 1000000 }, + "DriveEntityHistory": { driveId, after: null, minBlockHeight: null, maxBlockHeight: null, ownerAddress: owner, entityType: "folder" }, + "DriveEntityHistoryWithEntityTypeFilter": { driveId, after: null, minBlockHeight: null, maxBlockHeight: null, ownerAddress: owner }, + "FirstDriveEntityWithIdOwner": { driveId, after: null }, + "FirstFileEntityWithIdOwner": { fileId, after: null }, + "FirstTxBlockHeightForWallet": { owner }, + "FirstTxForWallet": { owner }, + "InfoOfTransactionsToBePinned": { transactionIds: [dataTxId] }, + "InfoOfTransactionToBePinned": { txId: dataTxId }, + "LatestDriveEntityWithId": { driveId, owner, after: null }, + "LatestFileEntityWithId": { fileId, owner, after: null }, + "LicenseAssertions": { transactionIds: [dataTxId] }, + "LicenseDataBundled": { transactionIds: [dataTxId] }, + "PendingTxFees": { walletAddress: owner }, + "SingleTransaction": { txId: dataTxId }, + "SnapshotEntityHistory": { driveId, after: null, lastBlockHeight: 1749673, ownerAddress: owner }, + "TransactionsAtHeight": { owner: "vh-NTHVvlKZqRxc8LyyTNok65yQ55a_PJ1zWLb9G2JI", height: 1000000 }, + "TransactionStatuses": { transactionIds: [dataTxId] }, + "UserDriveEntities": { owner, after: null }, +} + +export const createVerifyCommand = () => { + const command = new Command('verify'); + + command + .description('Verify ArDrive GraphQL requests between reference and target gateways') + .option('--reference-gateway ', 'Reference gateway URL', 'https://arweave.net') + .requiredOption('--target-gateway ', 'Target gateway URL') + .option('--debug', 'Always show reference and target responses') + .action(async (options) => { + const { referenceGateway, targetGateway, wallet, debug } = options; + + console.log(`Reference Gateway: ${referenceGateway}`); + console.log(`Target Gateway: ${targetGateway}`); + + // Path to .graphql files relative to this script + const queriesDir = path.join(__dirname, '../../../lib/services/arweave/graphql/queries'); + + if (!fs.existsSync(queriesDir)) { + console.error(`Queries directory not found: ${queriesDir}`); + process.exit(1); + } + + // Load the TransactionCommon fragment + const fragmentPath = path.join(__dirname, '../../../lib/services/arweave/graphql/fragments/TransactionCommon.graphql'); + const transactionCommonFragment = fs.existsSync(fragmentPath) ? fs.readFileSync(fragmentPath, 'utf-8') : ''; + + const referenceClient = new GraphQLClient(`${referenceGateway}/graphql`); + const targetClient = new GraphQLClient(`${targetGateway}/graphql`); + + for (const [queryName, params] of Object.entries(queries)) { + const queryPath = path.join(__dirname, '../../../lib/services/arweave/graphql/queries', `${queryName}.graphql`); + + if (!fs.existsSync(queryPath)) { + console.error(`Query file not found: ${queryPath}`); + continue; + } + + let query = fs.readFileSync(queryPath, 'utf-8'); + + // If the query uses TransactionCommon fragment, prepend the fragment definition + if (query.includes('...TransactionCommon') && transactionCommonFragment) { + query = transactionCommonFragment + '\n\n' + query; + } + + try { + console.log(`\nVerifying query: ${queryName}`); + const referenceResult = await referenceClient.request(gql`${query}`, params); + const referenceJson = JSON.stringify(referenceResult, null, 2); + + const targetResult = await targetClient.request(gql`${query}`, params); + const targetJson = JSON.stringify(targetResult, null, 2); + + // Compare results using diff for better output + const diffs = diffLines(referenceJson, targetJson); + const hasDifferences = diffs.some((diff: any) => diff.added || diff.removed); + + if (!hasDifferences) { + console.log(`✓ Match for ${queryName}`); + if (debug) { + console.log(`Reference (${referenceGateway}):`, referenceJson); + console.log(`Target (${targetGateway}):`, targetJson); + } + } else { + console.log(`✗ Mismatch for ${queryName}`); + + // Show diff output + console.log('Differences:'); + diffs.forEach((part: any, index: number) => { + if (part.added) { + console.log(`\x1b[32m+ ${part.value}\x1b[0m`); // Green for additions + } else if (part.removed) { + console.log(`\x1b[31m- ${part.value}\x1b[0m`); // Red for removals + } else { + // Show context for unchanged parts (first few lines) + if (index < 5) { + const lines = part.value.split('\n').slice(0, 3); + console.log(` ${lines.join('\n ')}`); + } + } + }); + + if (debug) { + console.log(`Reference (${referenceGateway}):`, referenceJson); + console.log(`Target (${targetGateway}):`, targetJson); + } + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`Error with ${queryName}:`, message); + } + } + }); + + return command; +} diff --git a/ardrive-gql-verify/src/index.ts b/ardrive-gql-verify/src/index.ts new file mode 100644 index 0000000000..993a98ad8d --- /dev/null +++ b/ardrive-gql-verify/src/index.ts @@ -0,0 +1,18 @@ +#!/usr/bin/env node + +import { Command } from 'commander'; +import { createDriveCompareCommand } from './commands/drive-compare'; +import { createVerifyCommand } from './commands/verify'; + +const program = new Command(); + +program + .name('ardrive-gql') + .description('ArDrive GraphQL verification and comparison tool') + .version('1.0.0'); + +// Add commands +program.addCommand(createVerifyCommand()); +program.addCommand(createDriveCompareCommand()); + +program.parse(); diff --git a/ardrive-gql-verify/tsconfig.json b/ardrive-gql-verify/tsconfig.json new file mode 100644 index 0000000000..50572daa3e --- /dev/null +++ b/ardrive-gql-verify/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020"], + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +}