Mvp ranking publish

.github/workflows/build_and_test.yml

@@ -0,0 +1,37 @@
+name: Build project and run tests
+
+on: [ push, pull_request ]
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.14
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.14'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Build dockerfile and update dataset
+        run: |
+          python -m backend.launch_service.app_setup
+
+      - name: Create .env file for tests
+        run: |
+          cat > .env << EOF
+          EMAIL=test@example.com
+          APP_PASSWORD=test_password_123
+          DB_URL=sqlite://
+          EOF
+
+      - name: Run Python tests
+        run: |
+          python -m pytest -v

.github/workflows/linting.yml

@@ -0,0 +1,41 @@
+name: Lint and Format Check
+
+on: [ push, pull_request ]
+
+jobs:
+  lint-and-format:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run ruff check
+        uses: astral-sh/ruff-action@v3
+        with:
+          args: "check"
+
+      - name: Run ruff format check
+        uses: astral-sh/ruff-action@v3
+        with:
+          args: "format --check"
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: 'frontend/package-lock.json'
+
+      - name: Install frontend dependencies
+        run: |
+          cd frontend
+          npm ci
+
+      - name: Run ESLint
+        run: |
+          cd frontend
+          npm run lint
+
+      - name: Run Prettier check
+        run: |
+          cd frontend
+          npm run format:check

.gitignore

@@ -135,7 +135,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
+backend/.env
 .envrc
 .venv
 env/
@@ -171,7 +171,7 @@ cython_debug/
 # PyCharm
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  and can be added to the global gitignore or merged into this file_utils.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
@@ -184,14 +184,14 @@ cython_debug/
 # Visual Studio Code
 #  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
 #  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
-#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  and can be added to the global gitignore or merged into this file_utils. However, if you prefer,
 #  you could uncomment the following to ignore the entire vscode folder
 # .vscode/
 
 # Ruff stuff:
 .ruff_cache/
 
-# PyPI configuration file
+# PyPI configuration file_utils
 .pypirc
 
 # Cursor
@@ -205,3 +205,26 @@ cython_debug/
 marimo/_static/
 marimo/_lsp/
 __marimo__/
+
+#IDE
+.idea/
+.DS_Store
+
+# runstrat artifacts and config
+.env
+backend/tmp
+
+#Frontend
+node_modules/
+frontend/.gitignore
+frontend/README.md
+
+#dynamic created files
+frontend/src/components/components/Methods.ts
+backend/resources/dataset.json
+
+# Pre-commit
+.pre-commit-config.local
+
+# Memory benchmark scripts, inputs and results
+mem_benchmark/

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.2
+    hooks:
+      - id: ruff-check
+        args: [ --fix ]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-eslint
+    rev: v9.38.0
+    hooks:
+      - id: eslint
+        files: ^frontend/.*\.(js|jsx|ts|tsx)$
+        args: [ '--fix', '--config', 'frontend/eslint.config.js' ]
+        additional_dependencies:
+          - eslint@9.38.0
+          - '@typescript-eslint/eslint-plugin@8.58.1'
+          - '@typescript-eslint/parser@8.58.1'
+          - '@eslint/js@9.38.0'
+          - 'eslint-plugin-prettier@5.2.1'
+          - 'eslint-config-prettier@10.0.1'
+          - globals@15.12.0
+
+  - repo: local
+    hooks:
+      - id: prettier-frontend
+        name: Prettier Frontend
+        entry: bash -c 'cd frontend && npx prettier --write .'
+        language: system
+        files: ^frontend/.*\.(js|jsx|ts|tsx|json|css|md)$
+        pass_filenames: false

README.md

@@ -1,2 +1,211 @@
 # PySymBench
-Infrastructure for models comparison and evalustion
+
+Infrastructure for **AI model comparison and evaluation in symbolic execution workflows**.
+
+PySymBench is a **local web application** for running ONNX models as guiding strategies in symbolic execution. Experiments run inside Docker using [PySymGym](https://github.com/PySymGym/PySymGym) tools on a fixed test set; results are emailed back to the user and saved to a public leaderboard.
+
+The platform is designed to cover three target languages — **C#**, **Java** and **C++** — but only **C#** is available right now; Java and C++ are in development.
+
+## Features
+
+- **Run Experiment** — name the experiment, upload an ONNX model, pick a test set (currently C# or the "All" mode that runs the model against every available test set), and provide an email. The experiment is launched in Docker; metrics and artifacts are emailed back when it finishes. While the task is in progress it can be cancelled via a one-click link in the confirmation email.
+- **Model Ranking** — leaderboard of all completed experiments, split into tabs by language plus an **All Methods** tab. Rows are sorted by mean coverage (with ties broken by total tests, total errors, recency and runtime); every column is also sortable in the UI. Per-experiment metrics include mean/median coverage, total tests, errors, runtime and the share of methods that produced results.
+- **All Methods mode** — a dedicated experiment mode that runs the model against every language's test set. Each per-language run produces its own leaderboard entry (with that language's metrics), and an additional aggregated entry covering all languages is shown in the **All Methods** tab.
+- **Pairwise Comparison** — select any two experiments from a ranking tab and produce side-by-side comparison artifacts (PDFs), downloadable individually or as a single zip.
+- **Model Interface docs** — page describing the ONNX input/output specification a model must satisfy to be runnable by the experiment pipeline (tensor names, shapes, graph encoding).
+
+### Routes
+
+The frontend is a multi-page React SPA using `react-router-dom`:
+
+| Route | Page |
+|---|---|
+| `/` | Home — navigation hub |
+| `/experiment` | Run Experiment form |
+| `/ranking` | Model Ranking leaderboard + pairwise comparison |
+| `/interface` | Model Interface specification |
+
+### Backend API
+
+| Method | Path | Purpose |
+|---|---|---|
+| `POST` | `/api/upload` | Submit a new experiment (multipart: ONNX file, `email`, `language`, `experiment`) |
+| `GET` | `/api/status/{task_uid}` | Celery task state |
+| `POST` | `/api/cancel/{task_uid}` | Cancel a running experiment |
+| `GET` | `/api/cancel/{task_uid}?token=...` | One-click cancellation link sent by email |
+| `GET` | `/api/ranking?language=csharp\|java\|cpp\|all` | Leaderboard entries |
+| `POST` | `/api/compare` | Start a pairwise comparison between two experiment IDs |
+| `GET` | `/api/compare/{uid}/status` | Comparison task state and result file list |
+| `GET` | `/api/compare/{uid}/file/{name}` | Stream a single comparison artifact |
+| `GET` | `/api/compare/{uid}/files.zip` | Download all comparison PDFs as a zip |
+
+# Installation
+
+The repository contains **both frontend and backend components**, and **both must be launched** for the application to work.
+
+---
+
+## Email Communication (Gmail)
+
+To enable email delivery of results, add Gmail credentials to your `.env` file:
+
+```
+EMAIL=your_email@gmail.com
+APP_PASSWORD=your_app_password
+```
+
+`EMAIL` — your Gmail address
+`APP_PASSWORD` — your Gmail **App Password** (not your regular account password)
+
+---
+
+## Database (PostgreSQL)
+
+The ranking leaderboard stores experiment results in a PostgreSQL database. Add the connection URL to your `.env` file:
+
+```
+DB_URL=postgresql://user:password@localhost:5432/pysymbench
+```
+
+The required tables are created automatically on server startup. You can run a local PostgreSQL instance via Docker:
+
+```
+docker run --name postgres-pysymbench -e POSTGRES_USER=user -e POSTGRES_PASSWORD=password \
+  -e POSTGRES_DB=pysymbench -p 5432:5432 -d postgres
+```
+
+---
+
+## Object Storage (MinIO)
+
+Experiments store their ONNX model and result artifacts in MinIO; the pairwise comparison feature also reads artifacts from there. MinIO must be reachable — if it is not configured or unavailable, the task fails and the user is notified by email. Add the following to your `.env` file:
+
+```
+MINIO_ENDPOINT=localhost:9000
+MINIO_ACCESS_KEY=your_access_key
+MINIO_SECRET_KEY=your_secret_key
+MINIO_SECURE=false
+MINIO_BUCKET=pysymbench
+```
+
+You can run a local MinIO instance via Docker:
+
+```
+docker run --name minio -p 9000:9000 -p 9001:9001 \
+  -e MINIO_ROOT_USER=your_access_key -e MINIO_ROOT_PASSWORD=your_secret_key \
+  -d minio/minio server /data --console-address ":9001"
+```
+
+---
+
+## Redis (Celery Broker)
+
+By default, the Celery broker is expected at `redis://localhost:6379`. To use a remote Redis instance (e.g., for running Celery workers on separate machines), set `REDIS_URL` in your `.env` file:
+
+```
+REDIS_URL=redis://<host>:6379
+```
+
+All services that connect to Redis — the FastAPI app and every Celery worker — must have the same `REDIS_URL`. Workers on remote machines need only the `backend/` code, Docker, and access to the shared Redis instance.
+
+---
+
+## URLs for email links
+
+Cancellation links sent by email are absolute, so the backend needs to know its own public URL and the URL of the frontend. Defaults match a local setup; override them in `.env` if the app is reachable elsewhere:
+
+```
+BASE_URL=http://localhost:8000      # base URL of the FastAPI app
+FRONTEND_URL=http://localhost:5173  # base URL of the React frontend
+```
+
+---
+
+## Backend Setup
+
+1. Install **Python 3.14** and **Docker**, then install the project dependencies:
+
+```
+pip install -r requirements.txt
+```
+
+2. Run the application setup script (this builds a Docker container with the **PySymGym repository** and downloads the required dataset):
+
+```
+python -m backend.launch_service.app_setup
+```
+
+3. Start the **Celery broker (Redis)**:
+
+```
+docker run --name redis-for-celery -p 6379:6379 -d redis
+```
+
+4. Start the **Celery worker** and the **application server** (in separate terminals):
+
+```
+celery -A backend.utils.task worker --loglevel=info
+uvicorn backend.main:app
+```
+
+---
+
+## Frontend Setup
+
+1. Install **Node.js** with **npm**.
+
+2. Install frontend dependencies:
+
+```
+cd frontend
+npm install
+```
+
+3. Start the frontend development server:
+
+```
+npm run dev
+```
+
+Or build for production:
+
+```
+npm run build
+```
+
+### Frontend technology stack
+
+| Package | Purpose |
+|---|---|
+| `react-router-dom` | Client-side routing between pages |
+| `antd` | UI component library (forms, tables, buttons, modals) |
+| `tailwindcss` | Utility-first CSS framework |
+| `vite` | Build tool and dev server |
+
+---
+
+## Development
+
+### Python
+
+```
+ruff check .          # Lint
+ruff check . --fix    # Auto-fix
+ruff format .         # Format
+pytest -v             # Run tests
+```
+
+### Frontend
+
+```
+cd frontend
+npm run lint:fix      # ESLint auto-fix
+npm run format        # Prettier format
+npm run format:check  # Check formatting without writing
+```
+
+## CI/CD
+
+GitHub Actions runs on push/PR:
+- **`linting.yml`** — ruff check + format, ESLint + Prettier
+- **`build_and_test.yml`** — builds Docker image, runs `pytest -v`