Gurobi Script Improvements

.gitignore

@@ -1,2 +1,3 @@
 /results/
-/data/
+**/data/
+**/__pycache__/

docs/source/index.rst

@@ -13,6 +13,8 @@ RL4Ising
    overview/introduction
    overview/key_concepts
    overview/content
+   overview/hardware_reqs
+
 
 .. toctree::
    :maxdepth: 2

docs/source/overview/hardware_reqs.rst

@@ -0,0 +1,10 @@
+# Software Requirements for FOCI Cluster
+
+A100 - CUDA 11.4 or later supported
+tensorflow-2.12.0 (minimum version for CUDA 11.8)
+python 3.8-3.11
+Minor compatibility is available for all versions within 11 meaning new code and old hardware versions should be supported with the right packages.
+
+Sources:
+<https://github.com/Open-Finance-Lab/RL4Ising/blob/main/src/envs/vca_environment.yaml>
+<https://www.tensorflow.org/install/source#gpu>

src/baseline/.ipynb_checkpoints/gurobi_qubo-checkpoint.ipynb

@@ -0,0 +1,177 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Prerequisites\n",
+    "Need to have Gurobi Optimizer and obtain license.\n",
+    "\n",
+    "1. Install Gurobi\n",
+    "- https://www.gurobi.com/downloads/gurobi-software/\n",
+    "- https://support.gurobi.com/hc/en-us/articles/4534161999889-How-do-I-install-Gurobi-Optimizer\n",
+    "2. Obtain License\n",
+    "- https://portal.gurobi.com/iam/licenses/list/\n",
+    "\n",
+    "\n",
+    "%pip install gurobipy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Imports\n",
+    "import os\n",
+    "import json\n",
+    "import numpy as np\n",
+    "import networkx as nx\n",
+    "from natsort import natsorted\n",
+    "import gurobipy as gp\n",
+    "from math import sqrt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def read_nxgraph(filename: str) -> nx.Graph(): # type: ignore\n",
+    "    graph = nx.Graph()\n",
+    "    with open(filename, 'r') as file:\n",
+    "        # lines = []\n",
+    "        line = file.readline()\n",
+    "        is_first_line = True\n",
+    "        while line is not None and line != '':\n",
+    "            if '//' not in line:\n",
+    "                if is_first_line:\n",
+    "                    strings = line.split(\" \")\n",
+    "                    num_nodes = int(strings[0])\n",
+    "                    num_edges = int(strings[1])\n",
+    "                    nodes = list(range(num_nodes))\n",
+    "                    graph.add_nodes_from(nodes)\n",
+    "                    is_first_line = False\n",
+    "                else:\n",
+    "                    node1, node2, weight = line.split()\n",
+    "                    graph.add_edge(int(node1), int(node2), weight=weight)\n",
+    "            line = file.readline()\n",
+    "    return graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def gurobi_maxcut(graph):\n",
+    "\n",
+    "    # Create QUBO matrix\n",
+    "    nodes = len(list(graph.nodes))\n",
+    "    J = nx.to_numpy_array(graph)\n",
+    "\n",
+    "    # Construct gurobi model\n",
+    "    model = gp.Model(\"maxcut_qubo\")\n",
+    "\n",
+    "    # Set time limit 1 hr\n",
+    "    model.Params.LogFile = \"../logs/gurobi.log\"\n",
+    "    model.setParam('TimeLimit', 3600)\n",
+    "    model.Params.LogToConsole = 1\n",
+    "    model.setParam(\"MIPGap\", 0.0)\n",
+    "\n",
+    "    # Create variable for each vertex\n",
+    "    x = model.addVars(nodes, vtype=gp.GRB.BINARY)\n",
+    "\n",
+    "    objective = gp.quicksum(\n",
+    "        -J[i, j] * (2 * x[i] - 1) * (2 * x[j] - 1) * (1/sqrt(nodes))\n",
+    "        for i in range(nodes) for j in range(i+1, nodes) if J[i, j] != 0.0\n",
+    "    )\n",
+    "    model.setObjective(objective, gp.GRB.MINIMIZE)\n",
+    "    \n",
+    "    # Solve\n",
+    "    model.optimize()\n",
+    "    obj_val = model.ObjVal\n",
+    "    obj_bnd = model.ObjBound\n",
+    "    solution = \"\"\n",
+    "    for i in range(nodes):\n",
+    "        solution += f\"{int(x[i].X)}\"\n",
+    "\n",
+    "    return obj_val, solution"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# graph = read_nxgraph(\"../data/vna/ER/100_SK_seed40.txt\")\n",
+    "# obj_val, sol = gurobi_maxcut(graph)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " 4081189 3127413   -8.84763   56   71   -7.15263   -8.84763  23.7%  29.7 1190s\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Parse all files in data_dir \n",
+    "data_dir = f\"../data/vna/SK\"\n",
+    "input_files = [ f for f in os.listdir(data_dir) ]\n",
+    "input_files = natsorted(input_files)\n",
+    "\n",
+    "# Write directories & files\n",
+    "results_dir = f\"../results/{'/'.join(data_dir.split('/')[2:])}\"\n",
+    "solutions_dir = f\"../solutions/{'/'.join(data_dir.split('/')[2:])}\"\n",
+    "os.makedirs(results_dir, exist_ok=True)\n",
+    "os.makedirs(solutions_dir, exist_ok=True)\n",
+    "\n",
+    "i = 0\n",
+    "for file in input_files:\n",
+    "    i += 1\n",
+    "    if i > 25: break\n",
+    "\n",
+    "    graph = read_nxgraph(f'{data_dir}/{file}') \n",
+    "    obj_val, sol = gurobi_maxcut(graph)\n",
+    "\n",
+    "    with open(f\"{results_dir}/GUROBI.txt\", \"a\") as f:\n",
+    "        f.write(f\"{obj_val}\\n\")\n",
+    "\n",
+    "    with open(f\"{solutions_dir}/GUROBI.txt\", \"a\") as f:\n",
+    "        f.write(f\"{sol}\\n\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

src/baseline/README.md

@@ -1,11 +1,95 @@
 
 # Baseline MIP Solvers
 
-
-
-| Solvers   | 
+| Solvers   |
 |-----------|
 | Gurobi    |
 | ILOG CPLEX|
 | COPT      |
 
+## gurobi.py
+
+### Single file
+
+python script.py path/to/graph1.txt -t 3600
+
+### Multiple specific files
+
+python script.py path/to/graph1.txt path/to/graph2.txt -t 3600
+
+### Directory Batch Mode (Recommended)
+
+Automatically finds and processes all .txt files recursively within a given directory, outputting the results to a CSV.
+
+python script.py -i src/data/graphs -o results/gurobi/newest --csv_out batch_results.csv
+
+---
+
+### Command Line Arguments
+
+| Flag | Long Name | Default | Description |
+| :--- | :--- | :--- | :--- |
+| paths | N/A | [specific test file] | Unflagged arguments at the end of the command are treated as manual file paths. |
+| -t | --time_limit | 3600 | Time limit for the Gurobi solver per graph, in seconds. |
+| -i | --in_dir | None | Input directory. Processes all .txt files found inside. |
+| -o | --out_dir | results/gurobi/newest | Output directory. All .log files, individual .txt solution files, and the CSV are saved here. Creates the folder if it does not exist. |
+| N/A | --csv_out | gurobi_batch_results.csv | The name of the aggregate CSV file. It is saved directly inside the --out_dir. |
+| N/A | --retry_license | False | Retry entries that previously failed due to license limits. |
+| N/A | --retry_timeouts | False | Retry entries that previously hit the time limit. |
+
+### Output Structure
+
+For every successfully solved graph, the script generates three pieces of data in the --out_dir:
+
+1. [graph_name].log: The raw solver output straight from Gurobi, containing simplex iterations, heuristic steps, and final bounds.
+2. [graph_name].txt: A summarized output file containing the Objective Value, Objective Bound, Duration, MIP Gap, and the raw binary sequence of the best solution.
+3. CSV Entry: A single row in the aggregate .csv file containing the Absolute Path to the original graph and its Objective Value (or state flag like TIMEOUT / LICENSE_LIMIT).
+
+---
+
+### State Management
+
+When running in Directory Batch Mode (-i), the script executes a Synchronization Phase before solving any graphs. It cross-references the existing output CSV with the .log files in the output directory to map the state of the data.
+
+### Smart Resumption & Interruption Handling
+
+If a batch is canceled midway using Ctrl+C, or if the system crashes:
+
+* Clean Exits: Pressing Ctrl+C immediately halts the batch. The script will not attempt to write incomplete data to the CSV.
+* Orphan Cleanup: On the next run, the script detects .log files containing "Solve interrupted". It automatically deletes these junk logs and incomplete .txt files, forcing a clean rerun of that specific graph.
+* Resume Capability: Fully completed graphs recorded in the CSV (or validated by a successful .log file) are permanently skipped, allowing large batches to resume exactly where they left off.
+
+### Timeout Management
+
+If Gurobi reaches a defined time limit (e.g., 3600 seconds) before finding the optimal solution:
+
+* Gurobi gracefully stops and saves the best objective value found up to that point.
+* The script records this partial solution to the .txt and .log files.
+* In the CSV, the value is appended with a (TIMEOUT) flag (e.g., -70.5 (TIMEOUT)) to allow for easy filtering of sub-optimal runs during data analysis.
+
+### Directory Timeout Skipping
+
+When processing large nested datasets, graphs within the same subfolder often share similar complexity and execution times. To optimize batch processing, the script tracks timeouts at the directory level:
+
+* Subfolder Monitoring: If any single graph triggers a time limit (TIMEOUT) during execution, the script flags the parent subfolder.
+* Automatic Bypassing: Once a subfolder is flagged, the script instantly bypasses all remaining unprocessed .txt files within that specific subfolder and moves forward to the next directory.
+* Prevention of Bottlenecks: This mechanism prevents the solver from stalling on a directory filled with massive graphs, allowing the batch to continue clearing out simpler subfolders elsewhere in the dataset.
+
+### License Limit Protection
+
+If a graph is too large for the current Gurobi license, the solver will throw a size-limit exception.
+
+* Instead of crashing the entire batch or entering an infinite retry loop, the script catches this specific error.
+* It writes LICENSE_LIMIT to the CSV for that specific graph.
+* On future runs, the script recognizes LICENSE_LIMIT as a "completed" state and skips the graph permanently, saving processing time.
+
+### Duplicate Conflict Handling
+
+If the script detects that a file has been processed twice with conflicting results (e.g., folders were moved and the script was run twice), it triggers an interactive terminal prompt:
+
+[!] DUPLICATE CONFLICT: 100_SK_seed33.txt
+    1: Keep First Value  -> -70.5
+    2: Keep Second Value -> -71.2
+    3: Delete Both (forces rerun)
+
+If the duplicate values are identical, the script silently resolves the conflict and keeps one entry, requiring no manual input.