diff --git a/.gitignore b/.gitignore index ffff840..0c6c822 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ build/ .vscode/ +__pycache__/ +/build diff --git a/.gitmodules b/.gitmodules index 6675e20..adf0bfd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,3 +22,6 @@ [submodule "workloads/linux/kvmtool/source/dtc"] path = workloads/linux/kvmtool/source/dtc url = https://git.kernel.org/pub/scm/utils/dtc/dtc.git +[submodule "workloads/linux/gapbs/source"] + path = workloads/linux/gapbs/source + url = https://github.com/sbeamer/gapbs.git diff --git a/workloads/linux/gapbs/README.md b/workloads/linux/gapbs/README.md new file mode 100644 index 0000000..0343096 --- /dev/null +++ b/workloads/linux/gapbs/README.md @@ -0,0 +1,97 @@ +# GAPBS Linux Workloads + +NEMU SimPoint checkpoint images for the +[GAP Benchmark Suite](https://github.com/sbeamer/gapbs). Each workload is one +`kernel × graph` pair: a static, single-threaded GAPBS kernel that loads a +pre-serialized reference graph and runs a single trial. There are 18 cases — +the cartesian product of six kernels and three graphs. + +| kernel | graph file | arguments | +|--------|---------------|-----------| +| bfs | `.sg` | `-n 1` | +| sssp | `.wsg` | `-d -n 1` (road `50000`; twitter/web `2`) | +| pr | `.sg` | `-i 1000 -t 1e-4 -n 1` | +| cc | `.sg` | `-n 1` | +| bc | `.sg` | `-i 1 -n 1` | +| tc | `U.sg` | `-n 1` | + +Graphs: `road`, `twitter`, `web`. Cases are named `_` +(e.g. `bfs_road`, `sssp_web`). + +## Building + +```sh +make gapbs-list # list the 18 case names +make linux/gapbs-bfs_road -jN # build one case +make gapbs-images -jN # build all 18 images +``` + +Images are written to `build/images/gapbs/bin/.fw_payload.bin`. + +## Graphs + +Kernels load a pre-serialized graph with `-f`, so graph construction is not +part of the profiled region. The serialized files are read from +`GAPBS_GRAPH_DIR`: + +```sh +make gapbs-images GAPBS_GRAPH_DIR=/path/to/serialized -jN +``` + +Convert the reference graphs once with the GAPBS `converter` (in `source/`): +`.sg` is used by bfs/pr/cc/bc, `.wsg` (weighted) by sssp, and `U.sg` +(symmetrized) by tc. + +## How large graphs are delivered (split + FIFO) + +Each serialized graph is bundled inside the initramfs. The Linux `newc` cpio +format stores every file's size in a 32-bit field, so **no single file may be +4 GiB or larger**. The twitter and web graphs are 9.5–30 GB and cannot be +shipped as one file. They are delivered without modifying GAPBS: + +1. **Build time** — a graph at or above 4 GiB is split into ordered + sub-4-GiB parts (`split -d -b 3G`) under `/gapbs/parts/`. Smaller graphs + (all `road` cases) are bundled whole. +2. **Run time** — the generated `run.sh` recreates the single stream GAPBS + expects through a **named FIFO** that carries the real `.sg`/`.wsg`/`U.sg` + suffix, streaming the parts into it in order and freeing each part as it is + consumed: + + ```sh + mkfifo /gapbs/ + ( for p in /gapbs/parts/.*; do cat "$p"; rm -f "$p"; done ) > /gapbs/ & + /usr/bin/ -f /gapbs/ + ``` + +GAPBS reads the FIFO exactly as it would a real file. This works because its +serialized reader is strictly sequential — it reads the header, then each CSR +array front-to-back, with no seek — so a forward-only pipe suffices. The +suffix must ride on the FIFO *name* because GAPBS selects serialized mode from +the filename extension. The split and stitch live entirely in the build +packager and the generated `run.sh`; the GAPBS source is untouched. + +## Memory and device tree + +A graph of size *G* peaks at roughly *2 × G* during the initramfs boot-unpack: +the cpio image and the unpacked rootfs coexist before the kernel frees the +source. The largest weighted graphs (`web.wsg` 30 GB, `twitter.wsg` 23 GB) +therefore need more than 64 GiB of guest RAM and are built against the +`mem128g` device-tree profile; all other cases fit the default `mem64g`: + +```sh +make linux/gapbs-sssp_web \ + GAPBS_DEFAULT_DTB=xiangshan-fpga-noAIA-mem128g-novec \ + GAPBS_DTB_MIN_MEMORY_BYTES=137438953472 -jN +``` + +NEMU must be built with `CONFIG_MSIZE` at least as large as the guest device +tree declares. Total initramfs beyond 4 GiB relies on the 64-bit +`linux,initrd-start/end` device-tree cells emitted by `build-firmware-linux.sh`. + +## Files + +- `gapbs-package.py` — lists cases; cross-compiles the kernel (static, + `SERIAL=1`); stages or splits the graph; writes `run.sh` + `inittab`. +- `build.sh` — thin wrapper invoked per case by the build system. +- `rules.mk` — generates the per-case build and image targets. +- `source/` — the GAPBS submodule (kernels and `converter`). diff --git a/workloads/linux/gapbs/build.sh b/workloads/linux/gapbs/build.sh new file mode 100644 index 0000000..5e8a154 --- /dev/null +++ b/workloads/linux/gapbs/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GAPBS_CASE:?GAPBS_CASE is required}" +: "${CROSS_COMPILE:?CROSS_COMPILE is required}" +: "${PKG_DIR:=$WORKLOAD_BUILD_DIR/package}" +: "${GAPBS_GRAPH_DIR:=/nfs/share/manyang/gapbs-graphs/serialized}" + +python3 "$WORKLOAD_DIR/gapbs-package.py" \ + --case "$GAPBS_CASE" \ + --src-dir "$WORKLOAD_DIR/source" \ + --graph-dir "$GAPBS_GRAPH_DIR" \ + --pkg-dir "$PKG_DIR" \ + --out-dir "$WORKLOAD_BUILD_DIR" \ + --cross-compile "$CROSS_COMPILE" diff --git a/workloads/linux/gapbs/gapbs-package.py b/workloads/linux/gapbs/gapbs-package.py new file mode 100755 index 0000000..2f31987 --- /dev/null +++ b/workloads/linux/gapbs/gapbs-package.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +import argparse, os, shutil, subprocess, sys +from pathlib import Path + +KERNELS = ("bfs", "sssp", "pr", "cc", "bc", "tc") +GRAPHS = ("road", "twitter", "web") + +# graph file suffix + per-kernel args by graph +SUFFIX = {"bfs": ".sg", "pr": ".sg", "cc": ".sg", "bc": ".sg", + "sssp": ".wsg", "tc": "U.sg"} +SSSP_DELTA = {"road": "50000", "twitter": "2", "web": "2"} + +# initramfs newc cpio caps each file at 4 GiB (2**32); split any graph at/above it. +SPLIT_THRESHOLD = 4 * 1024**3 +PART_SIZE = "3G" # each part is well under the 4 GiB cpio limit + +def kernel_args(kernel, graph): + if kernel == "bfs": return ["-n", "1"] + if kernel == "cc": return ["-n", "1"] + if kernel == "pr": return ["-i", "1000", "-t", "1e-4", "-n", "1"] + if kernel == "bc": return ["-i", "1", "-n", "1"] + if kernel == "sssp": return ["-d", SSSP_DELTA[graph], "-n", "1"] + if kernel == "tc": return ["-n", "1"] + raise RuntimeError(f"unknown kernel {kernel}") + +def all_cases(): + return [f"{k}_{g}" for g in GRAPHS for k in KERNELS] + +def parse_case(case): + kernel, graph = case.split("_", 1) + if kernel not in KERNELS or graph not in GRAPHS: + raise RuntimeError(f"unknown gapbs case {case!r}") + return kernel, graph + +def compile_kernel(src_dir, kernel, cross_compile, pkg_dir, log_dir): + log = log_dir / "build.log" + cxx = f"{cross_compile}g++ -static" + cmd = ["make", "-C", str(src_dir), "SERIAL=1", f"CXX={cxx}", kernel] + log_dir.mkdir(parents=True, exist_ok=True) + with open(log, "w") as f: + r = subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT, text=True) + if r.returncode != 0: + sys.stderr.write(f"kernel build failed; see {log}\n") + sys.exit(1) + dst = pkg_dir / "usr" / "bin" / kernel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_dir / kernel, dst) + dst.chmod(0o755) + +def stage_graph(graph_dir, graph, kernel, pkg_dir): + fname = f"{graph}{SUFFIX[kernel]}" + src = Path(graph_dir) / fname + if not src.is_file(): + raise RuntimeError(f"missing graph file: {src}") + gdir = pkg_dir / "gapbs" + gdir.mkdir(parents=True, exist_ok=True) + guest_path = f"/gapbs/{fname}" + if src.stat().st_size < SPLIT_THRESHOLD: + shutil.copy2(src, gdir / fname) # small graph: bundle whole + return guest_path, False + parts_dir = gdir / "parts" # large graph: split into parts + parts_dir.mkdir(parents=True, exist_ok=True) + # zero-padded numeric suffixes so a lexical glob concatenates them in order + subprocess.run(["split", "-d", "-a", "3", "-b", PART_SIZE, + str(src), f"{fname}."], cwd=parts_dir, check=True) + return guest_path, True # guest path is a FIFO fed from parts + +def write_runtime(pkg_dir, case, kernel, graph_path, streamed, args): + cmd = " ".join(["/usr/bin/" + kernel, "-f", graph_path, *args]) + lines = ["#!/bin/sh", "set -e", + f"echo '======== BEGIN {case} ========'", "date -R || true"] + if streamed: + # Rejoin sub-4-GiB parts into the single serialized stream GAPBS expects, + # through a named FIFO carrying the real .sg/.wsg suffix. GAPBS reads it + # strictly forward, so a pipe suffices; free each part as it is sent. + name = graph_path.rsplit("/", 1)[-1] + lines += [ + f"rm -f {graph_path}", + f"mkfifo {graph_path}", + f'( for p in /gapbs/parts/{name}.*; do cat "$p"; rm -f "$p"; done ) > {graph_path} &', + ] + lines += [f"echo 'CMD: {cmd}'", "set +e", cmd, "status=$?", "set -e", + "date -R || true", + f"echo '======== END {case} ========'", "exit $status", ""] + run_sh = pkg_dir / "gapbs" / "run.sh" + run_sh.write_text("\n".join(lines), encoding="utf-8") + run_sh.chmod(0o755) + etc = pkg_dir / "etc"; etc.mkdir(parents=True, exist_ok=True) + (etc / "inittab").write_text("::sysinit:nemu-exec /bin/sh /gapbs/run.sh\n", encoding="utf-8") + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--case") + ap.add_argument("--list-cases", action="store_true") + ap.add_argument("--src-dir") + ap.add_argument("--graph-dir", default=os.environ.get( + "GAPBS_GRAPH_DIR", "/nfs/share/manyang/gapbs-graphs/serialized")) + ap.add_argument("--pkg-dir") + ap.add_argument("--out-dir") + ap.add_argument("--cross-compile", default=os.environ.get("CROSS_COMPILE", "")) + args = ap.parse_args() + + if args.list_cases: + print(" ".join(all_cases())); return + for req in ("case", "src_dir", "pkg_dir", "out_dir"): + if not getattr(args, req): + ap.error(f"--{req.replace('_','-')} is required") + + kernel, graph = parse_case(args.case) + pkg_dir = Path(args.pkg_dir).resolve() + log_dir = Path(args.out_dir).resolve() / "logs" + compile_kernel(Path(args.src_dir).resolve(), kernel, args.cross_compile, pkg_dir, log_dir) + graph_path, streamed = stage_graph(args.graph_dir, graph, kernel, pkg_dir) + write_runtime(pkg_dir, args.case, kernel, graph_path, streamed, kernel_args(kernel, graph)) + +if __name__ == "__main__": + main() diff --git a/workloads/linux/gapbs/rules.mk b/workloads/linux/gapbs/rules.mk new file mode 100644 index 0000000..e186acd --- /dev/null +++ b/workloads/linux/gapbs/rules.mk @@ -0,0 +1,67 @@ +GAPBS_WORKLOAD_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +GAPBS_REPO_ROOT := $(abspath $(GAPBS_WORKLOAD_DIR)/../../..) +GAPBS_SELF_MAKEFILE := $(GAPBS_WORKLOAD_DIR)/rules.mk +GAPBS_ROOT_MAKEFILE := $(GAPBS_REPO_ROOT)/Makefile +GAPBS_RECURSE_MAKEFILE := $(if $(filter $(GAPBS_ROOT_MAKEFILE),$(abspath $(firstword $(MAKEFILE_LIST)))),$(GAPBS_ROOT_MAKEFILE),$(GAPBS_SELF_MAKEFILE)) +GAPBS_SCRIPTS_DIR := $(GAPBS_REPO_ROOT)/scripts +GAPBS_DTS_DIR := $(GAPBS_REPO_ROOT)/dts +GAPBS_BUILD_DIR ?= $(GAPBS_REPO_ROOT)/build/linux-workloads/gapbs +GAPBS_IMAGE_DIR ?= $(GAPBS_REPO_ROOT)/build/images/gapbs +GAPBS_HELPER := $(GAPBS_WORKLOAD_DIR)/gapbs-package.py +GAPBS_GRAPH_DIR ?= /nfs/share/manyang/gapbs-graphs/serialized +GAPBS_CROSS_COMPILE ?= $(if $(CROSS_COMPILE),$(CROSS_COMPILE),riscv64-unknown-linux-gnu-) +GAPBS_DEFAULT_DTB ?= xiangshan-fpga-noAIA-mem64g-novec +GAPBS_DTB_MIN_MEMORY_BYTES ?= 68719476736 +GAPBS_BUILDROOT_DIR ?= $(abspath $(if $(BUILDROOT_DIR),$(BUILDROOT_DIR),$(GAPBS_REPO_ROOT)/build/buildroot)) +GAPBS_LINUX_IMAGE ?= $(if $(LINUX_IMAGE),$(LINUX_IMAGE),$(GAPBS_BUILDROOT_DIR)/output/images/Image) +GAPBS_GCPT_BIN ?= $(if $(GCPT_BIN),$(GCPT_BIN),$(GAPBS_REPO_ROOT)/build/LibCheckpointAlpha/build/gcpt.bin) +GAPBS_SBI_BUILD_DIR ?= $(if $(SBI_BUILD_DIR),$(SBI_BUILD_DIR),$(GAPBS_REPO_ROOT)/build/opensbi) +GAPBS_SBI_BIN ?= $(if $(SBI_BIN),$(SBI_BIN),$(GAPBS_SBI_BUILD_DIR)/build/platform/generic/firmware/fw_jump.bin) +GAPBS_BUILDROOT_CROSS_COMPILE ?= $(GAPBS_BUILDROOT_DIR)/output/host/bin/riscv64-linux- +GAPBS_DTC ?= $(GAPBS_BUILDROOT_DIR)/output/host/bin/dtc +GAPBS_DTS_SOURCES := $(shell find $(GAPBS_DTS_DIR) -type f 2>/dev/null) +GAPBS_ALL_CASES := $(shell python3 $(GAPBS_HELPER) --list-cases) + +WORKLOAD_DIRS += $(GAPBS_BUILD_DIR) + +define add_gapbs_case +$(GAPBS_BUILD_DIR)/$(1)/download/sentinel: + @mkdir -p "$$(@D)" + @touch "$$@" + +$(GAPBS_BUILD_DIR)/$(1)/rootfs.cpio: $$(GAPBS_HELPER) $$(GAPBS_WORKLOAD_DIR)/build.sh $(GAPBS_BUILD_DIR)/$(1)/download/sentinel $$(GAPBS_SCRIPTS_DIR)/build-workload-linux.sh + @GAPBS_CASE="$(1)" \ + CROSS_COMPILE="$$(GAPBS_BUILDROOT_CROSS_COMPILE)" \ + GAPBS_GRAPH_DIR="$$(GAPBS_GRAPH_DIR)" \ + bash "$$(GAPBS_SCRIPTS_DIR)/build-workload-linux.sh" "$$(GAPBS_WORKLOAD_DIR)" "$(GAPBS_BUILD_DIR)/$(1)" + +$(GAPBS_BUILD_DIR)/$(1)/fw_payload.bin: $$(GAPBS_DTS_SOURCES) $$(GAPBS_GCPT_BIN) $$(GAPBS_SCRIPTS_DIR)/build-firmware-linux.sh $(GAPBS_BUILD_DIR)/$(1)/rootfs.cpio $$(GAPBS_LINUX_IMAGE) $$(GAPBS_SBI_BIN) + @printf '[gapbs] Assembling firmware for $(1)\n' + @CROSS_COMPILE="$$(GAPBS_BUILDROOT_CROSS_COMPILE)" \ + DTC="$$(GAPBS_DTC)" \ + DEFAULT_DTB="$$(GAPBS_DEFAULT_DTB)" \ + DTB_MIN_MEMORY_BYTES="$$(GAPBS_DTB_MIN_MEMORY_BYTES)" \ + bash "$$(GAPBS_SCRIPTS_DIR)/build-firmware-linux.sh" "$$(GAPBS_GCPT_BIN)" "$$(GAPBS_SBI_BUILD_DIR)" "$$(GAPBS_DTS_DIR)" "$$(GAPBS_LINUX_IMAGE)" "$(GAPBS_BUILD_DIR)/$(1)" + +linux/gapbs-$(1): $(GAPBS_BUILD_DIR)/$(1)/fw_payload.bin + +WORKLOAD_PHONY_TARGETS += linux/gapbs-$(1) + +$(GAPBS_IMAGE_DIR)/bin/$(1).fw_payload.bin: $(GAPBS_BUILD_DIR)/$(1)/fw_payload.bin + @mkdir -p "$(GAPBS_IMAGE_DIR)/bin" + @cp "$(GAPBS_BUILD_DIR)/$(1)/fw_payload.bin" "$(GAPBS_IMAGE_DIR)/bin/$(1).fw_payload.bin" +endef + +$(foreach case,$(GAPBS_ALL_CASES),$(eval $(call add_gapbs_case,$(case)))) + +gapbs-list: + @echo $(GAPBS_ALL_CASES) + +gapbs-images: + @[ -n "$(strip $(GAPBS_ALL_CASES))" ] || { echo '[gapbs] error: no cases -- gapbs-package.py --list-cases produced nothing'; exit 1; } + @for case in $(GAPBS_ALL_CASES); do \ + $(MAKE) --no-print-directory -f "$(GAPBS_RECURSE_MAKEFILE)" "$(GAPBS_IMAGE_DIR)/bin/$$case.fw_payload.bin" || exit $$?; \ + done + @printf '[gapbs] Output written to %s\n' "$(abspath $(GAPBS_IMAGE_DIR))" + +.PHONY: gapbs-list gapbs-images diff --git a/workloads/linux/gapbs/source b/workloads/linux/gapbs/source new file mode 160000 index 0000000..b5e3e19 --- /dev/null +++ b/workloads/linux/gapbs/source @@ -0,0 +1 @@ +Subproject commit b5e3e19c2845f22fb338f4a4bc4b1ccee861d026