Skip to content

feat(compiler)!: Out of source builds #2384

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ospencer wants to merge 5 commits into
base: oscar/gc-rebased
Choose a base branch
from
Open

feat(compiler)!: Out of source builds #2384

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5af3e7a
feat(compiler)!: Out of source builds
ospencer Apr 9, 2026
ea2bf0c
Remove library includes for graindoc and lsp
ospencer Apr 13, 2026
69826ee
format
ospencer Apr 13, 2026
46c9b77
fix stamps for js
ospencer Apr 13, 2026
bd7abaa
format js
ospencer Apr 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ myocamlbuild.ml
/*.gr
_esy/
pkg/
target/

# Autogenerated .messages file from Menhir
*.messages.generated
Expand Down
23 changes: 22 additions & 1 deletion cli/bin/grain.js
View file
Open in desktop
Copy link
Copy Markdown
Member

@spotandjake spotandjake Apr 17, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this pr allow us to drop the pkg.js file in the cli?

It only exists to map the build outputs to a target directory but we are doing that inside the compiler now?

(Also just remembered that we are going to need to update the playground to handle the out of source stuff before we release this version if we merge this).

Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
#!/usr/bin/env node

const commander = require("commander");
const path = require("path");
const exec = require("./exec.js");
const pkgJson = require("../package.json");

const stdlibPath = require("@grain/stdlib");

function defaultWasmLocation(file, options) {
const targetDir = options.targetDir
? path.resolve(options.targetDir)
: path.resolve("target");
const profile = options.release ? "release" : "debug";
const basename = path.basename(file).replace(/\.gr$/, ".wasm");
return path.join(targetDir, profile, basename);
}

function list(val) {
return val.split(",");
}
Expand Down Expand Up @@ -89,12 +99,23 @@ class GrainCommand extends commander.Command {
list,
[],
);
cmd.forwardOption(
"-L, --library <libs>",
"load libraries: -L name1=dir1,name2=dir2",
list,
[],
);
cmd.forwardOption(
"-S, --stdlib <path>",
"override the standard library with your own",
null,
stdlibPath,
);
cmd.forwardOption(
"--target-dir <dir>",
"directory where build artifacts are written",
);
cmd.forwardOption("--project-root <dir>", "project root directory");
cmd.forwardOption(
"--initial-memory-pages <size>",
"initial number of WebAssembly memory pages",
Expand Down Expand Up @@ -180,7 +201,7 @@ program
.action(function (file, options, program) {
const success = exec.grainc(file, options, program);
if (success) {
const outFile = options.o ?? file.replace(/\.gr$/, ".wasm");
const outFile = options.o ?? defaultWasmLocation(file, options);
Copy link
Copy Markdown
Member

@spotandjake spotandjake Apr 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How would we feel about setting options.o by default in the cli to the root directory so the wasm file is still copied out by our cli.

I think this provides a nicer user experience with the js api having the wasm file just be there.

Copy link
Copy Markdown
Member Author

@ospencer ospencer Apr 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's better to not have the file be in the source directory by default. grain file.gr will still run the file for you, and it's not hard to find. Plus, if you want it somewhere else you can always pass -o.

exec.grainrun(unprocessedArgs, outFile, options, program);
}
});
Expand Down
66 changes: 28 additions & 38 deletions compiler/grainc/grainc.re
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ open Grain;
open Grain_typed;
open Compile;
open Printf;
open Lexing;
open Filename;
open Cmdliner;

let () =
Expand Down Expand Up @@ -49,36 +47,33 @@ let error_wrapped = f =>
exit(2);
};

let compile_file = (~outfile=?, filename) => {
let outfile =
Option.value(
~default=Compile.default_object_filename(filename),
outfile,
);
ignore(Compile.compile_file(~outfile, filename));
};
let compile_file = (~outfile=?, filename) =>
error_wrapped(() => compile_file(~outfile?, filename));
let compile_file = (~object_outfile=?, filename) =>
error_wrapped(() =>
ignore(Compile.compile_file(~object_outfile?, filename))
);

let grainc = (single_file_mode, name, outfile) => {
let grainc = (single_file_mode, input_path, object_outfile) => {
Grain_utils.Config.set_root_config();

if (!Printexc.backtrace_status() && Grain_utils.Config.verbose^) {
Printexc.record_backtrace(true);
};

let name = Fp.toString(input_path);

if (single_file_mode) {
compile_file(~outfile?, name);
compile_file(~object_outfile?, name);
} else {
switch (Grain_utils.Config.wasi_polyfill^) {
| Some(name) =>
| Some(polyfill) =>
let poyfill = Fp.toString(polyfill);
Grain_utils.Config.preserve_config(() => {
Grain_utils.Config.compilation_mode := Grain_utils.Config.Runtime;
Module_resolution.load_dependency_graph(name);
Module_resolution.load_dependency_graph(poyfill);
let to_compile = Module_resolution.get_out_of_date_dependencies();
List.iter(compile_file, to_compile);
compile_file(name);
})
compile_file(poyfill);
});
| None => ()
};

Expand All @@ -90,21 +85,20 @@ let grainc = (single_file_mode, name, outfile) => {
if (Grain_utils.Config.statically_link^) {
let main_object = Compile.default_object_filename(name);
let outfile =
Option.value(~default=Compile.default_wasm_filename(name), outfile);
Option.value(
~default=Compile.default_wasm_filename(name),
object_outfile,
);
let dependencies = Module_resolution.get_dependencies();

error_wrapped(() => Link.link(~main_object, ~outfile, dependencies));
};
};

`Ok();
};

/** Converter which checks that the given output filename is valid */
Copy link
Copy Markdown
Member

@spotandjake spotandjake Apr 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why did we remove this comment? The function still looks todo the same thing?


let output_file_conv = {
let parse = s => {
let s_dir = dirname(s);
let s_dir = Filename.dirname(s);
Sys.file_exists(s_dir)
? if (Sys.is_directory(s_dir)) {
`Ok(s);
Expand All @@ -116,19 +110,17 @@ let output_file_conv = {
(parse, Format.pp_print_string);
};

let input_file_conv = {
open Arg;
let (prsr, prntr) = non_dir_file;

(filename => prsr(filename), prntr);
};

let input_filename = {
let doc = sprintf("Grain source file to compile");
let docv = "FILE";
Arg.(
required
& pos(~rev=true, 0, some(input_file_conv), None)
& pos(
~rev=true,
0,
some(Grain_utils.Filepath.Args.ExistingFile.cmdliner_converter),
None,
)
& info([], ~docv, ~doc)
);
};
Expand Down Expand Up @@ -156,12 +148,10 @@ let cmd = {
Cmd.v(
Cmd.info(Sys.argv[0], ~version, ~doc),
Term.(
ret(
Grain_utils.Config.with_cli_options(grainc)
$ single_file_mode
$ input_filename
$ output_filename,
)
Grain_utils.Config.with_cli_options(grainc)
$ single_file_mode
$ input_filename
$ output_filename
),
);
};
Expand Down
9 changes: 1 addition & 8 deletions compiler/graindoc/graindoc.re
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,7 @@ let compile_typed = file => {
Module_resolution.load_dependency_graph(file);
let to_compile = Module_resolution.get_out_of_date_dependencies();
List.iter(
file =>
ignore(
compile_file(
~hook=stop_after_object_emitted,
~outfile=Compile.default_wasm_filename(file),
file,
),
),
file => ignore(compile_file(~hook=stop_after_object_emitted, file)),
to_compile,
);
compile_file(~hook=stop_after_typed, file);
Expand Down
4 changes: 2 additions & 2 deletions compiler/src/codegen/compcore.re
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3317,8 +3317,8 @@ let compile_wasm_module =
};

switch (Config.profile^) {
| Some(Release) => Optimize_mod.optimize(wasm_mod)
| None => ()
| Debug => ()
| Release => Optimize_mod.optimize(wasm_mod)
};
wasm_mod;
};
Expand Down
2 changes: 1 addition & 1 deletion compiler/src/codegen/data_representations.re
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ let build_core_data_representations = (wasm_mod: Module.t) => {
switch (Type_builder.build_and_dispose(builder)) {
| Ok([ty]) =>
switch (name) {
| Some(name) when Config.profile^ != Some(Release) =>
| Some(name) when Config.profile^ != Release =>
Heap_type.set_type_name(wasm_mod, ty, name);
List.iteri(
(index, (name, _)) => {
Expand Down
6 changes: 3 additions & 3 deletions compiler/src/codegen/emitmod.re
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ let emit_object = (mashed: mash_program, outfile) => {
close_out(oc);
};

let oc = open_out_bin(outfile);
let oc = Fs_access.open_file_for_writing(outfile);

output_bytes(oc, Cmi_format.magic);
let version_length = String.length(Config.version);
Expand Down Expand Up @@ -106,8 +106,8 @@ let emit_binary = (asm, signature, outfile) => {
close_out(oc);
};
switch (Config.profile^) {
| Some(Release) => Binaryen.Settings.set_debug_info(false)
| _ => Binaryen.Settings.set_debug_info(true)
| Debug => Binaryen.Settings.set_debug_info(true)
| Release => Binaryen.Settings.set_debug_info(false)
};
let source_map_name =
if (Config.source_map^) {
Expand Down
10 changes: 5 additions & 5 deletions compiler/src/codegen/linkedtree.re
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,13 @@ let internal_name = (id, dep_id) => {
};

let link = (~main_object, dependencies) => {
let new_base_dir = Filepath.String.dirname;

let resolve = (~base_dir=?, mod_name) =>
Module_resolution.locate_unit_object_file(~base_dir?, mod_name);

let wasi_polyfill =
Option.map(
Module_resolution.get_object_name,
Config.wasi_polyfill_path(),
p => Module_resolution.get_object_name(Fp.toString(p)),
Config.wasi_polyfill^,
);

let dependencies =
Expand Down Expand Up @@ -75,6 +73,8 @@ let link = (~main_object, dependencies) => {

let process_mashtree = (~main, dep, tree) => {
let globals = tree.mash_code.globals;
let dep_base_dir =
Fp.toString(Fp.dirName(Filepath.String.derelativize(dep)));

let imports =
List.fold_left(
Expand Down Expand Up @@ -114,7 +114,7 @@ let link = (~main_object, dependencies) => {
}
| MImportGrain =>
let resolved_module =
resolve(~base_dir=new_base_dir(dep), import.mimp_mod);
resolve(~base_dir=dep_base_dir, import.mimp_mod);
process_import(resolved_module);
imports;
};
Expand Down
49 changes: 17 additions & 32 deletions compiler/src/compile.re
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,23 @@ type compilation_state = {
cstate_desc: compilation_state_desc,
cstate_filename: option(string),
cstate_object_outfile: option(string),
cstate_wasm_outfile: option(string),
};

type compilation_action =
| Continue(compilation_state)
| Stop;

let default_wasm_filename = name =>
Filepath.String.replace_extension(name, "wasm");
Module_resolution.source_output_filename(~ext="wasm", name);
let default_object_filename = name =>
Filepath.String.replace_extension(name, "gro");
Module_resolution.source_artifact_filename(~ext="gro", name);
let default_mashtree_filename = name =>
Filepath.String.replace_extension(name, "mashtree");
Module_resolution.source_artifact_filename(~ext="mashtree", name);

let save_mashed = (mashed, outfile) => {
switch (outfile) {
| Some(outfile) =>
let outfile = default_mashtree_filename(outfile);
let save_mashed = (mashed, filename) => {
switch (filename) {
| Some(filename) =>
let outfile = default_mashtree_filename(filename);
Grain_utils.Fs_access.ensure_parent_directory_exists(outfile);
let mash_string =
Sexplib.Sexp.to_string_hum @@ Mashtree.sexp_of_mash_program(mashed);
Expand Down Expand Up @@ -148,7 +147,7 @@ let next_state = ({cstate_desc, cstate_filename} as cs) => {
| Optimized(optimized) =>
let mashed = Transl_anf.transl_anf_program(optimized);
if (Config.debug^) {
save_mashed(mashed, cs.cstate_object_outfile);
save_mashed(mashed, cs.cstate_filename);
};
Mashed(mashed);
| Mashed(mashed) =>
Expand Down Expand Up @@ -225,23 +224,6 @@ let stop_after_object_emitted =
| {cstate_desc: ObjectEmitted} => Stop
| s => Continue(s);

let compile_wasi_polyfill = () => {
switch (Grain_utils.Config.wasi_polyfill^) {
| Some(file) =>
Grain_utils.Config.preserve_config(() => {
Grain_utils.Config.compilation_mode := Grain_utils.Config.Runtime;
let cstate = {
cstate_desc: Initial(InputFile(file)),
cstate_filename: Some(file),
cstate_wasm_outfile: Some(default_wasm_filename(file)),
cstate_object_outfile: Some(default_object_filename(file)),
};
ignore(compile_resume(~hook=stop_after_object_emitted, cstate));
})
| None => ()
};
};

let reset_compiler_state = () => {
Driver.reset();
Location.reset_exceptions();
Expand All @@ -253,26 +235,29 @@ let reset_compiler_state = () => {
Grain_utils.Warnings.reset_warnings();
};

let compile_string = (~hook=?, ~name=?, ~outfile=?, str) => {
let compile_string = (~hook=?, ~name=?, ~object_outfile=?, str) => {
Ident.setup();
let cstate = {
cstate_desc: Initial(InputString(str)),
cstate_filename: name,
cstate_wasm_outfile: outfile,
cstate_object_outfile: Option.map(default_object_filename, outfile),
cstate_object_outfile: object_outfile,
};
Grain_utils.Config.preserve_all_configs(() =>
compile_resume(~hook?, cstate)
);
};

let compile_file = (~hook=?, ~outfile=?, filename) => {
let compile_file = (~hook=?, ~object_outfile=?, filename) => {
Ident.setup();
let object_outfile =
switch (object_outfile) {
| Some(_) as o => o
| None => Some(default_object_filename(filename))
};
let cstate = {
cstate_desc: Initial(InputFile(filename)),
cstate_filename: Some(filename),
cstate_wasm_outfile: outfile,
cstate_object_outfile: Option.map(default_object_filename, outfile),
cstate_object_outfile: object_outfile,
};
Grain_utils.Config.preserve_all_configs(() =>
compile_resume(~hook?, cstate)
Expand Down
Loading
Loading