diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..64c75ba --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,55 @@ +name: CI + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +env: + CARGO_TERM_COLOR: always + +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Rust toolchain + run: | + rustup update stable + rustup default stable + + - name: Run tests + run: cargo test --verbose + + stub-check: + name: Verify stub.lua + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Rust toolchain + run: | + rustup update stable + rustup default stable + + - name: Generate stub.lua + run: cargo run --release > generated_stub.lua + + - name: Compare with committed stub.lua + run: | + if diff -q stub.lua generated_stub.lua > /dev/null; then + echo "✓ stub.lua matches generated output" + else + echo "✗ stub.lua does not match generated output!" + echo "" + echo "Differences:" + diff stub.lua generated_stub.lua || true + echo "" + echo "Please regenerate stub.lua by running: cargo run > stub.lua" + exit 1 + fi diff --git a/Cargo.lock b/Cargo.lock index 5eb6acc..ba04d97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -230,9 +230,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "typenum", @@ -1035,20 +1035,19 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.8.0" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "198db74531d58c70a361c42201efde7e2591e976d518caf7662a47dc5720e7b6" +checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7" dependencies = [ "memchr", - "thiserror", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.8.0" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d725d9cfd79e87dccc9341a2ef39d1b6f6353d68c4b33c177febbe1a402c97c5" +checksum = "68f9dbced329c441fa79d80472764b1a2c7e57123553b8519b36663a2fb234ed" dependencies = [ "pest", "pest_generator", @@ -1056,9 +1055,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.0" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7d01726be8ab66ab32f9df467ae8b1148906685bbe75c82d1e65d7f5b3f841" +checksum = "3bb96d5051a78f44f43c8f712d8e810adb0ebf923fc9ed2655a7f66f63ba8ee5" dependencies = [ "pest", "pest_meta", @@ -1069,11 +1068,10 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.8.0" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0" +checksum = "602113b5b5e8621770cfd490cfd90b9f84ab29bd2b0e49ad83eb6d186cef2365" dependencies = [ - "once_cell", "pest", "sha2", ] @@ -1511,9 +1509,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -1693,26 +1691,6 @@ dependencies = [ "unicode-width 0.2.0", ] -[[package]] -name = "thiserror" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tinystr" version = "0.7.6" @@ -1867,9 +1845,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "ucd-trie" diff --git a/Cargo.toml b/Cargo.toml index c3995a5..bf4fdef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,15 +6,12 @@ version = "0.2.5" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] - clap = { version = "4.5.37", features = ["derive", "string"] } +pest = "2" +pest_derive = "2" regex = "1.11.1" reqwest = { version = "0.12.15", features = ["blocking"] } scraper = "0.23.1" serde = { version = "1.0.219", features = ["derive"] } - -# preserve_order uses "IndexMap" instead of "BTreeMap" preserving order after parsing. -pest = "2.7.9" -pest_derive = "2.7.9" textwrap = "0.16.2" toml = { version = "0.8.21", features = ["preserve_order"] } diff --git a/DESIGN.md b/DESIGN.md index 5e5b5f0..ef8266d 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -5,23 +5,25 @@ tldr: This is an overly complicated tool with the following parts: 1. DSL (domain specific language) for Lua typing information (LUARS) -2. PEG Grammar for LUARS using the [pest.rs](https://pest.rs) crate (see: [luars.pest](src/luars.pest)) -3. Manually documented Types for the PlaydateSDK expressed in LUARS (see: [playdate.luars](playdate.luars)) -4. A web scraper which scrapes the PlaydateSDK documentation using the [scraper](https://crates.io/crates/scraper) crate. -5. A rust which combines the scraped documentation and types and generates LuaLS compatible Type Annotations with documentation and types for the entire SDK. +2. Manually documented Types for the PlaydateSDK expressed in LUARS (see: [playdate.luars](playdate.luars)) +3. Grammar/Parser for LUARS using [pest.rs](https://pest.rs) crate (see: [luars.rs](./src/luars.rs)) +4. A web scraper which scrapes the PlaydateSDK HTML documentation using the [scraper](https://crates.io/crates/scraper) crate (see: [scraper.rs](./src/scraper.rs)) +5. Rust which combines the scraped documentation and types and generates LuaLS compatible Type Annotations with documentation and types for the entire SDK. ## No seriously, what the actual fuck? 1. I've been coding Lua for Playdate. -2. The PlaydateSDK has no types and leverages Lua dynamic nature for variadic params and returns. -3. There's a pretty good Language server for Lua which created a standard for Lua typing via comments (LuaCATS) -4. I've been learning rust. -6. I wrote a scraper for the PlaydateSDK docs (in rust) -7. For each function I generated a TOML skeleton and manually determined parameter and return types. -8. TOML turned out to overly verbose, fragile and unsearchable for ~1000 functions and ~3500 type definitions (~15K lines). -9. So I came up with a format for function signatures which only requires one line per function. -10. I learned PEG and wrote a parser for function signature format I came up with. -11. Iterate. Iterate. Iterate. +1. The PlaydateSDK has no types and leverages Lua dynamic nature for variadic params and returns. +1. There's a pretty good Language server for Lua (lua-language-server) with a standard for Lua typing via comments (LuaCATS) +1. I've been learning rust. +1. I wrote a scraper for the PlaydateSDK docs (in rust) +1. For each function I generated a TOML skeleton and manually determined parameter and return types. +1. TOML turned out to overly verbose, fragile and unsearchable for ~1000 functions and ~3500 type definitions (~15K lines). +1. So I came up with a format for function signatures which only requires one line per function. +1. I learned PEG and wrote a parser for function signature format I came up with using [pest.rs](https://pest.rs). +1. Iterate. Iterate. Iterate. +1. Years pass +1. I had Claude Code rewrite my LUARS parser and simplify my scraper. Now we can do static code analysis, type checking and autocomplete in VSCode and other IDEs that support the LuaLS Language Server's LUACATS style type annotation comments. @@ -37,4 +39,4 @@ Local is for table types used by the SDK. Fun is functions (and parameters, parameter types, returns, return types and optionality of each). -See the [LUARS pest.rs PEG Grammar](src/luars.pest) and [Playdate.luars](playdate.luars) for more. +See [Playdate.luars](playdate.luars) for examples. diff --git a/README.md b/README.md index a9afbb0..3ec77fd 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This repository contains the Playdate Lua API specification and tools for generating Lua Comment And Type System (LUACats) compliant Lua comments for use with the Lua Language Server ([sumneko.lua](https://marketplace.visualstudio.com/items?itemName=sumneko.lua)) -in VSCode, NeoVIM, etc. +in VSCode, Zed, NeoVIM, etc. These tools can optionally integrate the offical Playdate Docs into the generated output by scraping the API diff --git a/src/args.rs b/src/args.rs index 5f60aa9..4eedc61 100644 --- a/src/args.rs +++ b/src/args.rs @@ -68,13 +68,14 @@ fn fetch_url(url: String) -> String { } /// Retrieves the contents of the docs (from file or url) -pub fn fetch_docs(args: Args) -> String { - match args.url { - Some(url) => fetch_url(url), - _ => fetch_file(&args.path.clone().unwrap()), +pub fn fetch_docs(args: &Args) -> String { + match &args.url { + Some(url) => fetch_url(url.clone()), + None => fetch_file(args.path.as_ref().unwrap()), } } -pub fn setup() -> Args { +/// Parse command line arguments +pub fn parse() -> Args { Args::parse() } diff --git a/src/finstub.rs b/src/finstub.rs deleted file mode 100644 index 2107747..0000000 --- a/src/finstub.rs +++ /dev/null @@ -1,77 +0,0 @@ -use crate::luars::LuarsStatement; -use crate::stub::{Stub, StubFn, Table, TableContents}; - -/// Final Stubs, ready for outputting -pub enum FinStub { - FunctionStub(StubFn), - VariableStub(Table), -} - -impl FinStub { - pub fn from_stub(stub: &Stub) -> FinStub { - match stub { - Stub::Function(fn_stub) => FinStub::FunctionStub(fn_stub.clone()), - } - } - pub fn from_luars(luars: &LuarsStatement) -> FinStub { - let prefix: String = match luars { - LuarsStatement::Local(_, _, _) => "local ".to_string(), - _ => "".to_string(), - }; - match luars { - LuarsStatement::Global(table_name, table_type, table_contents) - | LuarsStatement::Local(table_name, table_type, table_contents) => { - let var_name = table_name.to_string(); - let var_type = table_type.to_string(); - let mut var_contents: Vec = Vec::new(); - for tc in table_contents { - let (key_name, key_type, key_value) = tc; - var_contents.push(TableContents { - name: key_name.to_string(), - r#type: key_type.to_string(), - value: key_value.to_string(), - }); - } - FinStub::VariableStub(Table { - prefix, - name: var_name, - r#type: var_type, - contents: var_contents, - }) - } - LuarsStatement::Function(fun_name, fun_params, fun_returns) => { - let title = fun_name.to_string(); - let anchor = "".to_string(); - let params: Vec<(String, String)> = fun_params - .iter() - .map(|(fname, ftype)| (fname.to_string(), ftype.to_string())) - .collect(); - let returns: Vec<(String, String)> = fun_returns - .iter() - .map(|(fname, ftype)| (fname.to_string(), ftype.to_string())) - .collect(); - let text: Vec = Vec::new(); - FinStub::FunctionStub(StubFn { - title, - anchor, - params, - returns, - text, - }) - } - } - } - pub fn generate_stub(&self) -> Vec { - // Returns the complete stub for a class or function. - let mut out: Vec = Vec::new(); - match self { - FinStub::VariableStub(stub) => { - out.extend(stub.get_luacats()); - } - FinStub::FunctionStub(stub) => { - out.extend(stub.get_luacats()); - } - } - out - } -} diff --git a/src/fixes.rs b/src/fixes.rs deleted file mode 100644 index 9a53cc3..0000000 --- a/src/fixes.rs +++ /dev/null @@ -1,172 +0,0 @@ -use crate::stub::StubFn; -use regex::Regex; -use serde::Deserialize; -use std::{collections::HashMap, sync::LazyLock}; - -#[derive(Deserialize)] -pub struct FunctionReplacement { - pub name: String, - pub parameters: Vec, -} - -// TOML Files -static TOML_STR_NOTES: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/Notes.toml")); -static TOML_STR_FUNCTION: &str = - include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/RenameFn.toml")); -static TOML_STR_INVALID: &str = - include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/Invalid.toml")); - -// Static HashMaps from TOML files -static NOTES: LazyLock>> = - LazyLock::new(|| toml::from_str(TOML_STR_NOTES).expect("Loading Notes.toml failed.")); -static RENAME_FUNCTION: LazyLock> = - LazyLock::new(|| toml::from_str(TOML_STR_FUNCTION).expect("Loading RenameFn.toml failed.")); -static INVALID: LazyLock> = - LazyLock::new(|| toml::from_str(TOML_STR_INVALID).expect("Loading Invalid.toml failed.")); - -/// HTML Link tags -static HTML_A: LazyLock = LazyLock::new(|| Regex::new(r"]*>").unwrap()); -static HTML_TAG: LazyLock = LazyLock::new(|| Regex::new(r"<[^>]*>").unwrap()); -/// Lua function signature: 'function(a,b,c)' -static LUA_FUNC: LazyLock = LazyLock::new(|| { - Regex::new( - &format!( - r"^(?P(?:{id}\.)*{id}[:\.]{id}|{id})\((?P.*)\)", - id = r"[\w_][\w\d_]*" - ) - .to_string(), - ) - .unwrap() -}); - -/// Given an anchor, return an notes from Notes.toml (hard coded stuff) -pub fn apply_notes(name: &String) -> Vec { - if let Some(note) = NOTES.get(name.as_str()) { - note.clone() - } else { - Vec::new() - } -} - -/// Given a function return a stub with overrides, parameter types and return types applied -pub fn apply_fn_types(anchor: &str, title: &String, text: &Vec) -> StubFn { - let name: String; - let params: Vec<(String, String)>; - - // Apply overrides - if let Some(fixed) = RENAME_FUNCTION.get(anchor) { - params = fixed - .parameters - .iter() - .map(|p| (p.clone(), "any".to_string())) - .collect(); - // eprintln!("WARN: Found function override: {} -> {}", anchor, fixed); - name = fixed.name.clone(); - } else { - (name, params) = params_from_title(title); - } - - let returns: Vec<(String, String)> = Vec::new(); - - StubFn { - title: name, - anchor: anchor.to_string(), - text: text.clone(), - params, - returns, - } -} - -// Takes a valid function signature and returns a function name and a vector of parameters. -fn params_from_title(title: &String) -> (String, Vec<(String, String)>) { - let mut params: Vec<(String, String)> = Vec::new(); - let lua_func = LUA_FUNC - .captures(title) - .expect(format!("ERROR: Could not parse function signature (typo?): {title}").as_str()); - - let params_str = lua_func.name("params").unwrap().as_str(); - let fname = lua_func.name("fname").unwrap().as_str(); - let mut optional = false; - if params_str.trim() != "" { - for p in params_str.split(",") { - let mut param_name = p.trim().to_string(); - // once with hit an open bracket every param afterwards is optional - // TODO: technically there's like one func(a, [b], callback) but let's ignore that. - if param_name.contains("[") { - optional = true; - } - if optional { - // strip the brackets - param_name = param_name - .clone() - .replace("[", "") - .replace("]", "") - .trim() - .to_string(); - // add the optional ? to param name - param_name = format!("{}?", param_name); - } - params.push((param_name.clone(), "any".to_string())); - // Ignore parameters shown in docs after the ... like: fun(a1, a2, ..., aN) - if param_name == "..." || param_name == "...?" { - break; - } - } - } - (fname.to_string(), clean_parameters(¶ms)) -} - -pub fn clean_text(text: String) -> String { - let t0 = text - .replace("", "`") - .replace("", "`") - .replace("
", "`")
-        .replace("
", "`") - .replace("", "*") - .replace("", "*") - .replace("", "**") - .replace("", "**") - .replace("\n", " ") - .replace("
", "\n---\n---") - .replace(">", ">") - .replace("<", "<") - .replace("", "{bundleid}") - .replace("", "{text}") - .replace("", "{message}") - .trim() - .to_string(); - let tn = HTML_A.replace_all(&t0, ""); - // The restuling Markdown should not have HTML tags. - if HTML_TAG.is_match(&tn) { - eprintln!( - "WARN: Extra HTML tag in description text: {}", - tn.to_string() - ); - } - tn.to_string() -} - -pub fn clean_code(text: String) -> Vec { - let mut lines: Vec = Vec::new(); - for line in text.lines() { - if line.trim() != "" { - // remove empty lines in middle of code blocks - lines.push(line.to_string()); - } - } - lines -} - -fn clean_parameters(params: &Vec<(String, String)>) -> Vec<(String, String)> { - let mut out: Vec<(String, String)> = Vec::new(); - for (p_name, lua_type) in params { - let p_an = p_name.replace("?", ""); // without "?" at the the end for optional - if let Some(fixed_name) = INVALID.get(&p_an) { - // eprintln!("WARN: Fixed invalid parameter: {p_an} -> {fixed_name}"); - out.push((fixed_name.clone(), lua_type.to_string())); - } else { - out.push((p_name.to_string(), lua_type.to_string())); - } - } - out -} diff --git a/src/luars.pest b/src/luars.pest index dbc53c4..38cfb65 100644 --- a/src/luars.pest +++ b/src/luars.pest @@ -1,117 +1,124 @@ -/* - Maybe do: - * Handle tabs and " " (non-breaking space) - * Handle comments - * Re-evaluate Pest Implicit whitepsace: https://pest.rs/book/grammars/syntax.html#implicit-whitespace - * Switch from "\n" to NEWLINE ("\n" | "\r\n" | "\r") - * Unicode support (not today satan) - * Better handling of accidential `))` end of function -*/ -Document = { Statement* } +// LUARS Grammar for Pest parser +// +// This grammar parses the .luars file format which defines Lua API types. +// Format examples: +// global json; +// fun json.decode(str: string): table; +// global playdate = { argv: string[], isSimulator: boolean }; +// local Timer: playdate.timer; + +Document = { SOI ~ Statement* ~ EOI } + Statement = _{ - ( - WHITE_LINE - | ((Function | Global | Local) ~ ";" ~ W ~ "\n"?) - ) + WHITE_LINE + | ((Function | Global | Local) ~ ";" ~ W ~ NEWLINE?) } + +// Keywords Fun_ = _{ "fun " } Local_ = _{ "local " } Global_ = _{ "global " } -Local = { - Local_ ~ Table -} -Global = { - Global_ ~ Table -} +// Top-level declarations +Local = { Local_ ~ Table } +Global = { Global_ ~ Table } + Function = { Fun_ ~ W ~ FunctionName ~ W - ~ oParen ~ W + ~ "(" ~ W ~ FunctionalParameters? ~ W - ~ cParen + ~ ")" ~ Return? } -TableConstants = { - TableKey ~ W ~ ":" ~ W ~ CaptureType ~ W ~ ("=" ~ W ~ IntegerValue ~ W)? - ~ ("," ~ W ~ TableKey ~ W ~ ":" ~ W ~ CaptureType ~ W ~ ("=" ~ W ~ IntegerValue ~W)?)* - ~ W ~ ","? -} -Return = { - ( - (":" ~ W ~ OptionalType ~ W) - | (":" ~ W ~ oParen ~ W ~ FunctionalParameters? ~ W ~ cParen ~ W) - ) - ~ Optional? -} -OptionalType = { TypeLua ~ Optional? } +// Table declaration: name, optional parent type, optional contents Table = _{ Identifier ~ W ~ (":" ~ W ~ CaptureType ~ W)? ~ ("=" ~ W ~ "{" ~ W ~ TableConstants? ~ W ~ "}" ~ W)? } + +TableConstants = { + TableField ~ (W ~ "," ~ W ~ TableField)* ~ W ~ ","? +} + +TableField = { + FieldName ~ W ~ ":" ~ W ~ CaptureType ~ (W ~ "=" ~ W ~ IntegerValue)? +} + +FieldName = { (QuotedString | DotDotDot | LuaIdentifier) ~ Optional? } + +// Function components FunctionName = { - (Identifier ~ ":" ~ Identifier) - | Identifier + (Identifier ~ ":" ~ Identifier) // Method: Class:method + | Identifier // Function: module.func } FunctionalParameters = { - (VariableParameter ~ W ~ ":" ~ W ~ OptionalType) - | ((FunctionalParameter ~ W ~ ("," ~ W ~ FunctionalParameter)*) ~ ("," ~ W ~ "...")? ~ ","?) + (VariableParameter ~ W ~ ":" ~ W ~ OptionalType) // Variadic only: ...?: any + | (FunctionalParameter ~ (W ~ "," ~ W ~ FunctionalParameter)* ~ (W ~ "," ~ W ~ DotDotDot)? ~ ","?) } -TableKey = !{ - ( DotDotDot | QuotedString | LuaIdentifier) ~ Optional? - | (LuaIdentifier ~ Optional? ~ W ~ ":" ~ W ~ (TypeLua)) -} -DotDotDot = _{ "..." } -// these optionals and ... are overly friendly -// foo(bar?: integer): (baz:integer?, boop:string?) -FunctionalParameter = _{ ParameterIdentifier ~ W ~ ":" ~ W ~ OptionalType} -ParameterIdentifier = { ("..." | LuaIdentifier) ~ Optional? } +FunctionalParameter = _{ ParameterIdentifier ~ W ~ ":" ~ W ~ OptionalType } +ParameterIdentifier = { (DotDotDot | LuaIdentifier) ~ Optional? } VariableParameter = { DotDotDot ~ Optional? } -oParen = _{ "(" } -cParen = _{ ")" } - -Optional = _{ "?" } -Identifier = @{ - LuaIdentifier ~ ("." ~ LuaIdentifier)* +Return = { + ":" ~ W ~ ( + ("(" ~ W ~ FunctionalParameters? ~ W ~ ")") // Multiple returns: (a: int, b: str) + | OptionalType // Single return: type + ) ~ Optional? } -LuaIdentifier = _{ ( "_" | ASCII_ALPHA ) ~ ("_" | ASCII_DIGIT | ASCII_ALPHA)* } - -IntegerValue = { ("-"? ~ ASCII_DIGIT+) } +// Type expressions +OptionalType = { TypeLua ~ Optional? } CaptureType = @{ TypeLua } + TypeLua = _{ - LT - | "(" ~ W ~ LT ~ W ~ ("|" ~ W ~ LT ~ W)+ ~ ")" + ParenthesizedUnion + | SingleType } -LT = _{ - | ("fun(" ~ W ~ FunctionalParameters? ~ W ~ ")" ~ W ~ (":" ~ W ~ UnparsedType ~ Optional? ~ W)?) - | ("table<" ~ W ~ Identifier ~ W ~ "," ~ W ~ TypeLua ~ W ~ ">" ~ W) - | UnparsedType + +ParenthesizedUnion = _{ "(" ~ W ~ SingleType ~ (W ~ "|" ~ W ~ SingleType)+ ~ W ~ ")" } + +SingleType = _{ + FunctionType + | TableGeneric + | BasicType +} + +FunctionType = _{ + "fun(" ~ W ~ FunctionalParameters? ~ W ~ ")" ~ (W ~ ":" ~ W ~ UnparsedType ~ Optional?)? } + +TableGeneric = _{ + "table<" ~ W ~ Identifier ~ W ~ "," ~ W ~ TypeLua ~ W ~ ">" +} + +BasicType = _{ UnparsedType } + UnparsedType = _{ - Identifier - ~ ("[][][]" | "[][]" | "[]")? + Identifier ~ ArraySuffix? } -QuotedString = _{ Quote ~ StringLiteral ~ Quote } -Quote = _{ "\"" } -StringLiteral = { QuotedChar* } -QuotedChar = _{ !"\"" ~ ANY } +ArraySuffix = _{ "[][][]" | "[][]" | "[]" } +// Identifiers +Identifier = @{ LuaIdentifier ~ ("." ~ LuaIdentifier)* } +LuaIdentifier = _{ ("_" | ASCII_ALPHA) ~ ("_" | ASCII_DIGIT | ASCII_ALPHA)* } -// TypeLua = @{ -// (LuaCatsBuiltIn ~ "?") -// | (LuaCatsBuiltIn ~ "[]") -// | (LuaCatsBuiltIn ~ ("|" ~ LuaCatsBuiltIn)*) -// } +// Literals +IntegerValue = @{ "-"? ~ ASCII_DIGIT+ } +QuotedString = _{ "\"" ~ StringContent ~ "\"" } +StringContent = @{ QuotedChar* } +QuotedChar = _{ !"\"" ~ ANY } + +// Tokens +DotDotDot = _{ "..." } +Optional = _{ "?" } -// LuaCatsBuiltIn = { "any" | "nil" | "boolean" | "string" | "number" | "function" | "table" | "userdata" | "thread" | "integer" | "float" } -// LuaTypes_non_nil = @{ "boolean" | "string" | "number" | "function" | "table" | "userdata" | "thread" | "integer" | "float" } -// ConsType = { "boolean" | "string" | "number" | "integer" | "float" } -W = _{ (" " | "\n" )* } -WHITE_LINE = _{ " "* ~ "\n" } +// Whitespace +W = _{ (" " | "\t" | NEWLINE)* } +WHITE_LINE = _{ " "* ~ NEWLINE } +NEWLINE = _{ "\r\n" | "\n" | "\r" } diff --git a/src/luars.rs b/src/luars.rs index d580170..372f51f 100644 --- a/src/luars.rs +++ b/src/luars.rs @@ -1,352 +1,368 @@ -use std::{cmp::Ordering, collections::BTreeMap}; +//! LUARS parser using Pest +//! +//! This module parses the .luars file format which defines Lua API types. -use pest::{Parser, iterators::Pair}; +use pest::Parser; use pest_derive::Parser; +use std::collections::BTreeMap; #[derive(Parser)] #[grammar = "luars.pest"] pub struct LuarsParser; -#[derive(Debug, PartialEq, Eq, Hash)] -pub enum LuarsStatement<'a> { - /// Global Tables (name, parent, attributes: Vec<(name, type, value)) - Global(&'a str, &'a str, Vec<(&'a str, &'a str, &'a str)>), - /// Local Tables (name, parent, attributes: Vec<(name, type, value)) - Local(&'a str, &'a str, Vec<(&'a str, &'a str, &'a str)>), - /// Function (name, parameters: Vec<(pname, ptype)>, returns: Vec<(rname, rtype)>) - Function(&'a str, Vec<(&'a str, &'a str)>, Vec<(&'a str, &'a str)>), +/// Parsed statement from a .luars file +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Statement { + /// Global table: (name, parent_type, fields) + Global(String, String, Vec), + /// Local type alias: (name, parent_type, fields) + Local(String, String, Vec), + /// Function: (name, params, returns) + Function(String, Vec, Vec), } -#[derive(PartialEq, Eq, Hash, PartialOrd, Ord)] -struct LuarsSortKey<'a> { - id: usize, // [1, 2, 3, 4] = [global, local, method, instance method - namespace: &'a str, // namespace: playdate.graphics.image - name: &'a str, // complete name: playdate.graphics.image:draw - i_or_c: isize, // [0, 1] = [class, instance] - sub_id: isize, // number of parameters (longer first) +/// A field in a table definition +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Field { + pub name: String, + pub typ: String, + pub value: String, } -impl LuarsStatement<'_> { - fn id(&self) -> LuarsSortKey { - let (id, name, i_or_c, sub_id) = match self { - LuarsStatement::Global(name, _, _) => (1, name, 0, 0), - LuarsStatement::Local(name, _, _) => (2, name, 0, 0), - LuarsStatement::Function(name, params, _) => { - if name.contains(":") { - (3, name, 1, -1 * params.len() as isize) // instance methods - } else { - (3, name, 0, -1 * params.len() as isize) // class methods - } - } - }; - let (namespace, name) = if name.contains(":") { - name.split_at(name.rfind(":").unwrap()) - } else if name.contains(".") { - name.split_at(name.rfind(".").unwrap()) - } else { - ("", *name) - }; - LuarsSortKey { - namespace, - id, - name, - i_or_c, - sub_id, - } - } +/// A function parameter or return value +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Param { + pub name: String, + pub typ: String, +} - /// The rough lua code equivalent. Used for hashmap keys and matching +impl Statement { + /// Generate the Lua function signature (used as map key) pub fn lua_def(&self) -> String { match self { - LuarsStatement::Function(name, params, _) => { - let func_params: Vec = params + Statement::Function(name, params, _) => { + let param_names: Vec<&str> = params .iter() - .map(|(fname, _)| fname.trim_matches('?').to_string()) - .collect::>(); - format!("{name}({})", func_params.join(", ")) + .map(|p| p.name.trim_end_matches('?')) + .collect(); + format!("{}({})", name, param_names.join(", ")) } - LuarsStatement::Local(name, _parent, _attrs) => { - format!("local {name} = {{}}") - } - LuarsStatement::Global(name, _parent, _attrs) => { - format!("{name} = {{}}") + Statement::Local(name, _, _) => format!("local {} = {{}}", name), + Statement::Global(name, _, _) => format!("{} = {{}}", name), + } + } + + /// Sort key for consistent ordering + fn sort_key(&self) -> (usize, String, String, isize, isize) { + match self { + Statement::Global(name, _, _) => (1, namespace(name), name.clone(), 0, 0), + Statement::Local(name, _, _) => (2, namespace(name), name.clone(), 0, 0), + Statement::Function(name, params, _) => { + let i_or_c = if name.contains(':') { 1 } else { 0 }; + (3, namespace(name), name.clone(), i_or_c, -(params.len() as isize)) } } } } -impl PartialOrd for LuarsStatement<'_> { - fn partial_cmp(&self, other: &Self) -> Option { - self.id().partial_cmp(&other.id()) +fn namespace(name: &str) -> String { + if let Some(pos) = name.rfind(':') { + name[..pos].to_string() + } else if let Some(pos) = name.rfind('.') { + name[..pos].to_string() + } else { + String::new() } } -impl Ord for LuarsStatement<'_> { - fn cmp(&self, other: &Self) -> Ordering { - self.id().cmp(&other.id()) +/// Parse a Global or Local table declaration +fn parse_table(pair: pest::iterators::Pair) -> (String, String, Vec) { + let mut name = String::new(); + let mut parent = String::new(); + let mut fields = Vec::new(); + + for inner in pair.into_inner() { + match inner.as_rule() { + Rule::Identifier => name = inner.as_str().to_string(), + Rule::CaptureType => parent = inner.as_str().to_string(), + Rule::TableConstants => fields = parse_table_constants(inner), + _ => {} + } } + + (name, parent, fields) } -pub fn parse_tbl(pair: Pair) -> LuarsStatement { - let localglobal = match pair.as_rule() { - Rule::Global => LuarsStatement::Global, - Rule::Local => LuarsStatement::Local, - _ => { - eprintln!( - "Unexpected Rule: {:?}. Was expecting Local or Global.", - pair.as_rule() - ); - unreachable!() - } - }; - let mut iterator = pair.into_inner(); - let mut obj_name: &str = "INVALID"; - let mut obj_type: &str = ""; - let mut obj_proto: Vec<(&str, &str, &str)> = Vec::new(); - while iterator.peek().is_some() { - let chunk = iterator.next().unwrap(); - match chunk.as_rule() { - Rule::Identifier => { - obj_name = chunk.as_str(); - } - Rule::CaptureType => { - obj_type = chunk.as_str(); +/// Parse table fields: { name: type, name2: type2 = value, ... } +fn parse_table_constants(pair: pest::iterators::Pair) -> Vec { + let mut fields = Vec::new(); + + for field_pair in pair.into_inner() { + if field_pair.as_rule() == Rule::TableField { + let mut name = String::new(); + let mut typ = String::new(); + let mut value = String::new(); + + for inner in field_pair.into_inner() { + match inner.as_rule() { + Rule::FieldName => name = inner.as_str().to_string(), + Rule::CaptureType => typ = inner.as_str().to_string(), + Rule::IntegerValue => value = inner.as_str().to_string(), + _ => {} + } } - Rule::TableConstants => { - let mut field = chunk.into_inner(); - let mut field_name: &str; - let mut field_type: &str; - let mut field_value: &str; - while field.peek().is_some() { - let obj = field.next().unwrap(); - if obj.as_rule() == Rule::TableKey { - field_name = obj.as_str(); - if field.peek().is_some() - && field.peek().unwrap().as_rule() == Rule::CaptureType - { - field_type = field.next().unwrap().as_str(); - if field.peek().is_some() - && field.peek().unwrap().as_rule() == Rule::IntegerValue - { - field_value = field.next().unwrap().as_str(); - obj_proto.push((field_name, field_type, field_value)); - } else { - obj_proto.push((field_name, field_type, "")); - } - } else { - eprint!("Unexpected parse: {:?} {:#?}", obj.as_rule(), obj); - unreachable!() - } - } else { - eprintln!("Unexpected parse: {:?} {:#?}", obj.as_rule(), obj); - unreachable!() + + fields.push(Field { name, typ, value }); + } + } + + fields +} + +/// Parse function parameters +fn parse_parameters(pair: pest::iterators::Pair) -> Vec { + let mut params = Vec::new(); + + let mut inner = pair.into_inner().peekable(); + while let Some(item) = inner.next() { + match item.as_rule() { + Rule::ParameterIdentifier | Rule::VariableParameter => { + let name = item.as_str().to_string(); + // Next should be OptionalType + if let Some(type_pair) = inner.next() { + if type_pair.as_rule() == Rule::OptionalType { + let typ = type_pair.as_str().to_string(); + params.push(Param { name, typ }); } } } - _ => { - eprintln!("Rule: {:?}", chunk.as_rule()); - unreachable!() - } + _ => {} } } - localglobal(obj_name, obj_type, obj_proto) + + params } -pub fn parse_function(pair: Pair) -> LuarsStatement { - let iterator = pair.into_inner(); - let mut name: &str = "INVALID"; - let mut params: Vec<(&str, &str)> = Vec::new(); - let mut returns: Vec<(&str, &str)> = Vec::new(); - for chunk in iterator { - match chunk.as_rule() { - Rule::FunctionName => { - name = chunk.as_str(); - } +/// Parse function return type(s) +fn parse_return(pair: pest::iterators::Pair) -> Vec { + let mut returns = Vec::new(); + + for inner in pair.into_inner() { + match inner.as_rule() { Rule::FunctionalParameters => { - let mut field = chunk.into_inner(); - while field.peek().is_some() { - let field_name: &str = field.next().unwrap().as_str(); - let field_type: &str = field.next().unwrap().as_str(); - params.push((field_name, field_type)); - } + // Multiple named returns: (x: int, y: int) + returns = parse_parameters(inner); } - Rule::Return => { - let mut field = chunk.into_inner(); - match field.peek().unwrap().as_rule() { - Rule::OptionalType => { - returns.push(("", field.next().unwrap().as_str())); - } - Rule::FunctionalParameters => { - let mut field = field.next().unwrap().into_inner(); - while field.peek().is_some() { - let field_name: &str = field.next().unwrap().as_str(); - let field_type: &str = field.next().unwrap().as_str(); - returns.push((field_name, field_type)); - } - } - _ => { - eprintln!("Rule: {:?}", field.peek()); - unreachable!() - } - } - } - _ => { - eprintln!("Rule: {:?}", chunk.as_rule()); - unreachable!() + Rule::OptionalType => { + // Single unnamed return + returns.push(Param { + name: String::new(), + typ: inner.as_str().to_string(), + }); } + _ => {} } } - LuarsStatement::Function(name, params, returns) + + returns } -pub fn parse_document(unparsed_file: &str) -> BTreeMap { - let document = LuarsParser::parse(Rule::Document, &unparsed_file) - .expect("unsuccessful parse") - .next() - .unwrap(); - - let mut statements: Vec = Vec::new(); - - for line in document.into_inner() { - let f = match line.as_rule() { - Rule::Global => parse_tbl(line), - Rule::Local => parse_tbl(line), - Rule::Function => parse_function(line), - _ => { - eprintln!("Rule: {:?}", line.as_rule()); - unreachable!() - } - }; - //println!("{:?}", f); - statements.push(f); + +/// Parse a function declaration +fn parse_function(pair: pest::iterators::Pair) -> Statement { + let mut name = String::new(); + let mut params = Vec::new(); + let mut returns = Vec::new(); + + for inner in pair.into_inner() { + match inner.as_rule() { + Rule::FunctionName => name = inner.as_str().to_string(), + Rule::FunctionalParameters => params = parse_parameters(inner), + Rule::Return => returns = parse_return(inner), + _ => {} + } + } + + Statement::Function(name, params, returns) +} + +/// Parse a .luars document and return a sorted map of statements +pub fn parse_document(input: &str) -> Result, String> { + let pairs = LuarsParser::parse(Rule::Document, input) + .map_err(|e| format!("Parse error: {}", e))?; + + let mut statements = Vec::new(); + + for pair in pairs { + for inner in pair.into_inner() { + let stmt = match inner.as_rule() { + Rule::Global => { + let (name, parent, fields) = parse_table(inner); + Statement::Global(name, parent, fields) + } + Rule::Local => { + let (name, parent, fields) = parse_table(inner); + Statement::Local(name, parent, fields) + } + Rule::Function => parse_function(inner), + Rule::EOI => continue, + _ => continue, + }; + statements.push(stmt); + } } - statements.sort(); - let mut out = BTreeMap::new(); - for statement in statements { - let key = statement.lua_def(); - if !out.contains_key(&key) { - out.insert(key, statement); + + // Sort by namespace, type, name for consistent output + statements.sort_by(|a, b| a.sort_key().cmp(&b.sort_key())); + + let mut result = BTreeMap::new(); + for stmt in statements { + let key = stmt.lua_def(); + if !result.contains_key(&key) { + result.insert(key, stmt); } else { - eprint!("Duplicate definition of {}", key); + eprintln!("Duplicate definition: {}", key); } } - out + + Ok(result) } #[cfg(test)] mod tests { - use std::fs; - - // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; + + #[test] + fn test_simple_global() { + let result = parse_document("global json;").unwrap(); + assert_eq!(result.len(), 1); + assert!(result.contains_key("json = {}")); + } + + #[test] + fn test_global_with_parent() { + let result = parse_document("global playdate.sound.twopolefilter: SoundEffect;").unwrap(); + let stmt = result.get("playdate.sound.twopolefilter = {}").unwrap(); + match stmt { + Statement::Global(name, parent, _) => { + assert_eq!(name, "playdate.sound.twopolefilter"); + assert_eq!(parent, "SoundEffect"); + } + _ => panic!("Expected Global"), + } + } + #[test] - fn global_simple() { - let document = LuarsParser::parse(Rule::Global, "global json;\n") - .expect("unsuccessful parse") - .next() - .unwrap(); - assert_eq!( - parse_tbl(document), - LuarsStatement::Global("json", "", Vec::new()) - ); + fn test_global_with_contents() { + let input = "global playdate = { argv: string[], isSimulator: boolean };"; + let result = parse_document(input).unwrap(); + let stmt = result.get("playdate = {}").unwrap(); + match stmt { + Statement::Global(name, _, fields) => { + assert_eq!(name, "playdate"); + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].name, "argv"); + assert_eq!(fields[0].typ, "string[]"); + } + _ => panic!("Expected Global"), + } } + #[test] - fn global_table() { - let document = LuarsParser::parse( - Rule::Global, - "global playdate.sound.twopolefilter: SoundEffect;", - ) - .expect("unsuccessful parse") - .next() - .unwrap(); - assert_eq!( - parse_tbl(document), - LuarsStatement::Global("playdate.sound.twopolefilter", "SoundEffect", Vec::new()) - ); + fn test_local() { + let result = parse_document("local File: playdate.file.file;").unwrap(); + let stmt = result.get("local File = {}").unwrap(); + match stmt { + Statement::Local(name, parent, _) => { + assert_eq!(name, "File"); + assert_eq!(parent, "playdate.file.file"); + } + _ => panic!("Expected Local"), + } } + #[test] - fn local_type() { - let document = LuarsParser::parse(Rule::Local, "local File: playdate.file.file;") - .expect("unsuccessful parse") - .next() - .unwrap(); - assert_eq!( - parse_tbl(document), - LuarsStatement::Local("File", "playdate.file.file", Vec::new()) - ); + fn test_simple_function() { + let result = parse_document("fun where(): nil;").unwrap(); + let stmt = result.get("where()").unwrap(); + match stmt { + Statement::Function(name, params, returns) => { + assert_eq!(name, "where"); + assert!(params.is_empty()); + assert_eq!(returns.len(), 1); + assert_eq!(returns[0].typ, "nil"); + } + _ => panic!("Expected Function"), + } } + #[test] - fn local_literal() { - let document = LuarsParser::parse( - Rule::Local, - "local Size: playdate.geometry.size = { width: number, height: number, };", - ) - .expect("unsuccessful parse") - .next() - .unwrap(); - assert_eq!( - parse_tbl(document), - LuarsStatement::Local( - "Size", - "playdate.geometry.size", - vec![("width", "number", ""), ("height", "number", ""),] - ) - ); + fn test_function_with_params() { + let input = "fun playdate.timer.new(duration: integer, callback: function, ...?: any): Timer;"; + let result = parse_document(input).unwrap(); + let stmt = result.get("playdate.timer.new(duration, callback, ...)").unwrap(); + match stmt { + Statement::Function(name, params, returns) => { + assert_eq!(name, "playdate.timer.new"); + assert_eq!(params.len(), 3); + assert_eq!(params[0].name, "duration"); + assert_eq!(params[0].typ, "integer"); + assert_eq!(params[2].name, "...?"); + assert_eq!(returns[0].typ, "Timer"); + } + _ => panic!("Expected Function"), + } } + #[test] - fn playdate_grammar() { - let unparsed_file = fs::read_to_string("playdate.luars").expect("cannot read file"); - let playdate_luars = parse_document(&unparsed_file); - assert_eq!(playdate_luars.len(), unparsed_file.matches(";").count()); + fn test_method_with_multi_return() { + let input = "fun GridView:getScrollPosition(): (x: integer, y: integer);"; + let result = parse_document(input).unwrap(); + let stmt = result.get("GridView:getScrollPosition()").unwrap(); + match stmt { + Statement::Function(name, params, returns) => { + assert_eq!(name, "GridView:getScrollPosition"); + assert!(params.is_empty()); + assert_eq!(returns.len(), 2); + assert_eq!(returns[0].name, "x"); + assert_eq!(returns[1].name, "y"); + } + _ => panic!("Expected Function"), + } } + #[test] - fn funcs() { - let document = LuarsParser::parse(Rule::Function, "fun where(): nil;") - .expect("unsuccessful parse") - .next() - .unwrap(); - assert_eq!( - parse_function(document), - LuarsStatement::Function("where", Vec::new(), vec![("", "nil"),]), - ) + fn test_union_type() { + let input = "fun playdate.buttonIsPressed(button: (integer|string)): boolean;"; + let result = parse_document(input).unwrap(); + let stmt = result.get("playdate.buttonIsPressed(button)").unwrap(); + match stmt { + Statement::Function(_, params, _) => { + assert_eq!(params[0].typ, "(integer|string)"); + } + _ => panic!("Expected Function"), + } } + #[test] - fn funcs2() { - let document = LuarsParser::parse( - Rule::Function, - "fun playdate.timer.new(duration: integer, callback: function, ...?: any): Timer;", - ) - .expect("unsuccessful parse") - .next() - .unwrap(); - assert_eq!( - parse_function(document), - LuarsStatement::Function( - "playdate.timer.new", - vec![ - ("duration", "integer"), - ("callback", "function"), - ("...?", "any"), - ], - vec![("", "Timer"),] - ), - ) + fn test_function_type_param() { + let input = "fun playdate.getServerTime(callback: fun(time?: string, error?: string));"; + let result = parse_document(input).unwrap(); + let stmt = result.get("playdate.getServerTime(callback)").unwrap(); + match stmt { + Statement::Function(_, params, _) => { + assert_eq!(params[0].typ, "fun(time?: string, error?: string)"); + } + _ => panic!("Expected Function"), + } } + #[test] - fn funcs3() { - let document = LuarsParser::parse( - Rule::Function, - "fun GridView:getScrollPosition(): (x: integer, y: integer);", - ) - .expect("bad parse") - .next() - .unwrap(); - assert_eq!( - parse_function(document), - LuarsStatement::Function( - "GridView:getScrollPosition", - vec![], - vec![("x", "integer"), ("y", "integer"),] - ), - ) + fn test_full_playdate_luars() { + let input = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/playdate.luars")); + let result = parse_document(input); + assert!(result.is_ok(), "Failed to parse: {:?}", result.err()); + let stmts = result.unwrap(); + let expected = input.matches(';').count(); + assert_eq!(stmts.len(), expected, "Expected {} statements", expected); } } diff --git a/src/main.rs b/src/main.rs index 8e15ff1..c8ed516 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,104 +1,38 @@ +//! Playdate DocDef - Generate LuaCATS type annotations for Playdate SDK +//! +//! This tool parses the .luars type definition file and optionally scrapes +//! the official Playdate SDK documentation to generate comprehensive +//! LuaCATS-compatible stub files for IDE autocompletion. + mod args; -mod finstub; -mod fixes; mod luars; -mod scrape; -mod stub; - -use args::{fetch_docs, setup}; -use luars::{LuarsStatement, parse_document}; -use stub::Stub; +mod output; +mod scraper; -use crate::{args::Action, finstub::FinStub}; -use std::collections::{BTreeMap, HashSet}; +use args::Action; +use output::StubOutput; static PLAYDATE_LUARS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/playdate.luars")); -fn go_out(fin_stubs: Vec) { - let header = [ - "---@meta", - "--- This file contains function stubs for autocompletion. DO NOT include it in your game.", - ]; - println!("{}\n", header.join("\n")); - - for stub in fin_stubs { - let output = stub.generate_stub(); - if !output.is_empty() { - println!("{}\n", output.join("\n")); - } - } - println!("--- End of LuaCATS stubs."); -} - -fn stubs_with_docs( - playdate_luars: BTreeMap>, - docs: String, -) -> Vec { - let mut variables = Vec::new(); - let mut functions = Vec::new(); - let scraped_stubs = scrape::scrape(docs, &playdate_luars); - - let mut processed_funs: HashSet = HashSet::new(); +fn main() { + let args = args::parse(); - // Process scraped stubs, separating into variables and functions - for stub in scraped_stubs.values() { - match stub { - Stub::Function(func_stub) => { - processed_funs.insert(func_stub.lua_def()); - functions.push(FinStub::from_stub(&stub)); - } - } - } + // Parse the .luars type definitions + let statements = + luars::parse_document(PLAYDATE_LUARS).expect("Failed to parse playdate.luars"); - // Finally collect remaining functions from luars that weren't in scraped docs - for s in playdate_luars.values() { - match s { - LuarsStatement::Global(_name, _parent, _attrs) - | LuarsStatement::Local(_name, _parent, _attrs) => { - // eprintln!("{_name}:{_parent}"); - variables.push(FinStub::from_luars(s)); - } - LuarsStatement::Function(_, _, _) => { - if !processed_funs.contains(s.lua_def().as_str()) { - functions.push(FinStub::from_luars(s)); - } - } + match args.action { + Action::Stub => { + // Generate stubs without documentation + let output = StubOutput::from_statements(&statements); + output.print(); } - } - - // Variables have to come first because the types are used for function params/returns - let mut fin_stubs = Vec::new(); - fin_stubs.extend(variables); - fin_stubs.extend(functions); - fin_stubs -} - -/// Outputs just the stubs as defined in the .luars source -fn stubs_without_docs(statements: BTreeMap>) -> Vec { - let mut variables = Vec::new(); - let mut functions = Vec::new(); - for statement in statements.values() { - match statement { - LuarsStatement::Local(_, _, _) | LuarsStatement::Global(_, _, _) => { - variables.push(FinStub::from_luars(statement)); - } - LuarsStatement::Function(_, _, _) => { - functions.push(FinStub::from_luars(statement)); - } + Action::Annotate => { + // Scrape documentation and generate annotated stubs + let html = args::fetch_docs(&args); + let scraped = scraper::scrape(&html, &statements); + let output = StubOutput::from_statements_with_docs(&statements, &scraped); + output.print(); } } - let mut fin_stubs = Vec::new(); - fin_stubs.extend(variables); - fin_stubs.extend(functions); - fin_stubs -} - -fn main() { - let args = setup(); - let playdate_luars = parse_document(&PLAYDATE_LUARS); - let fin_stubs = match args.action { - Action::Annotate => stubs_with_docs(playdate_luars, fetch_docs(args)), - Action::Stub => stubs_without_docs(playdate_luars), - }; - go_out(fin_stubs); } diff --git a/src/output.rs b/src/output.rs new file mode 100644 index 0000000..0ac0ced --- /dev/null +++ b/src/output.rs @@ -0,0 +1,312 @@ +//! LuaCATS output generation +//! +//! This module generates LuaCATS-format stub files from parsed .luars +//! statements and optionally scraped documentation. + +use std::collections::{BTreeMap, HashSet}; +use textwrap; + +use crate::luars::{Field, Param, Statement}; +use crate::scraper::ScrapedFunction; + +/// Maximum line length for documentation text (excluding "--- " prefix) +const MAX_LINE_LENGTH: usize = 96; + +/// Notes/deprecations loaded from TOML +static NOTES: std::sync::LazyLock>> = std::sync::LazyLock::new(|| { + let toml_str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/Notes.toml")); + toml::from_str(toml_str).unwrap_or_default() +}); + +/// Generate LuaCATS output for a class/table +pub fn generate_class(name: &str, parent: &str, fields: &[Field], prefix: &str) -> Vec { + let mut out = Vec::new(); + + // Class annotation + if parent.is_empty() { + out.push(format!("---@class {}", name)); + } else { + out.push(format!("---@class {} : {}", name, parent)); + } + + // Field annotations + for field in fields { + if field.value.is_empty() { + out.push(format!("---@field {} {}", field.name, field.typ)); + } else { + out.push(format!( + "---@field {} {} {}", + field.name, field.typ, field.value + )); + } + } + + // Lua assignment + out.push(format!("{}{} = {{}}", prefix, name)); + + out +} + +/// Generate LuaCATS output for a function +pub fn generate_function( + name: &str, + params: &[Param], + returns: &[Param], + docs: Option<&ScrapedFunction>, +) -> Vec { + let mut out = Vec::new(); + + // Apply any notes (deprecations, etc.) + let param_names: Vec<&str> = params + .iter() + .map(|p| p.name.trim_end_matches('?')) + .collect(); + let lua_def = format!("{}({})", name, param_names.join(", ")); + if let Some(notes) = NOTES.get(&lua_def) { + out.extend(notes.clone()); + } + + // Documentation from scraped HTML + if let Some(func) = docs { + out.extend(generate_docs(&func.docs, &func.anchor, name)); + } + + // Parameter annotations + for param in params { + out.push(format!("---@param {} {}", param.name, param.typ)); + } + + // Return annotations + for ret in returns { + if ret.name.is_empty() { + out.push(format!("---@return {}", ret.typ)); + } else { + out.push(format!("---@return {} {}", ret.typ, ret.name)); + } + } + + // Function stub + out.push(format!("function {} end", lua_def)); + + out +} + +/// Check if a line is a list item (including nested/indented lists) +fn is_list_item(line: &str) -> bool { + let trimmed = line.trim_start(); + trimmed.starts_with("* ") +} + +/// Generate documentation comment lines +fn generate_docs(docs: &[String], anchor: &str, title: &str) -> Vec { + if anchor.is_empty() { + return Vec::new(); + } + + let mut out = Vec::new(); + let mut in_code = false; + + for (i, line) in docs.iter().enumerate() { + // Code blocks and bullet lists get fewer line breaks + let this_is_list = is_list_item(line); + let next_is_list = docs.get(i + 1).map_or(false, |l| is_list_item(l)); + let no_break = in_code || line.starts_with("```") || (this_is_list && next_is_list); + + if no_break { + out.push(format!("--- {}", line)); + } else { + // Wrap long lines + for wrapped in textwrap::wrap(line, MAX_LINE_LENGTH) { + out.push(format!("--- {}", wrapped)); + } + out.push("---".to_string()); + } + + // Track code block state + if line.starts_with("```") { + in_code = !in_code; + } + } + + // Link to official docs + out.push(format!( + "--- [Inside Playdate: {}](https://sdk.play.date/Inside%20Playdate.html#{})", + title, anchor + )); + + out +} + +/// Full stub generator output +pub struct StubOutput { + lines: Vec>, +} + +impl StubOutput { + /// Create stub output from parsed statements only (no docs) + pub fn from_statements(statements: &BTreeMap) -> Self { + let mut classes = Vec::new(); + let mut functions = Vec::new(); + + for stmt in statements.values() { + match stmt { + Statement::Global(name, parent, fields) => { + classes.push(generate_class(name, parent, fields, "")); + } + Statement::Local(name, parent, fields) => { + classes.push(generate_class(name, parent, fields, "local ")); + } + Statement::Function(name, params, returns) => { + functions.push(generate_function(name, params, returns, None)); + } + } + } + + // Classes must come before functions + let mut lines = Vec::new(); + lines.extend(classes); + lines.extend(functions); + + StubOutput { lines } + } + + /// Create stub output from statements with scraped documentation + pub fn from_statements_with_docs( + statements: &BTreeMap, + scraped: &BTreeMap, + ) -> Self { + let mut classes = Vec::new(); + let mut functions = Vec::new(); + let mut processed: HashSet = HashSet::new(); + + // First, output all classes/tables from statements + for stmt in statements.values() { + match stmt { + Statement::Global(name, parent, fields) => { + classes.push(generate_class(name, parent, fields, "")); + } + Statement::Local(name, parent, fields) => { + classes.push(generate_class(name, parent, fields, "local ")); + } + _ => {} + } + } + + // Process scraped functions (they have docs) + for func in scraped.values() { + let key = func.lua_def(); + processed.insert(key.clone()); + + // Get types from statements if available + let (params, returns) = if let Some(Statement::Function(_, p, r)) = statements.get(&key) + { + (p.as_slice(), r.as_slice()) + } else { + (func.params.as_slice(), func.returns.as_slice()) + }; + + functions.push(generate_function(&func.name, params, returns, Some(func))); + } + + // Add remaining functions from statements (those not in scraped docs) + for stmt in statements.values() { + if let Statement::Function(name, params, returns) = stmt { + let key = stmt.lua_def(); + if !processed.contains(&key) { + functions.push(generate_function(name, params, returns, None)); + } + } + } + + let mut lines = Vec::new(); + lines.extend(classes); + lines.extend(functions); + + StubOutput { lines } + } + + /// Output to stdout + pub fn print(&self) { + println!("---@meta"); + println!( + "--- This file contains function stubs for autocompletion. DO NOT include it in your game." + ); + println!(); + + for block in &self.lines { + if !block.is_empty() { + println!("{}", block.join("\n")); + println!(); + } + } + + println!("--- End of LuaCATS stubs."); + } + + /// Get output as a single string + #[allow(dead_code)] + pub fn to_string(&self) -> String { + let mut result = String::new(); + result.push_str("---@meta\n"); + result.push_str("--- This file contains function stubs for autocompletion. DO NOT include it in your game.\n"); + result.push('\n'); + + for block in &self.lines { + if !block.is_empty() { + result.push_str(&block.join("\n")); + result.push_str("\n\n"); + } + } + + result.push_str("--- End of LuaCATS stubs.\n"); + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_class() { + let fields = vec![ + Field { + name: "foo".into(), + typ: "string".into(), + value: "".into(), + }, + Field { + name: "bar".into(), + typ: "integer".into(), + value: "42".into(), + }, + ]; + let result = generate_class("MyClass", "Parent", &fields, ""); + assert!(result.contains(&"---@class MyClass : Parent".to_string())); + assert!(result.contains(&"---@field foo string".to_string())); + assert!(result.contains(&"---@field bar integer 42".to_string())); + } + + #[test] + fn test_generate_function() { + let params = vec![ + Param { + name: "a".into(), + typ: "string".into(), + }, + Param { + name: "b?".into(), + typ: "integer".into(), + }, + ]; + let returns = vec![Param { + name: "".into(), + typ: "boolean".into(), + }]; + let result = generate_function("test.func", ¶ms, &returns, None); + assert!(result.contains(&"---@param a string".to_string())); + assert!(result.contains(&"---@param b? integer".to_string())); + assert!(result.contains(&"---@return boolean".to_string())); + assert!(result.contains(&"function test.func(a, b) end".to_string())); + } +} diff --git a/src/scrape.rs b/src/scrape.rs deleted file mode 100644 index 32ffcdc..0000000 --- a/src/scrape.rs +++ /dev/null @@ -1,179 +0,0 @@ -use std::collections::BTreeMap; - -use regex::Regex; -use scraper::{CaseSensitivity, Selector}; - -use crate::fixes::{apply_fn_types, clean_code, clean_text}; -use crate::luars::LuarsStatement; -use crate::stub::Stub; - -enum AnchorType { - Function, - Method, - Table, - Callback, - Variable, - Attribute, - Unkown, -} - -impl AnchorType { - pub fn new(anchor: &str) -> Self { - match anchor { - "lua-sample" => AnchorType::Function, - "f-metadata" => AnchorType::Variable, - _ => match anchor.get(0..2).unwrap_or("") { - "v-" => AnchorType::Variable, - "f-" => AnchorType::Function, - "m-" => AnchorType::Method, - "t-" => AnchorType::Table, - "c-" => AnchorType::Callback, - "a-" => AnchorType::Attribute, - _ => AnchorType::Unkown, - }, - } - } -} - -pub fn scrape( - response: String, - statements: &BTreeMap>, -) -> BTreeMap { - let document = scraper::Html::parse_document(&response); - let outer = Selector::parse(concat!( - "div.sect1>div.sectionbody>div.sect2>div.item", - ",div.sect1>div.sectionbody>div.sect2>div.sect3>div.item", - ",div.sect1>div.sectionbody>div.sect2>div.sect3>div.sect4>div.item", - ",div.sect1>div.sectionbody>div.sect2>div.sect3>div.sect4>div.sect5>div.item", - )) - .unwrap(); - let sel_title = Selector::parse("div.title").unwrap(); - let sel_content = Selector::parse("div.content").unwrap(); - - let sel_docs = Selector::parse( - vec![ - "div.paragraph", - "div.ulist", - "div.admonitionblock", - "div.literalblock", - "div.listingblock", - ] - .join(",") - .as_str(), - ) - .unwrap(); - let sel_docs_text = Selector::parse("p").unwrap(); - let sel_docs_list = Selector::parse("ul>li").unwrap(); - let sel_docs_coderay = Selector::parse("code").unwrap(); - let sel_docs_admonition = Selector::parse("table>tbody>tr>td.content").unwrap(); - - let re_function = Regex::new(r"^(?:[\w][\w\d]*\.)*(?:[\w][\w\d]*)(?:[:.][\w][\w\d]*)").unwrap(); - - let mut _poop = 0; - let mut _last_class: String = "".to_string(); - let mut stubs: BTreeMap = BTreeMap::new(); - - for element in document.select(&outer) { - _poop = _poop + 1; - - let anchor: &str = element.value().attr("id").unwrap_or(""); - let title: String = element - .select(&sel_title) - .next() - .unwrap() - .text() - .collect::(); - if anchor == "" { - // eprintln!("WARN: Docs missing anchor for: {}", title); - } - - let mut text: Vec = Vec::new(); - for c in element.select(&sel_content) { - // Main Paragraphs of text documentation - for div in c.select(&sel_docs) { - let dv = div.value(); - if dv.has_class("paragraph", CaseSensitivity::CaseSensitive) { - div.select(&sel_docs_text).for_each(|p| { - let t = clean_text(p.inner_html()); - text.push(t); - }); - } else if dv.has_class("listingblock", CaseSensitivity::CaseSensitive) { - text.push("```".to_string()); - div.select(&sel_docs_coderay).for_each(|d| { - let lines = clean_code(d.text().collect::()); - for l in lines { - text.push(l); - } - }); - text.push("```".to_string()); - } else if dv.has_class("ulist", CaseSensitivity::CaseSensitive) { - div.select(&sel_docs_list).for_each(|li| { - let mut t: String = clean_text(li.text().collect::()); - t.insert_str(0, "* "); - text.push(t); - }); - } else if dv.has_class("admonitionblock", CaseSensitivity::CaseSensitive) { - div.select(&sel_docs_admonition).for_each(|td| { - let adm = clean_text(td.inner_html()); - text.push(adm); - }); - } else if dv.has_class("literalblock", CaseSensitivity::CaseSensitive) { - div.select(&sel_content).for_each(|d| { - // TODO: This clobbers line breaks. pre should get ``` fencing. - let litb = clean_text(d.inner_html()); - text.push(litb); - }); - } else { - eprintln!("skipping {:?}", div.value()) - } - } - } - - match AnchorType::new(anchor) { - AnchorType::Attribute => { - // eprintln!("ATTRIBUTE {} {} {:?} ", anchor, title, text); - } - AnchorType::Variable => { - if title.contains(" ") { - // eprintln!("MULTILINE_VARIABLE {} {} {:?} ", anchor, title, text); - } - // eprintln!("VARIABLE {} {} {:?} ", anchor, title, text); - } - AnchorType::Table - | AnchorType::Method - | AnchorType::Callback - | AnchorType::Function => { - // Functions with multiple (e.g. playdate.easingFunctions.*, ) - if title.contains(" ") { - if !(anchor.starts_with("m-") || anchor.starts_with("f-")) {} - for t in title.split(" ") { - let mut stub = - apply_fn_types(&anchor.to_string(), &t.trim().to_string(), &text); - stub = stub.annotate(statements); - let key = stub.lua_def(); - if let Some(_val) = stubs.insert(key.clone(), Stub::Function(stub)) { - eprintln!("Duplicate stub {} (in multi-def)", key) - } - } - // Other functions - } else { - let stub = - apply_fn_types(&anchor.to_string(), &title, &text).annotate(statements); - let lua_def = stub.lua_def(); - if let Some(_val) = stubs.insert(lua_def.clone(), Stub::Function(stub)) { - eprintln!("Duplicate stub {} (function)", lua_def) - } - } - } - _ => { - eprintln!("UNKNOWN: {anchor} {title}"); - } - } - - // _last_class is context for the next loop. So if the title is missing a name (e.g. "p + p") we can infer it. - if re_function.is_match(&title) && title.contains(":") { - _last_class = title.split(":").next().unwrap_or("").to_string(); - } - } - stubs -} diff --git a/src/scraper.rs b/src/scraper.rs new file mode 100644 index 0000000..d28cf2c --- /dev/null +++ b/src/scraper.rs @@ -0,0 +1,451 @@ +//! HTML documentation scraper for Playdate SDK +//! +//! This module extracts API documentation from the official Playdate SDK +//! HTML documentation and converts it to structured data. + +use regex::Regex; +use scraper::{ElementRef, Html, Selector}; +use std::collections::BTreeMap; +use std::sync::LazyLock; + +use crate::luars::{Param, Statement}; + +/// A scraped function stub with documentation +#[derive(Debug, Clone)] +pub struct ScrapedFunction { + pub name: String, + pub anchor: String, + pub params: Vec, + pub returns: Vec, + pub docs: Vec, +} + +impl ScrapedFunction { + /// Generate the lua function definition string (used as map key) + pub fn lua_def(&self) -> String { + let param_names: Vec<&str> = self + .params + .iter() + .map(|p| p.name.trim_end_matches('?')) + .collect(); + format!("{}({})", self.name, param_names.join(", ")) + } + + /// Apply type information from parsed .luars statements + pub fn apply_types(&mut self, statements: &BTreeMap) { + let key = self.lua_def(); + if let Some(Statement::Function(_, params, returns)) = statements.get(&key) { + self.params = params.clone(); + self.returns = returns.clone(); + } + } +} + +/// Type of documentation item based on anchor prefix +#[derive(Debug, Clone, Copy, PartialEq)] +enum ItemType { + Function, // f-* + Method, // m-* + Callback, // c-* + Table, // t-* + Variable, // v-* + Attribute, // a-* + Unknown, +} + +impl ItemType { + fn from_anchor(anchor: &str) -> Self { + match anchor.get(..2) { + Some("f-") => ItemType::Function, + Some("m-") => ItemType::Method, + Some("c-") => ItemType::Callback, + Some("t-") => ItemType::Table, + Some("v-") => ItemType::Variable, + Some("a-") => ItemType::Attribute, + _ => ItemType::Unknown, + } + } + + fn from_class(class: &str) -> Self { + if class.contains("function") { + ItemType::Function + } else if class.contains("method") { + ItemType::Method + } else if class.contains("callback") { + ItemType::Callback + } else if class.contains("table") { + ItemType::Table + } else if class.contains("variable") { + ItemType::Variable + } else if class.contains("attribute") { + ItemType::Attribute + } else { + ItemType::Unknown + } + } + + fn is_function_like(&self) -> bool { + matches!( + self, + ItemType::Function | ItemType::Method | ItemType::Callback | ItemType::Table + ) + } +} + +// Lazy static selectors and regexes +static SEL_ITEM: LazyLock = LazyLock::new(|| { + Selector::parse(concat!( + "div.sect1>div.sectionbody>div.sect2>div.item", + ",div.sect1>div.sectionbody>div.sect2>div.sect3>div.item", + ",div.sect1>div.sectionbody>div.sect2>div.sect3>div.sect4>div.item", + ",div.sect1>div.sectionbody>div.sect2>div.sect3>div.sect4>div.sect5>div.item", + )) + .unwrap() +}); + +static SEL_TITLE: LazyLock = LazyLock::new(|| Selector::parse("div.title").unwrap()); +static SEL_CONTENT: LazyLock = LazyLock::new(|| Selector::parse("div.content").unwrap()); +static SEL_P_TAG: LazyLock = LazyLock::new(|| Selector::parse("p").unwrap()); +static SEL_CODE_TAG: LazyLock = LazyLock::new(|| Selector::parse("code").unwrap()); +static SEL_PRE_TAG: LazyLock = LazyLock::new(|| Selector::parse("pre").unwrap()); +static SEL_ADMONITION: LazyLock = + LazyLock::new(|| Selector::parse("table>tbody>tr>td.content").unwrap()); + +static RE_FUNC_SIG: LazyLock = LazyLock::new(|| { + Regex::new(r"^((?:[\w_][\w\d_]*\.)*[\w_][\w\d_]*[:.:][\w_][\w\d_]*|[\w_][\w\d_]*(?:\.[\w_][\w\d_]*)*)\s*\(([^)]*)\)").unwrap() +}); +static RE_HTML_LINK: LazyLock = LazyLock::new(|| Regex::new(r"]*>").unwrap()); +static RE_EM_TAG: LazyLock = LazyLock::new(|| Regex::new(r"]*>").unwrap()); + +/// Scrape the Playdate SDK documentation HTML +pub fn scrape( + html: &str, + statements: &BTreeMap, +) -> BTreeMap { + let document = Html::parse_document(html); + let mut result = BTreeMap::new(); + let overrides = load_overrides(); + let invalid_params = load_invalid_params(); + + for element in document.select(&SEL_ITEM) { + let anchor = element.value().attr("id").unwrap_or(""); + let class = element.value().attr("class").unwrap_or(""); + + // Try anchor prefix first, fall back to class attribute + let item_type = match ItemType::from_anchor(anchor) { + ItemType::Unknown => ItemType::from_class(class), + t => t, + }; + + if !item_type.is_function_like() { + continue; + } + + let title = extract_title(&element); + let docs = extract_docs(&element); + + // Handle multi-function definitions (separated by double spaces) + let titles: Vec<&str> = if title.contains(" ") { + title.split(" ").map(|s| s.trim()).collect() + } else { + vec![title.as_str()] + }; + + for title in titles { + if let Some(mut func) = parse_function_title(anchor, title, &overrides, &invalid_params) + { + func.docs = docs.clone(); + func.apply_types(statements); + let key = func.lua_def(); + result.insert(key, func); + } + } + } + + result +} + +/// Extract the title text from an item element +fn extract_title(element: &ElementRef) -> String { + element + .select(&SEL_TITLE) + .next() + .map(|e| e.text().collect::()) + .unwrap_or_default() +} + +/// Recursively extract list items with proper indentation for nested lists +fn extract_list_items(list_el: &ElementRef, docs: &mut Vec, depth: usize) { + let indent = " ".repeat(depth); + + // Find the direct ul child, then iterate its direct li children + for child in list_el.children() { + if let Some(ul) = ElementRef::wrap(child) { + if ul.value().name() == "ul" { + for li_node in ul.children() { + if let Some(li) = ElementRef::wrap(li_node) { + if li.value().name() != "li" { + continue; + } + + // Get direct text content from

tag if present + let text = if let Some(p) = li + .children() + .filter_map(ElementRef::wrap) + .find(|el| el.value().name() == "p") + { + clean_html_text(&p.inner_html()) + } else { + // Get only direct text nodes + li.children() + .filter_map(|c| c.value().as_text()) + .map(|t| t.text.as_ref()) + .collect::() + .trim() + .to_string() + }; + + if !text.is_empty() { + docs.push(format!("{}* {}", indent, text)); + } + + // Check for nested ulist inside this li + for li_child in li.children() { + if let Some(nested) = ElementRef::wrap(li_child) { + let classes = nested.value().attr("class").unwrap_or(""); + if classes.contains("ulist") { + extract_list_items(&nested, docs, depth + 1); + } + } + } + } + } + } + } + } +} + +/// Extract documentation text from an item element, preserving document order +fn extract_docs(element: &ElementRef) -> Vec { + let mut docs = Vec::new(); + + for content in element.select(&SEL_CONTENT) { + for child in content.children() { + if let Some(child_el) = ElementRef::wrap(child) { + let classes = child_el.value().attr("class").unwrap_or(""); + if classes.contains("paragraph") { + if let Some(p) = child_el.select(&SEL_P_TAG).next() { + docs.push(clean_html_text(&p.inner_html())); + } + } else if classes.contains("ulist") { + extract_list_items(&child_el, &mut docs, 0); + } else if classes.contains("listingblock") { + // Code block (with tag) + for code in child_el.select(&SEL_CODE_TAG) { + docs.push("```".to_string()); + for line in code.text().collect::().lines() { + if !line.trim().is_empty() { + docs.push(line.to_string()); + } + } + docs.push("```".to_string()); + } + } else if classes.contains("literalblock") { + // Literal block (with 

 tag)
+                    for pre in child_el.select(&SEL_PRE_TAG) {
+                        docs.push("```".to_string());
+                        for line in pre.text().collect::().lines() {
+                            if !line.trim().is_empty() {
+                                docs.push(line.to_string());
+                            }
+                        }
+                        docs.push("```".to_string());
+                    }
+                } else if classes.contains("admonitionblock") {
+                    // Admonition blocks (caution, note, warning, etc.)
+                    let prefix = "";
+                    // let prefix = if classes.contains("caution") { "CAUTION: " } else { "" };
+                    for adm in child_el.select(&SEL_ADMONITION) {
+                        docs.push(format!("{}{}", prefix, clean_html_text(&adm.inner_html())));
+                    }
+                }
+            }
+        }
+    }
+
+    docs
+}
+
+/// Clean HTML text by converting tags to markdown
+fn clean_html_text(text: &str) -> String {
+    // Handle em tags with attributes (like )
+    let result = RE_EM_TAG.replace_all(text, "*");
+    let result = result
+        .replace("", "*")
+        .replace("", "`")
+        .replace("", "`")
+        .replace("
", "`")
+        .replace("
", "`")
+        .replace("", "**")
+        .replace("", "**")
+        .replace("\n", " ")
+        .replace("
", "\n---\n---")
+        .replace(">", ">")
+        .replace("<", "<")
+        .replace("", "{bundleid}")
+        .replace("", "{text}")
+        .replace("", "{message}");
+
+    // Remove HTML links
+    let result = RE_HTML_LINK.replace_all(&result, "");
+    result.trim().to_string()
+}
+
+/// Parse a function title into a ScrapedFunction
+fn parse_function_title(
+    anchor: &str,
+    title: &str,
+    overrides: &BTreeMap,
+    invalid_params: &BTreeMap,
+) -> Option {
+    // Check for overrides first
+    if let Some(override_) = overrides.get(anchor) {
+        let params: Vec = override_
+            .parameters
+            .iter()
+            .map(|p| Param {
+                name: p.clone(),
+                typ: "any".to_string(),
+            })
+            .collect();
+        return Some(ScrapedFunction {
+            name: override_.name.clone(),
+            anchor: anchor.to_string(),
+            params,
+            returns: Vec::new(),
+            docs: Vec::new(),
+        });
+    }
+
+    // Parse the function signature
+    let caps = RE_FUNC_SIG.captures(title)?;
+    let name = caps.get(1)?.as_str().to_string();
+    let params_str = caps.get(2)?.as_str();
+
+    let mut params = Vec::new();
+    let mut is_optional = false;
+
+    if !params_str.trim().is_empty() {
+        for p in params_str.split(',') {
+            let mut param_name = p.trim().to_string();
+
+            // Track optional params (denoted by [])
+            if param_name.contains('[') {
+                is_optional = true;
+            }
+
+            // Clean brackets
+            param_name = param_name.replace(['[', ']'], "").trim().to_string();
+
+            // Mark as optional
+            if is_optional && !param_name.ends_with('?') {
+                param_name.push('?');
+            }
+
+            // Apply invalid param fixes
+            let clean_name = param_name.trim_end_matches('?');
+            if let Some(fixed) = invalid_params.get(clean_name) {
+                param_name = if param_name.ends_with('?') {
+                    format!("{}?", fixed)
+                } else {
+                    fixed.clone()
+                };
+            }
+
+            params.push(Param {
+                name: param_name.clone(),
+                typ: "any".to_string(),
+            });
+
+            // Stop after variadic
+            if param_name.starts_with("...") {
+                break;
+            }
+        }
+    }
+
+    Some(ScrapedFunction {
+        name,
+        anchor: anchor.to_string(),
+        params,
+        returns: Vec::new(),
+        docs: Vec::new(),
+    })
+}
+
+/// Function override from TOML config
+#[derive(Debug, Clone, serde::Deserialize)]
+struct FunctionOverride {
+    name: String,
+    parameters: Vec,
+}
+
+/// Load function overrides from TOML
+fn load_overrides() -> BTreeMap {
+    static TOML: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/RenameFn.toml"));
+    toml::from_str(TOML).unwrap_or_default()
+}
+
+/// Load invalid parameter name fixes from TOML
+fn load_invalid_params() -> BTreeMap {
+    static TOML: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/Invalid.toml"));
+    toml::from_str(TOML).unwrap_or_default()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_clean_html_text() {
+        assert_eq!(
+            clean_html_text("foo and bar"),
+            "`foo` and *bar*"
+        );
+    }
+
+    #[test]
+    fn test_parse_function_title() {
+        let overrides = BTreeMap::new();
+        let invalid = BTreeMap::new();
+
+        let func =
+            parse_function_title("f-test", "playdate.foo(a, b, c)", &overrides, &invalid).unwrap();
+        assert_eq!(func.name, "playdate.foo");
+        assert_eq!(func.params.len(), 3);
+        assert_eq!(func.params[0].name, "a");
+    }
+
+    #[test]
+    fn test_parse_function_with_optional_params() {
+        let overrides = BTreeMap::new();
+        let invalid = BTreeMap::new();
+
+        let func = parse_function_title("f-test", "foo(a, [b, c])", &overrides, &invalid).unwrap();
+        assert_eq!(func.params.len(), 3);
+        assert_eq!(func.params[0].name, "a");
+        assert_eq!(func.params[1].name, "b?");
+        assert_eq!(func.params[2].name, "c?");
+    }
+
+    #[test]
+    fn test_parse_method() {
+        let overrides = BTreeMap::new();
+        let invalid = BTreeMap::new();
+
+        let func =
+            parse_function_title("m-test", "Sprite:draw(x, y)", &overrides, &invalid).unwrap();
+        assert_eq!(func.name, "Sprite:draw");
+        assert_eq!(func.params.len(), 2);
+    }
+}
diff --git a/src/stub.rs b/src/stub.rs
deleted file mode 100644
index 5a24b25..0000000
--- a/src/stub.rs
+++ /dev/null
@@ -1,215 +0,0 @@
-use crate::fixes::apply_notes;
-use crate::luars::LuarsStatement;
-use std::collections::BTreeMap;
-use textwrap;
-
-// Wrap long lines of documentation at this length
-// Note: Function signatures are not wrapped (a couple are >100 chars)
-// Note: This also includes the leading "--- " (4 chars).
-static MAX_LINE_LENGTH: usize = 100 - 4;
-
-pub enum Stub {
-    Function(StubFn),
-}
-
-// Stub Struct containing extracted signature, url anchor, list of parameters and description text
-#[derive(Debug, Clone)]
-pub struct StubFn {
-    pub title: String,
-    pub anchor: String,
-    /// Function Parameters (name, type)
-    pub params: Vec<(String, String)>,
-    /// Return (name, type)
-    pub returns: Vec<(String, String)>,
-    pub text: Vec,
-}
-
-impl StubFn {
-    pub fn annotate(mut self, statements: &BTreeMap) -> Self {
-        let our_lua = self.lua_def();
-        if let Some(statement) = statements.get(&our_lua) {
-            match statement {
-                LuarsStatement::Function(_name, params, returns) => {
-                    if our_lua == statement.lua_def() {
-                        self.params = params
-                            .iter()
-                            .map(|(fname, ftype)| (fname.to_string(), ftype.to_string()))
-                            .collect();
-                        self.returns = returns
-                            .iter()
-                            .map(|(fname, ftype)| (fname.to_string(), ftype.to_string()))
-                            .collect();
-                    }
-                }
-                _ => eprintln!("eek, found non-function for {our_lua}"),
-            }
-        } else {
-            eprintln!("WARN: No Luars types for {our_lua} #{}", self.anchor);
-        }
-        self
-    }
-
-    /// Lua function signature (no types)
-    pub fn lua_def(&self) -> String {
-        let name = self.title.clone();
-        let params = self.params.clone();
-        let param_names: Vec = params
-            .iter()
-            .map(|(name, _)| name.clone().replace("?", ""))
-            .collect::>();
-        format!("{}({})", name, param_names.join(", "))
-    }
-
-    fn generate_description(&self) -> Vec {
-        if self.anchor == "" {
-            Vec::new()
-        } else {
-            let mut lines = Vec::new();
-            let mut i = 0;
-            let mut in_code = false;
-            while i < self.text.len() {
-                let line = &self.text[i];
-                // Bulleted list and code get fewer newlines.
-                // Everything else needs extra empty lines for proper markdown rendering.
-                let no_break = in_code
-                    || line.starts_with("```")
-                    || (line.starts_with("* ")
-                        && i < self.text.len() - 1
-                        && self.text[i + 1].starts_with("* "));
-                if no_break {
-                    lines.push(format!("--- {}", line));
-                } else {
-                    for wrapped_line in textwrap::wrap(line.as_str(), MAX_LINE_LENGTH) {
-                        lines.push(format!("--- {}", wrapped_line));
-                    }
-                    lines.push("---".to_string());
-                }
-                // this is hacky as hell
-                if line == "```" {
-                    in_code = !in_code;
-                }
-                i = i + 1;
-            }
-            lines.push(format!(
-                "--- [Inside Playdate: {}](https://sdk.play.date/Inside%20Playdate.html#{})",
-                self.title, self.anchor
-            ));
-            lines
-        }
-    }
-
-    /// Generate complete LuaCATS for function
-    pub fn get_luacats(&self) -> Vec {
-        let mut out = Vec::new();
-        out.extend(apply_notes(&self.lua_def()));
-        out.extend(self.generate_description());
-        out.extend(self.luacats_params());
-        out.extend(self.luacats_returns());
-        out.push(self.lua_statement());
-        out
-    }
-
-    /// Generatte '---@param name type' for each parameter to the function
-    fn luacats_params(&self) -> Vec {
-        self.params
-            .iter()
-            .map(|(name, _type)| format!("---@param {} {}", name, _type))
-            .collect::>()
-    }
-
-    /// Generate '---@return type [name]' for the function (multiple lines for multival returns)
-    fn luacats_returns(&self) -> Vec {
-        self.returns
-            .iter()
-            .map(|(_name, _type)| {
-                if _name.to_string() == "" {
-                    format!("---@return {}", _type)
-                } else {
-                    format!("---@return {_type} {_name}", _type = _type, _name = _name)
-                }
-            })
-            .collect::>()
-    }
-
-    /// Return a valid lua statement for the function.
-    fn lua_statement(&self) -> String {
-        format!("function {} end", self.lua_def())
-    }
-}
-
-pub struct TableContents {
-    pub name: String,
-    pub r#type: String,
-    pub value: String, // Some Enums have documented values
-}
-
-impl TableContents {
-    pub fn to_string(&self) -> String {
-        let mut line = Vec::new();
-        line.push("---@field");
-        line.push(&self.name);
-        line.push(&self.r#type);
-        if !self.value.is_empty() {
-            line.push(&self.value);
-        }
-        line.join(" ")
-    }
-}
-
-/// Global and Local Variables
-pub struct Table {
-    pub prefix: String,
-    pub name: String,
-    pub r#type: String,
-    pub contents: Vec,
-}
-
-impl Table {
-    pub fn get_luacats(&self) -> Vec {
-        let mut out = Vec::new();
-        out.push(self.luacats_class());
-        out.extend(self.luacats_fields());
-        out.push(self.lua_statement());
-        out
-    }
-
-    /// Return a valid lua statement for the class.
-    pub fn lua_statement(&self) -> String {
-        format!("{}{} = {{}}", self.prefix, self.name)
-    }
-    /// Returns '---@field name type [value]' for class/instance attributes
-    pub fn luacats_fields(&self) -> Vec {
-        self.contents
-            .iter()
-            .map(|a| a.to_string())
-            .collect::>()
-    }
-
-    /// Returns '---@class name : parent' for classes
-    pub fn luacats_class(&self) -> String {
-        if self.r#type.to_string() == "" {
-            format!("---@class {}", self.name)
-        } else {
-            format!("---@class {} : {}", self.name, self.r#type)
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_basic_stub() {
-        let stub = StubFn {
-            title: "test_func".to_string(),
-            anchor: "test_anchor".to_string(),
-            params: vec![("param1".to_string(), "string".to_string())],
-            returns: vec![("ret1".to_string(), "number".to_string())],
-            text: vec!["Test description".to_string()],
-        };
-
-        assert_eq!(stub.lua_def(), "test_func(param1)");
-        assert_eq!(stub.lua_statement(), "function test_func(param1) end");
-    }
-}