diff --git a/.rubocop.yml b/.rubocop.yml index 4006899..0761b4a 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,3 +1,6 @@ +AllCops: + NewCops: enable + TargetRubyVersion: 2.4 Layout/EndAlignment: Exclude: - 'lib/latexmath/converter.rb' diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..b0f6bf0 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +2.4.10 diff --git a/lib/latexmath.rb b/lib/latexmath.rb index 206c431..03c11e7 100644 --- a/lib/latexmath.rb +++ b/lib/latexmath.rb @@ -1,10 +1,20 @@ -#require 'byebug' unless RUBY_ENGINE == 'opal' +require 'byebug' unless RUBY_ENGINE == 'opal' require 'json' require 'htmlentities' require 'ox' require_relative 'latexmath/ext' require_relative 'latexmath/version' +require_relative 'latexmath/common/object' +require_relative 'latexmath/common/number' +require_relative 'latexmath/core/definition/expandable' +require_relative 'latexmath/core/mouth' +require_relative 'latexmath/core/package' +require_relative 'latexmath/core/state' +require_relative 'latexmath/core/token' +require_relative 'latexmath/core/tokens' require_relative 'latexmath/constants/symbols' +require_relative 'latexmath/packages/aas' +require_relative 'latexmath/packages/amsmath' require_relative 'latexmath/aggregator' require_relative 'latexmath/converter' require_relative 'latexmath/symbol' diff --git a/lib/latexmath/aggregator.rb b/lib/latexmath/aggregator.rb index 836184a..c1caea8 100644 --- a/lib/latexmath/aggregator.rb +++ b/lib/latexmath/aggregator.rb @@ -31,8 +31,15 @@ def initialize(tokens) @tokens = tokens end + def new_aggregate(tokens) + tokens.each do |token| + Latexmath::Common::Number.new(token) + end + end + def aggregate(tokens = @tokens) aggregated = [] + new_aggregate(tokens.clone) loop do begin diff --git a/lib/latexmath/common/number.rb b/lib/latexmath/common/number.rb new file mode 100644 index 0000000..205a502 --- /dev/null +++ b/lib/latexmath/common/number.rb @@ -0,0 +1,17 @@ +module Latexmath + module Common + class Number < Latexmath::Common::Object + def initialize(number) + @number = number + end + + def larger(other) + value_of > other.value_of ? self : other + end + + def value_of + @number + end + end + end +end diff --git a/lib/latexmath/common/object.rb b/lib/latexmath/common/object.rb new file mode 100644 index 0000000..418f014 --- /dev/null +++ b/lib/latexmath/common/object.rb @@ -0,0 +1,6 @@ +module Latexmath + module Common + class Object + end + end +end diff --git a/lib/latexmath/core/box.rb b/lib/latexmath/core/box.rb new file mode 100644 index 0000000..ab9d665 --- /dev/null +++ b/lib/latexmath/core/box.rb @@ -0,0 +1,14 @@ +module Latexmath + module Core + class Box + def initialize(tokens) + end + + def width + end + + def total_height + end + end + end +end diff --git a/lib/latexmath/core/definition/char_def.rb b/lib/latexmath/core/definition/char_def.rb new file mode 100644 index 0000000..9a67de0 --- /dev/null +++ b/lib/latexmath/core/definition/char_def.rb @@ -0,0 +1,11 @@ +module Latexmath + module Core + module Definition + class CharDef + def initialize(cs, parameters) + # ($class, $cs, $parameters, $replacement, %traits) + end + end + end + end +end diff --git a/lib/latexmath/core/definition/constructor.rb b/lib/latexmath/core/definition/constructor.rb new file mode 100644 index 0000000..33a6bba --- /dev/null +++ b/lib/latexmath/core/definition/constructor.rb @@ -0,0 +1,11 @@ +module Latexmath + module Core + module Definition + class Constructor + def initialize(cs, parameters) + # ($class, $cs, $parameters, $replacement, %traits) + end + end + end + end +end diff --git a/lib/latexmath/core/definition/expandable.rb b/lib/latexmath/core/definition/expandable.rb new file mode 100644 index 0000000..95e026e --- /dev/null +++ b/lib/latexmath/core/definition/expandable.rb @@ -0,0 +1,12 @@ +module Latexmath + module Core + module Definition + class Expandable + attr_reader :cs + def initialize(cs, sparamlist, expansion, options) + @cs = cs + end + end + end + end +end diff --git a/lib/latexmath/core/definition/primitive.rb b/lib/latexmath/core/definition/primitive.rb new file mode 100644 index 0000000..e3cc64f --- /dev/null +++ b/lib/latexmath/core/definition/primitive.rb @@ -0,0 +1,11 @@ +module Latexmath + module Core + module Definition + class Primitive + def initialize(cs, parameters) + # ($class, $cs, $parameters, $replacement, %traits) + end + end + end + end +end diff --git a/lib/latexmath/core/gullet.rb b/lib/latexmath/core/gullet.rb new file mode 100644 index 0000000..7154a49 --- /dev/null +++ b/lib/latexmath/core/gullet.rb @@ -0,0 +1,8 @@ +module Latexmath + module Core + class Gullet + def initialize(options) + end + end + end +end diff --git a/lib/latexmath/core/math/fraction.rb b/lib/latexmath/core/math/fraction.rb new file mode 100644 index 0000000..c8674ed --- /dev/null +++ b/lib/latexmath/core/math/fraction.rb @@ -0,0 +1,24 @@ +module Latexmath + module Core + module Math + class Fraction + def initialize(numerator, denominator) + @denominator = denominator + @numerator = numerator + end + + def w + @numerator.width.larger(denominator.width) + end + + def d + @denominator.total_height.multiply(0.5) + end + + def h + @numerator.total_height.add(d) + end + end + end + end +end diff --git a/lib/latexmath/core/math/matrix.rb b/lib/latexmath/core/math/matrix.rb new file mode 100644 index 0000000..ecc1b1d --- /dev/null +++ b/lib/latexmath/core/math/matrix.rb @@ -0,0 +1,8 @@ +module Latexmath + module Core + module Math + class Matrix + end + end + end +end diff --git a/lib/latexmath/core/mouth.rb b/lib/latexmath/core/mouth.rb new file mode 100644 index 0000000..83a73d6 --- /dev/null +++ b/lib/latexmath/core/mouth.rb @@ -0,0 +1,245 @@ +module Latexmath + module Core + class Mouth + def initialize(str, _options = {}) + @state = Latexmath::Core::State.new(catcodes: 'standard') + @lineno = 0 + @colno = 0 + @chars = [] + @nchars = 0 + + open_string(str) + end + + # Read the next token, or undef if exhausted. + # Note that this also returns COMMENT tokens containing source comments, + # and also locator comments (file, line# info). + # LaTeXML::Core::Gullet intercepts them and passes them on at appropriate times. + def read_token + loop do + if @colno >= @nchars + @lineno += 1 + @colno = 0 + line = next_line + + unless line # Exhausted the input. + @at_eof = 1 + @chars = [] + @nchars = 0 + return + end + # Remove trailing space, but NOT a control space! End with CR (not \n) since this gets tokenized! + line.gsub!(/((\\ )*)\s*$/, '') # Original perl regexp: s/((\\ )*)\s*$/$1/s + # Then append the appropriaate \endlinechar, or "\r" + if eol = @state.lookup_definition(Token.new("\endlinechar", Token::CC_CS)) + eol = eol.value + line += eol.chr if eol > 0 + else + line += "\r" + end + + @chars = split_chars(line) + @nchars = @chars.size + + while @colno < @nchars && ((@state.lookup_catcode(@chars[@colno]) || Token::CC_OTHER) == Token::CC_SPACE) + + @colno += 1 + end + + # # # Sneak a comment out, every so often. + # if (((@lineno % 25) == 0) && @state.lookup_value('INCLUDE_COMMENTS')) + # return T_COMMENT("**** " . (@shortsource || 'String') . " Line #{@lineno} ****") + # end + end + + if @skipping_spaces # Skip spaces now + while (ch, cc = next_char) && ch && cc == Token::CC_SPACE + end + @colno -= 1 if @colno < @nchars + if defined?(cc) && cc == Token::CC_EOL + if (@state.lookup_value('PRESERVE_NEWLINES') || 0) > 1 + + else + next_char + @colno -= 1 if @colno < @nchars + end + end + @skipping_spaces = false + end + + ch, cc = next_char + token = (defined?(cc) ? dispatch(ch, cc) : nil) + token = Core::Token.new(ch, cc) if token.nil? + return token if token + end + end + + # ********************************************************************** + # Read all tokens until a token equal to $until (if given), or until exhausted. + # Returns an empty Tokens list, if there is no input + def read_tokens + @tokens = [] + while token = read_token + @tokens.push(token) + end + @tokens.pop while @tokens.size > 0 && @tokens[-1].catcode == Token::CC_SPACE + Latexmath::Core::Tokens.new(@tokens) + end + + def handle_comment + n = @colno + @colno = @nchars + comment = @chars[n..(@nchars - 1)] + comment.gsub!(/^\s+/, '') + comment.gsub!(/\s+$/, '') + comment && @state.lookup_value('INCLUDE_COMMENTS') ? Token.new(comment, Token::CC_COMMENT) : nil + end + + def handle_escape + ch, cc = next_char + cs = "\\#{ch}" + if defined?(cc) && cc == Token::CC_LETTER + while (ch, cc = next_char) && ch && cc == Token::CC_LETTER + cs += ch + end + @skipping_spaces = true + @colno -= 1 + end + + Token.new(cs, Token::CC_CS) + end + + def handle_space + while (ch, cc = next_char) && ch && (cc == Token::CC_SPACE || cc == Token::CC_EOL) + end + @colno -= 1 if @colno < @nchars + Token.new(' ', Token::CC_SPACE) + end + + def handle_EOL + token = if @colno == 1 + Token.new('\\par', Token::CC_CS) + else + if @state.lookup_value('PRESERVE_NEWLINES') + Token.new("\n", Token::CC_SPACE) + else + Token.new(' ', Token::CC_SPACE) + end + end + + @colno = @nchars + token + end + + # # # Dispatch table for catcodes. + + # # Possibly want to think about caching (common) letters, etc to keep from + # # creating tokens like crazy... or making them more compact... or ??? + def dispatch(ch, cc) + case cc + when 0 + handle_escape + when 1 + Token.new(ch, Token::CC_BEGIN) + when 2 + Token.new(ch, Token::CC_END) + when 3 + Token.new(ch, Token::CC_MATH) + when 4 + Token.new(ch, Token::CC_ALIGN) + when 5 + handle_EOL + when 6 + Token.new(ch, Token::CC_PARAM) + when 7 + Token.new(ch, Token::CC_SUPER) + when 8 + Token.new(ch, Token::CC_SUB) + when 9 + nil + when 10 + handle_space + when 11 + Token.new(ch, Token::CC_LETTER) + when 12 + Token.new(ch, Token::CC_OTHER) + when 13 + Token.new(ch, Token::CC_ACTIVE) + when 14 + handle_comment + when 15 + Token.new(ch, Token::CC_OTHER) + end + end + + # Get the next character & it's catcode from the input, + # handling TeX's "^^" encoding. + # Note that this is the only place where catcode lookup is done, + # and that it is somewhat `inlined'. + def next_char + if @colno < @nchars + ch = @chars[@colno] + @colno += 1 + cc = @state.lookup_catcode(ch) || Token::CC_OTHER + + if cc == Token::CC_SUPER && (@colno + 1) < @nchars && ch == @chars[@colno] + if (@colno + 2) < @nchars && c1 = @chars[@colno + 1].match?(/^[0-9a-f]$/) && c2 = @chars[@colno + 2].match?(/^[0-9a-f]$/) + ch = "#{c1}#{c2}".hex.chr + 4.times { @chars.delete_at(@colno) } + @chars.insert(@colno, ch) + @nchars -= 3 + else # OR ^^ followed by a SINGLE Control char type code??? + c = @chars[@colno + 1] + cn = c.ord + ch = (cn + (cn >= 64 ? -64 : 64)).chr + 3.times { @chars.delete_at(@colno) } + @chars.insert(@colno, ch) + @nchars -= 2 + end + cc = @state.lookup_catcode(ch) || Token::CC_OTHER + end + [ch, cc] + else + [nil, nil] + end + end + + # This is (hopefully) a correct way to split a line into "chars", + # or what is probably more desired is "Grapheme clusters" (even "extended") + # These are unicode characters that include any following combining chars, accents & such. + # I am thinking that when we deal with unicode this may be the most correct way? + # If it's not the way XeTeX does it, perhaps, it must be that ALL combining chars + # have to be converted to the proper accent control sequences! + def split_chars(line) + line.scan(/\X/) + end + + def next_line + return unless @buffer.is_a?(Array) + return if @buffer.empty? + + line = @buffer.shift + @buffer.empty? ? "#{line}\r" : line # No CR on last line! + end + + def open_string(str) + if defined?(str) + if str.valid_encoding? + elsif true + end + end + + @string = str + @buffer = defined?(str) ? split_lines(str) : [] + end + + # # This is (hopefully) a platform independent way of splitting a string + # # into "lines" ending with CRLF, CR or LF (DOS, Mac or Unix). + # # Note that TeX considers newlines to be \r, ie CR, ie ^^M + def split_lines(str) + str.gsub!(/(?:\015\012|\015|\012)/, "\r") # Normalize remaining. Perl: s/(?:\015\012|\015|\012)/\r/sg + str.split("\r") # And split. + end + end + end +end diff --git a/lib/latexmath/core/package.rb b/lib/latexmath/core/package.rb new file mode 100644 index 0000000..798f8ea --- /dev/null +++ b/lib/latexmath/core/package.rb @@ -0,0 +1,80 @@ + + +module Latexmath + module Core + class Package + #====================================================================== + # Define a LaTeX environment + # Note that the body of the environment is treated is the 'body' parameter in the constructor. + ENVIRONMENT_OPTIONS = { + mode: 1, requireMath: 1, forbidMath: 1, + properties: 1, nargs: 1, font: 1, + beforeDigest: 1, afterDigest: 1, + afterDigestBegin: 1, beforeDigestEnd: 1, afterDigestBody: 1, + beforeConstruct: 1, afterConstruct: 1, + reversion: 1, sizer: 1, scope: 1, locked: 1 + } + + # Define a Macro: Essentially an alias for DefExpandable + # For convenience, the $expansion can be a string which will be tokenized. + MACRO_OPTIONS = { + scope: 1, locked: 1, mathactive: 1, + protected: 1, outer: 1, long: 1 + } + + + def initialize(state = Latexmath::Core::State.new(catcodes: 'standard')) + @state = state + end + + + def macro(proto, expansion, options = {}) + check_options("DefMacro (#{proto})", MACRO_OPTIONS, options) + cs, paramlist = parse_prototype(proto) + macro_i(cs, paramlist, expansion, options) + end + + def macro_i(cs, paramlist, expansion, options = []) + if(!defined?(expansion)) + expansion = Latexmath::Core::Tokens.new + end + + if cs.is_a?(Token) && options['mathactive'] + @state.assign_mathcode(cs: 0x8000, scope: options['scope']) + end + + @state.install_definition(Definition::Expandable.new(cs, paramlist, expansion, options), options['scope']) + end + + def check_options(operation, allowed, options) + badops = allowed.keys & options.keys + raise("#{operation} does not accept options:" . badops.join(', ')) if badops.any? + end + + def parse_prototype(proto) + if proto.is_a?(Latexmath::Core::Token) + return proto, nil + end + + if matches = proto.match(/^\\csname\s+(.*)\\endcsname/) + cs = Token.new("#{matches[1]}", Token::CC_CS) + elsif matches = proto.match(/^(\\[a-zA-Z@]+)/) # Mach a cs + cs = Token.new("#{matches[1]}", Token::CC_CS) + elsif matches = proto.match(/^(\\.)/) # Match a single char cs, env name,... + cs = Token.new("#{matches[1]}", Token::CC_CS) + elsif matches = proto.match(/^(.)/) # Match an active char + cs = tokenize_internal(matches[1]).unlist + else + raise("Definition prototype doesn't have proper control sequence: \"#{proto}\"") + end + + return cs, proto + end + + def tokenize_internal(str) + @sty_cattable = LaTeXML::Core::State.new(catcodes: 'style') unless @sty_cattable + return Latexmath::Core::Mouth.new(str, @sty_cattable).read_tokens + end + end + end +end diff --git a/lib/latexmath/core/state.rb b/lib/latexmath/core/state.rb new file mode 100644 index 0000000..0225641 --- /dev/null +++ b/lib/latexmath/core/state.rb @@ -0,0 +1,120 @@ +module Latexmath + module Core + class State + attr_accessor :unlocked + #====================================================================== + # Specialized versions of lookup & assign for dealing with definitions + + ACTIVE_OR_CS = [ + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 1, 0, 0, + 1, 0] + LETTER_OR_OTHER = [ + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1, + 1, 0, 0, 0, + 0, 0] + + # options: + # catcodes: standard|style|none + # stomach: a Stomach object. + # model: a Mod el object. + def initialize(params = {}) + @catcodes = params.fetch(:catcodes, 'standard') + @meaning = {} + + @value = {} + # Note that "100" is hardwired into TeX, The Program!!! + @value['MAX_ERRORS'] = 100 + @value['VERBOSITY'] = 0 + # Standard TeX units, in scaled points + @value['UNITS'] = { + "pt" => 65536, "pc" => 12 * 65536, "in" => 72.27 * 65536, "bp" => 72.27 * 65536 / 72, + "cm" => 72.27 * 65536 / 2.54, "mm" => 72.27 * 65536 / 2.54 / 10, "dd" => 1238 * 65536 / 1157, + "cc" => 12 * 1238 * 65536 / 1157, "sp" => 1, + "px" => 72.27 * 65536 / 72 # Assume px=bp ? + } + + if @catcodes.match?(/^(standard|style)/) + # Setup default catcodes. + @catcode = { + "\\" => Token::CC_ESCAPE, "{" => Token::CC_BEGIN, "}" => Token::CC_END, "\$" => Token::CC_MATH, + "&" => Token::CC_ALIGN, "\r" => Token::CC_EOL, "#" => Token::CC_PARAM, "^" => Token::CC_SUPER, + "_" => Token::CC_SUB, " " => Token::CC_SPACE, "\t" => Token::CC_SPACE, "%" => Token::CC_COMMENT, + "~" => Token::CC_ACTIVE, 0.chr => Token::CC_ESCAPE, "\f" => Token::CC_ACTIVE + } + + for c in 'A'.ord..'Z'.ord + @catcode[c.chr] = Token::CC_LETTER + @catcode[(c + 'a'.ord - 'A'.ord).chr] = Token::CC_LETTER + end + end + end + + + #====================================================================== + # Lookup & assign a character's Catcode + def lookup_catcode(key) + @catcode.fetch(key, nil) + end + + + # Lookup & assign a general Value + # [Note that the more direct $$self{value}{$_[1]}[0]; works, but creates entries + # this could concievably cause space issues, but timing doesn't show improvements this way] + def lookup_value(key) + @value.fetch(key, nil) + end + + # used for expansion & various queries + # Since we're not doing digestion here, we don't need to handle mathactive, + # nor cs let to executable tokens + # This returns a definition object, or undef + + # merge of @executable_catcode & @PRIMITIVE_NAME + EXECUTABLE_PRIMITIVE_NAME = [ # [CONSTANT] + nil, 'Begin', 'End', 'Math', + 'Align', nil, nil, 'Superscript', + 'Subscript', nil, nil, nil, + nil, nil, nil, nil, + nil, nil] + + def lookup_definition(token) + return unless token + + cc = token.catcode + lookupname = ACTIVE_OR_CS[cc] ? token.to_s : EXECUTABLE_PRIMITIVE_NAME[cc] + + if lookupname && entry = @meaning[lookupname] && defn = entry[0] && !defn.is_a?(Token) + return defn + end + end + + # And a shorthand for installing definitions + def install_definition(definition, scope) + token = definition.cs + cs = Token::PRIMITIVE_NAME[token.catcode] || token.to_s + + if (lookup_value("#{cs}:locked") && unlocked == false) + s = stomach.gullet.source + # report if the redefinition seems to come from document source + if(!defined?(s) || s.match?(/\.(tex|bib)$/)) && s.match?(/\.code\.tex$/) + raise "Ignoring redefinition of #{cs}" + end + end + + assign_internal('meaning', cs, definition, scope) + end + + def assign_internal(table, key, value, scope) + if table == 'meaning' + @meaning[key] = value + end + end + end + end +end + diff --git a/lib/latexmath/core/stomach.rb b/lib/latexmath/core/stomach.rb new file mode 100644 index 0000000..022e46f --- /dev/null +++ b/lib/latexmath/core/stomach.rb @@ -0,0 +1,21 @@ +module Latexmath + module Core + class Stomach + def initialize(options) + @gullet = Latexmath::Core::Gullet.new(options) + @boxing = [] + @token_stack = [] + + # $STATE->assignValue(MODE => 'text', 'global'); + # $STATE->assignValue(IN_MATH => 0, 'global'); + # $STATE->assignValue(PRESERVE_NEWLINES => 1, 'global'); + # $STATE->assignValue(afterGroup => [], 'global'); + # $STATE->assignValue(afterAssignment => undef, 'global'); + # $STATE->assignValue(groupInitiator => 'Initialization', 'global'); + # # Setup default fonts. + # $STATE->assignValue(font => LaTeXML::Common::Font->textDefault(), 'global'); + # $STATE->assignValue(mathfont => LaTeXML::Common::Font->mathDefault(), 'global'); + end + end + end +end diff --git a/lib/latexmath/core/token.rb b/lib/latexmath/core/token.rb new file mode 100644 index 0000000..d0e4da2 --- /dev/null +++ b/lib/latexmath/core/token.rb @@ -0,0 +1,144 @@ +module Latexmath + module Core + class Token + # Catcodes + CC_ESCAPE = 0 + CC_BEGIN = 1 + CC_END = 2 + CC_MATH = 3 + CC_ALIGN = 4 + CC_EOL = 5 + CC_PARAM = 6 + CC_SUPER = 7 + CC_SUB = 8 + CC_IGNORE = 9 + CC_SPACE = 10 + CC_LETTER = 11 + CC_OTHER = 12 + CC_ACTIVE = 13 + CC_COMMENT = 14 + CC_INVALID = 15 + # Extended Catcodes for expanded output. + CC_CS = 16 + CC_MARKER = 17 # non TeX extension! + + #====================================================================== + # Constructors. + + + + + # [The documentation for constant is a bit confusing about subs, + # but these apparently DO generate constants; you always get the same one] + # These are immutable + # T_BEGIN = bless ['{', CC_BEGIN], 'LaTeXML::Core::Token'; + # T_END = bless ['}', CC_END], 'LaTeXML::Core::Token'; + # T_MATH = bless ['$', CC_MATH], 'LaTeXML::Core::Token'; + # T_ALIGN = bless ['&', CC_ALIGN], 'LaTeXML::Core::Token'; + # T_PARAM = bless ['#', CC_PARAM], 'LaTeXML::Core::Token'; + # T_SUPER = bless ['^', CC_SUPER], 'LaTeXML::Core::Token'; + # T_SUB = bless ['_', CC_SUB], 'LaTeXML::Core::Token'; + # T_SPACE = bless [' ', CC_SPACE], 'LaTeXML::Core::Token'; + # T_CR = bless ["\n", CC_SPACE], 'LaTeXML::Core::Token'; + + T_BEGIN = ['{', CC_BEGIN] + T_END = ['}', CC_END] + T_MATH = ['$', CC_MATH] + T_ALIGN = ['&', CC_ALIGN] + T_PARAM = ['#', CC_PARAM] + T_SUPER = ['^', CC_SUPER] + T_SUB = ['_', CC_SUB] + T_SPACE = [' ', CC_SPACE] + T_CR = ["\n", CC_SPACE] + + CATCODES = { + '{': CC_BEGIN, + '}': CC_END, + '$': CC_MATH, + '&': CC_ALIGN, + '#': CC_PARAM, + '^': CC_SUPER, + '_': CC_SUB, + ' ': CC_SPACE, + "\n": CC_SPACE + } + + # #====================================================================== + # # Categories of Category codes. + # # For Tokens with these catcodes, only the catcode is relevant for comparison. + # # (if they even make it to a stage where they get compared) + # our @primitive_catcode = ( # [CONSTANT] + # 1, 1, 1, 1, + # 1, 1, 1, 1, + # 1, 0, 1, 0, + # 0, 0, 0, 0, + # 0, 0); + # our @executable_catcode = ( # [CONSTANT] + # 0, 1, 1, 1, + # 1, 0, 0, 1, + # 1, 0, 0, 0, + # 0, 1, 0, 0, + # 1, 0); + + # our @standardchar = ( # [CONSTANT] + # "\\", '{', '}', q{$}, + # q{&}, "\n", q{#}, q{^}, + # q{_}, undef, undef, undef, + # undef, undef, q{%}, undef, + # undef, undef); + + # our @CC_NAME = #[CONSTANT] + # qw(Escape Begin End Math + # Align EOL Parameter Superscript + # Subscript Ignore Space Letter + # Other Active Comment Invalid + # ControlSequence Marker); + PRIMITIVE_NAME = [ + 'Escape', 'Begin', 'End', 'Math', + 'Align', 'EOL', 'Parameter', 'Superscript', + 'Subscript', nil, 'Space', nil, + nil, nil, nil, nil, + nil, nil + ] + # our @CC_SHORT_NAME = #[CONSTANT] + # qw(T_ESCAPE T_BEGIN T_END T_MATH + # T_ALIGN T_EOL T_PARAM T_SUPER + # T_SUB T_IGNORE T_SPACE T_LETTER + # T_OTHER T_ACTIVE T_COMMENT T_INVALID + # T_CS + # ); + + # sub T_LETTER { my ($c) = @_; return bless [$c, CC_LETTER], 'LaTeXML::Core::Token'; } + # sub T_OTHER { my ($c) = @_; return bless [$c, CC_OTHER], 'LaTeXML::Core::Token'; } + # sub T_ACTIVE { my ($c) = @_; return bless [$c, CC_ACTIVE], 'LaTeXML::Core::Token'; } + # sub T_COMMENT { my ($c) = @_; return bless ['%' . ($c || ''), CC_COMMENT], 'LaTeXML::Core::Token'; } + # sub T_CS { my ($c) = @_; return bless [$c, CC_CS], 'LaTeXML::Core::Token'; } + # # Illegal: don't use unless you know... + # sub T_MARKER { my ($t) = @_; return bless [$t, CC_MARKER], 'LaTeXML::Core::Token'; } + + def initialize(token, cc) + @token = token + @cc = cc + end + + # Return the string or character part of the token + def to_s + @token + end + + # Return the catcode of the token. + def catcode + @cc + end + + def explode + end + + def explode_text + end + + def untex + end + end + end +end diff --git a/lib/latexmath/core/tokens.rb b/lib/latexmath/core/tokens.rb new file mode 100644 index 0000000..656a46f --- /dev/null +++ b/lib/latexmath/core/tokens.rb @@ -0,0 +1,13 @@ +module Latexmath + module Core + class Tokens + def initialize(tokens) + @tokens = tokens + end + + def to_a + @tokens + end + end + end +end diff --git a/lib/latexmath/packages/aas.rb b/lib/latexmath/packages/aas.rb new file mode 100644 index 0000000..c3b95e6 --- /dev/null +++ b/lib/latexmath/packages/aas.rb @@ -0,0 +1,29 @@ +#====================================================================== +# 2.17.3 Fractions + +# \case{1}{2} == textstyle fraction +# AND, apparently allowed in text mode! +# DefMacro('\case{}{}', '\ensuremath{\text@frac{#1}{#2}}'); +# DefConstructor('\text@frac ScriptStyle ScriptStyle', +# "" +# . "" +# . "#1#2" +# . "", +# sizer => sub { fracSizer($_[0]->getArg(1), $_[0]->getArg(2)); }); +# Let('\slantfrac', '\case'); + + +module Latexmath + module Packages + class Aas + #====================================================================== + # 2.17.3 Fractions + + # \case{1}{2} == textstyle fraction + # AND, apparently allowed in text mode! + def initialize(package) + package.macro('\case{}{}', '\ensuremath{\text@frac{#1}{#2}}') + end + end + end +end diff --git a/lib/latexmath/packages/amsmath.rb b/lib/latexmath/packages/amsmath.rb new file mode 100644 index 0000000..70a97e4 --- /dev/null +++ b/lib/latexmath/packages/amsmath.rb @@ -0,0 +1,25 @@ +module Latexmath + module Packages + class Amsmath + def initialize(package) + # NOTE: Use \@left,\@right here, to avoid the hidden grouping (see TeX.pool, \@hidden@bgroup) + # NOTE: These defns have an column spec [] (omit that for mathtools) + package.macro('\matrix', '\lx@ams@matrix{name=matrix,datameaning=matrix}') + package.macro('\endmatrix', '\lx@end@ams@matrix') + package.macro('\pmatrix', '\lx@ams@matrix{name=pmatrix,datameaning=matrix,left=\@left(,right=\@right)}') + package.macro('\endpmatrix', '\lx@end@ams@matrix') + package.macro('\bmatrix', '\lx@ams@matrix{name=bmatrix,datameaning=matrix,left=\@left[,right=\@right]}') + package.macro('\endbmatrix', '\lx@end@ams@matrix') + package.macro('\Bmatrix', '\lx@ams@matrix{name=Bmatrix,datameaning=matrix,left=\@left\{,right=\@right\}}') + package.macro('\endBmatrix', '\lx@end@ams@matrix') + package.macro('\vmatrix', '\lx@ams@matrix{name=vmatrix,delimitermeaning=determinant,datameaning=matrix,left=\@left|,right=\@right|}') + package.macro('\endvmatrix', '\lx@end@ams@matrix') + package.macro('\Vmatrix', '\lx@ams@matrix{name=Vmatrix,delimitermeaning=norm,datameaning=matrix,left=\@left\|,right=\@right\|}') + package.macro('\endVmatrix', '\lx@end@ams@matrix') + #package.macro('\smallmatrix', '\lx@ams@matrix{name=smallmatrix,atameaning=matrix,left=\scriptsize}') + package.macro('\smallmatrix', '\lx@ams@matrix{name=smallmatrix,atameaning=matrix,style=\scriptsize}') + package.macro('\endsmallmatrix', '\lx@end@ams@matrix') + end + end + end +end diff --git a/lib/latexmath/packages/latex.rb b/lib/latexmath/packages/latex.rb new file mode 100644 index 0000000..2dc4364 --- /dev/null +++ b/lib/latexmath/packages/latex.rb @@ -0,0 +1,62 @@ +module Latexmath + module Packages + module Latex + #====================================================================== + # C.1.2 Environments + #====================================================================== + + # In LaTeX, \newenvironment{env} defines \env and \endenv. + # \begin{env} & \end{env} open/close a group, and invoke these. + # In fact, the \env & \endenv don't have to have been created by + # \newenvironment; And in fact \endenv doesn't even have to be defined! + # [it is created by \csname, and equiv to \relax if no previous defn] + + # We need to respect these usages here, but we also want to be able + # to define environment constructors that `capture' the body so that + # it can be processed specially, if needed. These are the magic + # '\begin{env}', '\end{env}' control sequences created by DefEnvironment. + + + Macro.new('\begin{}') + + # DefMacro('\begin{}', sub { + # my ($gullet, $env) = @_; + # my $name = $env && ToString(Expand($env)); + # my $before = LookupValue('@environment@' . $name . '@beforebegin'); + # my $after = LookupValue('@environment@' . $name . '@atbegin'); + # if (IsDefined("\\begin{$name}")) { + # (($before ? @$before : ()), + # T_CS("\\begin{$name}")); } # Magic cs! + # else { + # my $token = T_CS("\\$name"); + # if (!IsDefined($token)) { + # my $undef = '{' . $name . '}'; + # $STATE->noteStatus(undefined => $undef); + # Error('undefined', $undef, $gullet, "The environment " . $undef . " is not defined."); + # $STATE->installDefinition(LaTeXML::Core::Definition::Constructor->new($token, undef, + # sub { $_[0]->makeError('undefined', $undef); })); } + # (($before ? @$before : ()), + # T_CS('\begingroup'), + # ($after ? @$after : ()), + # Invocation(T_CS('\lx@setcurrenvir'), $env), + # $token,); } }); + + # DefMacro('\end{}', sub { + # my ($gullet, $env) = @_; + # my $name = $env && ToString(Expand($env)); + # my $before = LookupValue('@environment@' . $name . '@atend'); + # my $after = LookupValue('@environment@' . $name . '@afterend'); + # my $t; + + # if (IsDefined($t = T_CS("\\end{$name}"))) { + # ($t, + # ($after ? @$after : ())); } # Magic CS! + # else { + # $t = T_CS("\\end$name"); + # (($before ? @$before : ()), + # (IsDefined($t) ? $t : ()), + # T_CS('\endgroup'), + # ($after ? @$after : ())); } }); + end + end +end diff --git a/lib/latexmath/packages/tex.rb b/lib/latexmath/packages/tex.rb new file mode 100644 index 0000000..39b0b53 --- /dev/null +++ b/lib/latexmath/packages/tex.rb @@ -0,0 +1,7 @@ +module Latexmath + module Packages + class Tex + + end + end +end diff --git a/lib/latexmath/post/mathml.rb b/lib/latexmath/post/mathml.rb new file mode 100644 index 0000000..aed24db --- /dev/null +++ b/lib/latexmath/post/mathml.rb @@ -0,0 +1,7 @@ +module Latexmath + module Post + class Mathml + + end + end +end diff --git a/lib/latexmath/tokenizer.rb b/lib/latexmath/tokenizer.rb index fa1fb8c..94be40c 100644 --- a/lib/latexmath/tokenizer.rb +++ b/lib/latexmath/tokenizer.rb @@ -67,6 +67,7 @@ def fetch_token elsif scan(/[0-9]+\.[0-9]+/) matched elsif scan(/[0-9]+/) + Latexmath::Common::Number.new(matched) matched elsif scan(/ /) matched diff --git a/spec/core/package_spec.rb b/spec/core/package_spec.rb new file mode 100644 index 0000000..5dd638b --- /dev/null +++ b/spec/core/package_spec.rb @@ -0,0 +1,17 @@ +RSpec.describe Latexmath::Core::Package do + context 'Package' do + it '#parse_prototype' do + tokens = Latexmath::Core::Package.new.parse_prototype('\case{}{}') + expect( + tokens.first.to_s + ).to eq("\\case") + expect( + tokens.first.catcode + ).to eq(16) + end + it 'Aas' do + aas = Latexmath::Packages::Aas.new(Latexmath::Core::Package.new) + expect(aas).to eq(true) + end + end +end diff --git a/spec/mouth_tokenizer_spec.rb b/spec/mouth_tokenizer_spec.rb new file mode 100644 index 0000000..ee9fb1e --- /dev/null +++ b/spec/mouth_tokenizer_spec.rb @@ -0,0 +1,100 @@ +RSpec.describe Latexmath::Core::Mouth do + + context 'Sample 1' do + tex = '\\varepsilon = \\frac{1}{2} ( J + J^t )' + it 'Tokenize latex string' do + expect( + Latexmath::Core::Mouth.new(tex).read_tokens.to_a.map(&:to_s) + ).to eq( + ['\\varepsilon', '=', ' ', '\\frac', '{', '1', '}', '{', '2', '}', ' ', '(', ' ', 'J', ' ', '+', ' ', 'J', '^', 't', ' ', ')'] + ) + end + + it 'Tokenize latex codes' do + expect( + Latexmath::Core::Mouth.new(tex).read_tokens.to_a.map(&:catcode) + ).to eq( + [16, 12, 10, 16, 1, 12, 2, 1, 12, 2, 10, 12, 10, 11, 10, 12, 10, 11, 7, 11, 10, 12] + ) + end + end + + context 'Sample 2' do + tex = '$$f_i = \\sum_{j=1}^2 s_{ij} n_j \\quad {\\rm for} \\quad i = 1,2$$' + it 'Tokenize latex string' do + expect( + Latexmath::Core::Mouth.new(tex).read_tokens.to_a.map(&:to_s) + ).to eq( + ['$', '$', 'f', '_', 'i', ' ', '=', ' ', '\\sum', '_', '{', 'j', '=', '1', '}', '^', '2', ' ', 's', '_', '{', 'i', 'j', '}', ' ', 'n', '_', 'j', ' ', '\\quad', '{', '\\rm', 'f', 'o', 'r', '}', ' ', '\\quad', 'i', ' ', '=', ' ', '1', ',', '2', '$', '$'] + ) + end + + it 'Tokenize latex codes' do + expect( + Latexmath::Core::Mouth.new(tex).read_tokens.to_a.map(&:catcode) + ).to eq( + [3, 3, 11, 8, 11, 10, 12, 10, 16, 8, 1, 11, 12, 12, 2, 7, 12, 10, 11, 8, 1, 11, 11, 2, + 10, 11, 8, 11, 10, 16, 1, 16, 11, 11, 11, 2, 10, 16, 11, 10, 12, 10, 12, 12, 12, 3, 3] + ) + end + end + + context 'Sample 3' do + tex = 'G = {\\displaystyle \\frac{E}{2(1 + \\nu)}}' + it 'Tokenize latex string' do + expect( + Latexmath::Core::Mouth.new(tex).read_tokens.to_a.map(&:to_s) + ).to eq( + ['G', ' ', '=', ' ', '{', '\\displaystyle', '\\frac', '{', 'E', '}', '{', '2', '(', '1', ' ', '+', ' ', '\\nu', ')', '}', '}'] + ) + end + + it 'Tokenize latex codes' do + expect( + Latexmath::Core::Mouth.new(tex).read_tokens.to_a.map(&:catcode) + ).to eq( + [11, 10, 12, 10, 1, 16, 16, 1, 11, 2, 1, 12, 12, 12, 10, 12, 10, 16, 12, 2, 2] + ) + end + end + + context 'Sample 4' do + tex = '\\bf{x^\\prime} = \\bf{\\xi}' + + it 'Tokenize latex string' do + expect( + Latexmath::Core::Mouth.new(tex).read_tokens.to_a.map(&:to_s) + ).to eq( + ["\\bf", "{", "x", "^", "\\prime", "}", " ", "=", " ", "\\bf", "{", "\\xi", "}"] + ) + end + + it 'Tokenize latex codes' do + expect( + Latexmath::Core::Mouth.new(tex).read_tokens.to_a.map(&:catcode) + ).to eq( + [16,1,11,7,16,2,10,12,10,16,1,16,2] + ) + end + end + + context 'Sample 5' do + tex = '\\bf{z^\\prime} = \\langle \\bf{\\xi} \\times \\bf{\\eta} \\rangle' + + it 'Tokenize latex string' do + expect( + Latexmath::Core::Mouth.new(tex).read_tokens.to_a.map(&:to_s) + ).to eq( + ["\\bf", "{", "z", "^", "\\prime", "}", " ", "=", " ", "\\langle", "\\bf", "{", "\\xi", "}", " ", "\\times", "\\bf", "{", "\\eta", "}", " ","\\rangle"] + ) + end + + it 'Tokenize latex codes' do + expect( + Latexmath::Core::Mouth.new(tex).read_tokens.to_a.map(&:catcode) + ).to eq( + [16,1,11,7,16,2,10,12,10,16,16,1,16,2,10,16,16,1,16,2,10,16] + ) + end + end +end