Local Python Tutor parity shell and Python 3.11/C++ local backend fixes

.gitignore

@@ -19,3 +19,9 @@ v4-cokapi/backends/javascript/node-v6.0.0-darwin-x64/
 
 # this .gz file is too big to fit into a github repo
 v4-cokapi/backends/java/jdk-8u20-linux-x64.tar.gz
+v5-unity/.venv/
+.venv/
+__pycache__/
+.DS_Store
+*.backup-before-imp-fix
+*.backup-before-*

README-local.md

@@ -0,0 +1,12 @@
+# Local runbook
+
+## Start
+Run:
+
+./start-all.sh
+
+Then open:
+http://localhost:5000/visualize.html
+
+## Stop
+Press Ctrl+C in the terminal running ./start-all.sh

local-cpp20-backend/Dockerfile

@@ -0,0 +1,72 @@
+FROM ubuntu:18.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+    git \
+    build-essential \
+    autotools-dev \
+    automake \
+    autoconf \
+    libtool \
+    pkg-config \
+    libc6-dbg \
+    python \
+    python3 \
+    software-properties-common \
+    ca-certificates \
+    curl \
+    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
+    && apt-get update \
+    && apt-get install -y gcc-10 g++-10 \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 \
+    && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN git clone https://github.com/JuezUN/opt-cpp-backend.git /tmp/opt-cpp-backend
+
+RUN cd /tmp/opt-cpp-backend/valgrind-3.11.0 \
+    && ./autogen.sh \
+    && ./configure --prefix=`pwd`/inst \
+    && make -j2 \
+    && make install
+
+RUN python3 - <<'PY'
+from pathlib import Path
+
+p = Path("/tmp/opt-cpp-backend/run_cpp_backend.py")
+text = p.read_text()
+
+old_text = text
+
+replacements = {
+    "DIALECT = '-std=c++11'": "DIALECT = '-std=gnu++20'",
+    'DIALECT = "-std=c++11"': 'DIALECT = "-std=gnu++20"',
+    "DIALECT='-std=c++11'": "DIALECT='-std=gnu++20'",
+    'DIALECT="-std=c++11"': 'DIALECT="-std=gnu++20"',
+    "-std=c++11": "-std=gnu++20",
+    "-std=gnu++11": "-std=gnu++20",
+    "-std=c++0x": "-std=gnu++20",
+    "-std=c11": "-std=gnu17",
+}
+
+for old, new in replacements.items():
+    text = text.replace(old, new)
+
+p.write_text(text)
+
+if "-std=c++11" in text or "-std=gnu++11" in text or "-std=c++0x" in text:
+    raise SystemExit("C++11 flag still exists in run_cpp_backend.py")
+
+if "-std=gnu++20" not in text:
+    raise SystemExit("GNU++20 flag was not inserted into run_cpp_backend.py")
+
+print("Verified run_cpp_backend.py now uses GNU++20.")
+PY
+
+RUN grep -n "DIALECT" /tmp/opt-cpp-backend/run_cpp_backend.py \
+    && g++ --version
+
+RUN useradd netuser \
+    && cd /tmp \
+    && find opt-cpp-backend | xargs chown netuser

local-cpp20-backend/Dockerfile.preserve-display

@@ -0,0 +1,276 @@
+FROM cpp-tutor/opt-cpp-backend-cpp20:local
+
+RUN cat > /tmp/opt-cpp-backend/run_cpp_backend_cpp20_wrapper.py <<'PY'
+#!/usr/bin/env python
+import json
+import re
+import sys
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+
+def split_names(names):
+    return [name.strip() for name in names.split(',') if name.strip()]
+
+
+def char_literal(ch):
+    if ch == "'":
+        return "\\'"
+    if ch == "\\":
+        return "\\\\"
+    if ch == "\n":
+        return "\\n"
+    if ch == "\t":
+        return "\\t"
+    return ch
+
+
+def rewrite_simple_string_literals(code):
+    known = {}
+
+    pattern = re.compile(
+        r'\bstd::string\s+(?P<name>[A-Za-z_]\w*)\s*=\s*(?P<literal>"[^"]*")\s*;'
+    )
+
+    def repl(match):
+        name = match.group('name')
+        literal = match.group('literal')
+        known[name] = literal[1:-1]
+        return 'const char* %s = %s;' % (name, literal)
+
+    rewritten = pattern.sub(repl, code)
+    return rewritten, known
+
+
+def rewrite_string_starts_ends_with(code, known_strings):
+    starts_pattern = re.compile(
+        r'(?P<obj>\b[A-Za-z_]\w*)\.starts_with\s*\((?P<arg>"[^"]*")\)'
+    )
+
+    ends_pattern = re.compile(
+        r'(?P<obj>\b[A-Za-z_]\w*)\.ends_with\s*\((?P<arg>"[^"]*")\)'
+    )
+
+    def starts_repl(match):
+        obj = match.group('obj')
+        prefix = match.group('arg')[1:-1]
+
+        if obj not in known_strings:
+            return match.group(0)
+
+        text_value = known_strings[obj]
+        return 'true' if text_value.startswith(prefix) else 'false'
+
+    def ends_repl(match):
+        obj = match.group('obj')
+        suffix = match.group('arg')[1:-1]
+
+        if obj not in known_strings:
+            return match.group(0)
+
+        text_value = known_strings[obj]
+        return 'true' if text_value.endswith(suffix) else 'false'
+
+    code = starts_pattern.sub(starts_repl, code)
+    code = ends_pattern.sub(ends_repl, code)
+
+    return code
+
+
+def rewrite_numbers(code):
+    code = code.replace('#include <numbers>', '// cpp-tutor: <numbers> constants rewritten for local visualizer')
+
+    replacements = {
+        'std::numbers::pi_v<double>': '3.14159265358979323846264338327950288',
+        'std::numbers::pi': '3.14159265358979323846264338327950288',
+        'std::numbers::e_v<double>': '2.7182818284590452353602874713527',
+        'std::numbers::e': '2.7182818284590452353602874713527',
+        'std::numbers::sqrt2_v<double>': '1.4142135623730950488016887242097',
+        'std::numbers::sqrt2': '1.4142135623730950488016887242097',
+    }
+
+    for old, new in replacements.items():
+        code = code.replace(old, new)
+
+    return code
+
+
+def rewrite_ssize(code):
+    return re.sub(
+        r'std::ssize\s*\(\s*(?P<c>[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s*\)',
+        lambda m: 'static_cast<long long>(%s.size())' % m.group('c'),
+        code
+    )
+
+
+def rewrite_safe_ranges(code):
+    code = code.replace('#include <ranges>', '// cpp-tutor: <ranges> rewritten for local visualizer')
+
+    code = re.sub(
+        r'std::ranges::sort\s*\(\s*(?P<c>[A-Za-z_]\w*)\s*\)',
+        lambda m: 'std::sort(%s.begin(), %s.end())' % (m.group('c'), m.group('c')),
+        code
+    )
+
+    code = re.sub(
+        r'std::ranges::reverse\s*\(\s*(?P<c>[A-Za-z_]\w*)\s*\)',
+        lambda m: 'std::reverse(%s.begin(), %s.end())' % (m.group('c'), m.group('c')),
+        code
+    )
+
+    return code
+
+
+def rewrite_structured_binding_for_loops(code):
+    lines = code.splitlines(True)
+    out = []
+    pending = None
+    counter = 0
+
+    pattern = re.compile(
+        r'^(?P<indent>\s*)for\s*\(\s*'
+        r'(?P<decl>(?:const\s+)?auto(?:\s*(?:&|&&))?)\s*'
+        r'\[\s*(?P<names>[A-Za-z_]\w*(?:\s*,\s*[A-Za-z_]\w*)+)\s*\]\s*'
+        r':\s*(?P<expr>.*?)\s*\)\s*(?P<brace>\{?)\s*(?P<tail>//.*)?(?P<nl>\r?\n?)$'
+    )
+
+    for line in lines:
+        match = pattern.match(line)
+
+        if match:
+            counter += 1
+            indent = match.group('indent')
+            names = split_names(match.group('names'))
+            expr = match.group('expr')
+            brace = match.group('brace')
+            tail = match.group('tail') or ''
+            nl = match.group('nl') or '\n'
+            temp = '__cpp_tutor_sb_%d' % counter
+
+            getters = ' '.join(
+                ['auto %s = std::get<%d>(%s);' % (name, i, temp) for i, name in enumerate(names)]
+            )
+
+            if brace:
+                out.append('%sfor (auto&& %s : %s) { %s%s%s' %
+                           (indent, temp, expr, getters, (' ' + tail) if tail else '', nl))
+            else:
+                out.append('%sfor (auto&& %s : %s)%s%s' %
+                           (indent, temp, expr, (' ' + tail) if tail else '', nl))
+                pending = (indent, getters)
+
+            continue
+
+        if pending and line.strip().startswith('{'):
+            indent, getters = pending
+            nl = '\n' if line.endswith('\n') else ''
+            out.append('%s{ %s%s' % (indent, getters, nl))
+            pending = None
+            continue
+
+        out.append(line)
+
+    return ''.join(out)
+
+
+def rewrite_standalone_structured_bindings_safely(code):
+    lines = code.splitlines(True)
+    out = []
+
+    pattern = re.compile(
+        r'^(?P<indent>\s*)'
+        r'(?P<decl>(?:const\s+)?auto(?:\s*(?:&|&&))?)\s*'
+        r'\[\s*(?P<names>[A-Za-z_]\w*(?:\s*,\s*[A-Za-z_]\w*)+)\s*\]\s*'
+        r'=\s*(?P<expr>.*?);\s*(?P<tail>//.*)?(?P<nl>\r?\n?)$'
+    )
+
+    for line in lines:
+        match = pattern.match(line)
+
+        if not match:
+            out.append(line)
+            continue
+
+        indent = match.group('indent')
+        names = split_names(match.group('names'))
+        expr = match.group('expr')
+        tail = match.group('tail') or ''
+        nl = match.group('nl') or '\n'
+
+        pieces = [
+            'auto %s = std::get<%d>(%s);' % (name, i, expr)
+            for i, name in enumerate(names)
+        ]
+
+        out.append('%s%s%s%s' % (
+            indent,
+            ' '.join(pieces),
+            (' ' + tail) if tail else '',
+            nl
+        ))
+
+    return ''.join(out)
+
+
+def add_needed_includes(original_code, rewritten_code):
+    prefix = ''
+
+    if ('auto [' in original_code or 'const auto [' in original_code or 'auto& [' in original_code or 'auto&& [' in original_code):
+        if '#include <tuple>' not in rewritten_code:
+            prefix += '#include <tuple>\n'
+
+    if ('std::ranges::sort' in original_code or 'std::ranges::reverse' in original_code):
+        if '#include <algorithm>' not in rewritten_code:
+            prefix += '#include <algorithm>\n'
+
+    return prefix + rewritten_code
+
+
+def rewrite_code(code):
+    rewritten = code
+
+    rewritten = rewrite_numbers(rewritten)
+    rewritten = rewrite_ssize(rewritten)
+
+    rewritten, known_strings = rewrite_simple_string_literals(rewritten)
+    rewritten = rewrite_string_starts_ends_with(rewritten, known_strings)
+
+    rewritten = rewrite_structured_binding_for_loops(rewritten)
+    rewritten = rewrite_standalone_structured_bindings_safely(rewritten)
+    rewritten = rewrite_safe_ranges(rewritten)
+    rewritten = add_needed_includes(code, rewritten)
+
+    return rewritten
+
+
+original_code = sys.argv[1]
+rewritten_code = rewrite_code(original_code)
+
+sys.argv[1] = rewritten_code
+
+old_stdout = sys.stdout
+buffer = StringIO()
+sys.stdout = buffer
+
+try:
+    execfile('/tmp/opt-cpp-backend/run_cpp_backend.py')
+finally:
+    sys.stdout = old_stdout
+
+output = buffer.getvalue()
+
+try:
+    data = json.loads(output)
+
+    if rewritten_code != original_code:
+        data['code'] = original_code
+
+    sys.stdout.write(json.dumps(data))
+except Exception:
+    sys.stdout.write(output)
+PY
+
+RUN chmod +x /tmp/opt-cpp-backend/run_cpp_backend_cpp20_wrapper.py