"""MATLAB-vs-Octave compatibility checking. We execute in GNU Octave and the user runs the result in real MATLAB, so there are three failure classes and only one of them is the model's fault: 1. Octave ACCEPTS it, MATLAB REJECTS it. The worst case - we would tell the user "it works" and it would not. 2. MATLAB has it, this sandbox does not (toolbox functions, display, network). The model must be told NOT to rewrite correct code. 3. Both run, results differ. Rarely detectable statically; we flag the few constructs worth flagging and are honest about the rest. Every rule runs against source with string BODIES and comment BODIES blanked out, markers kept. An adversarial review of the first version found three high-severity bugs in this file alone, all of them in the tokeniser rather than in the rules, so read the notes on strip_literals before touching it. The one exception is a rule marked `raw`, which sees the source unstripped. That exists because the worst trap we know of lives *inside* a string body: `regexp(s, '\b40\b', 'match')` returns nothing at all, so a stripped-source rule could never see it and the model gets a wrong count instead of an error. """ from __future__ import annotations import re from dataclasses import dataclass # After one of these keywords, a quote opens a string - it cannot be a # transpose, because there is no value to transpose. Without this, # `case '--'` mis-tokenised and the `--` inside the string was reported # as an Octave-only increment operator in perfectly good MATLAB. _KEYWORDS_BEFORE_STRING = frozenset({ "case", "otherwise", "return", "if", "elseif", "else", "while", "switch", "for", "parfor", "do", "until", "function", "global", "persistent", "try", "catch", "end", "break", "continue", "and", "or", "not", "spmd", "classdef", "properties", "methods", "events", "arguments", "disp", "error", "warning", "fprintf", "sprintf", "printf", "assert", "validateattributes", }) _TRANSPOSE_AFTER = frozenset(")]}._'") def strip_literals(code: str) -> str: """Blank string and comment BODIES, keeping their markers and length. Offsets and line breaks are preserved so a match position still maps to the right line of the original source. Three things here are load-bearing, each because getting them wrong produced a real, verified defect: * **The comment marker survives.** Blanking from the `%`/`#` itself made the `hash-comment` and `block-comment-hash` rules unable to match anything, ever - two declared high-severity rules were dead code, and `#` comments are the most common Octave-ism there is. * **State resets at a newline, but not across a `...` continuation.** Carrying the previous line's last character over meant a quote at the start of a line was read as a transpose, so the string body leaked into the stripped text and every rule scanned English prose. That produced false "NOT VALID MATLAB" verdicts on ordinary code like `plot(x, y, ...` / `'--r', 'LineWidth', 2);`. A continuation is the one case where state must carry, because the logical line genuinely continues. * **`%{ ... %}` blocks are handled as blocks.** Treating the opener as a line comment left the prose body to be linted as code. """ out = list(code) n = len(code) i = 0 prev_char = "" # last significant char on this logical line word = "" # identifier immediately before the cursor def blank(a: int, b: int) -> None: for k in range(a, min(b, n)): if out[k] != "\n": out[k] = " " def line_end(pos: int) -> int: j = code.find("\n", pos) return n if j == -1 else j while i < n: c = code[i] if c == "\n": prev_char = "" word = "" i += 1 continue # Line continuation. Everything after `...` is a comment, and the # logical line continues - so prev_char/word deliberately persist. if c == "." and code.startswith("...", i): j = line_end(i) blank(i, j) i = j + 1 if j < n else n continue if c in "%#": marker = code[i:i + 2] stripped_line = code[code.rfind("\n", 0, i) + 1:line_end(i)].strip() # A block comment opener is the only thing on its line. if stripped_line in ("%{", "#{"): closer = "%}" if marker[0] == "%" else "#}" j = i + 2 while j < n: ls = j le = line_end(j) if code[ls:le].strip() in ("%}", "#}"): blank(i + 2, ls) # body only i = le break j = le + 1 else: blank(i + 2, n) i = n continue keep = 2 if (i + 1 < n and code[i + 1] in "{}") else 1 j = line_end(i) blank(i + keep, j) i = j # let the newline branch reset state continue if c == '"': j = i + 1 while j < n and code[j] != "\n": if code[j] == '"': if j + 1 < n and code[j + 1] == '"': j += 2 continue break j += 1 blank(i + 1, j) i = min(j + 1, n) prev_char = '"' word = "" continue if c == "'": is_transpose = ( (prev_char.isalnum() or prev_char in _TRANSPOSE_AFTER) and word not in _KEYWORDS_BEFORE_STRING ) if is_transpose: prev_char = "'" word = "" i += 1 continue j = i + 1 while j < n and code[j] != "\n": if code[j] == "'": if j + 1 < n and code[j + 1] == "'": j += 2 continue break j += 1 blank(i + 1, j) i = min(j + 1, n) prev_char = "'" word = "" continue if c.isalnum() or c == "_": word += c prev_char = c elif c.isspace(): pass # whitespace keeps `word` (for `case 'x'`) else: word = "" prev_char = c i += 1 return "".join(out) @dataclass(frozen=True) class Rule: id: str pattern: re.Pattern severity: str # "high" | "medium" | "low" kind: str # "octave_only" | "sandbox" | "semantic" message: str # When set, a finding is dropped if the captured name is assigned # anywhere in the script - a user variable called `input` or `stderr` # is not a call to the builtin, and firing on it teaches the model # that this whole section cries wolf. name_group: int = 0 # Match against the ORIGINAL source instead of the literal-stripped # copy. Only for rules whose whole subject is the content of a string # - a regexp pattern - which is blank by the time the other rules # run. The cost is that comments are visible too, so a raw rule can # fire on a line of prose; keep raw rules narrow enough that this is # rare and harmless. raw: bool = False def _r(pat: str) -> re.Pattern: return re.compile(pat, re.MULTILINE) OCTAVE_ONLY: tuple[Rule, ...] = ( Rule("hash-comment", _r(r"(?:^|[ \t;,])#(?![{}])"), "high", "octave_only", "`#` starts a comment in Octave and is a syntax error in MATLAB. Use `%`."), Rule("block-comment-hash", _r(r"^[ \t]*#[{}][ \t]*$"), "high", "octave_only", "`#{ ... #}` block comments are Octave-only. MATLAB uses `%{ ... %}`."), Rule("bang-ne", _r(r"!="), "high", "octave_only", "`!=` is Octave-only. MATLAB spells not-equal `~=`."), # NOT anchored at statement start: `!cmd` there is MATLAB's own shell # escape and is valid MATLAB, so flagging it as Octave-only would be # wrong. shell-escape covers that case separately. Rule("bang-not", _r(r"(?!\w])(?` for word start and end, or a lookaround. Measured here: " "`regexp('a b', '\\bb', 'match')` returns 0 matches.", 0, raw=True), # The weight flag is documented backwards more often than it is used # wrongly: a graded deliverable computed std(X,0,1), called it the # population standard deviation in the docstring, the code comment and # the usage notes, and told the user to swap to std(X,1,1) for N-1. # Every clause inverted. Both dialects agree on the flag; say it once. Rule("std-weight-flag", _r(r"(? set[str]: """Identifiers the script assigns to, so builtin-name rules can skip them.""" names = set() for m in _ASSIGNED.finditer(stripped): if m.group(1): names.add(m.group(1)) for m in re.finditer(r"^[ \t]*function\b[^\n]*?(?:=\s*)?([A-Za-z_]\w*)\s*\(", stripped, re.MULTILINE): names.add(m.group(1)) return names def lint(code: str) -> list[Finding]: """Run every rule against literal-stripped source, report original lines. A rule marked `raw` is matched against the original instead. Line numbers come out the same either way: strip_literals blanks in place, so both strings have identical length and identical newline offsets. """ stripped = strip_literals(code) lines = code.split("\n") assigned = _assigned_names(stripped) seen: set[tuple[str, int]] = set() out: list[Finding] = [] for rule in ALL_RULES: target = code if rule.raw else stripped for m in rule.pattern.finditer(target): if rule.name_group: try: name = m.group(rule.name_group) except IndexError: # pragma: no cover name = None if name and name in assigned: continue line_no = target.count("\n", 0, m.start()) + 1 key = (rule.id, line_no) if key in seen: continue seen.add(key) text = lines[line_no - 1].strip() if line_no <= len(lines) else "" out.append(Finding(rule, line_no, text[:120])) out.sort(key=lambda f: (f.line, f.rule.id)) return out def is_function_file(code: str) -> str | None: """Return the function name if the source is a function-definition file. A file whose first statement is `function` defines a function rather than running anything, in both MATLAB and Octave. Executing it looks like a bizarre "'a' undefined" error at the first use of a parameter, which sends the model hunting for a bug that is not there. """ stripped = strip_literals(code) for raw in stripped.split("\n"): line = raw.strip() if not line or line.startswith("%") or line.startswith("#"): continue m = re.match(r"^function\b(?:[^=\n]*=)?\s*([A-Za-z_]\w*)", line) return m.group(1) if m else None return None # -------------------------------------------------------------------------- # Script-local functions # # MATLAB (R2016b+) lets a script define its functions at the END and call # them from anywhere above; Octave defines a function when execution # reaches it, so the same script dies on "'foo' undefined". It is the one # discrepancy that turns correct, ordinary MATLAB into a failed call - it # cost m07 a whole tool call in the v3 round - and the model cannot fix it # without writing something that is no longer the code the user asked for. # # So the tail is lifted out and written as separate .m files beside the # script, which is where Octave will find them and how MATLAB behaves. The # lines they came from are replaced with blank lines rather than deleted: # an error two thirds of the way down still reports the line number the # model wrote, and a diff of the file still lines up. # # Everything here bails rather than guesses. A file it does not fully # understand is passed through untouched and fails the way it did before, # which is a bad outcome; mangling a file that was about to work is a # worse one. # -------------------------------------------------------------------------- _MAX_LOCAL_FUNCTIONS = 40 _MAX_NAME = 63 # Statement-level block openers and closers. Matched at a statement # boundary only - start of line, or after a `,`/`;` - so `x(end)` is not a # block end and `numel(c)` is not a `for`. _OPENERS = ("if", "for", "parfor", "while", "switch", "try", "do", "function", "classdef", "unwind_protect", "spmd") _CLOSERS = ("end", "endif", "endfor", "endparfor", "endwhile", "endswitch", "end_try_catch", "endfunction", "end_unwind_protect", "until", "endclassdef", "endspmd") _BLOCK_TOKEN = re.compile( r"(?:^|[;,])[ \t]*(" + "|".join(_OPENERS + _CLOSERS) + r")\b") _FUNCTION_LINE = re.compile(r"^[ \t]*function\b(.*)$") @dataclass(frozen=True) class LocalFunction: """One function lifted out of a script.""" name: str line: int # 1-based line it started on, so its file can be padded text: str # the block, verbatim @dataclass(frozen=True) class LocalFunctions: """The result of lifting a script's trailing function definitions out.""" script: str # same line count as the original functions: tuple[LocalFunction, ...] bailed: str | None = None # why nothing was lifted, for diagnosis def _function_name(signature: str) -> str | None: """`[a, b] = foo(x)` -> foo. Returns None for anything unexpected.""" # The signature arrives with string and comment bodies already blanked, # which also blanks a `...` continuation: a signature split across lines # leaves its first line readable, and one that continues before naming # the function (`function y = ...`) leaves nothing to read and is # refused by the name pattern below. sig = signature.split("%")[0].split("#")[0].strip() head = sig.split("(", 1)[0] if "=" in head: head = head.split("=", 1)[1] name = head.strip() if not re.fullmatch(r"[A-Za-z][A-Za-z0-9_]{0,%d}" % (_MAX_NAME - 1), name): return None if name in _KEYWORDS_BEFORE_STRING or name in ("main", "varargin", "varargout"): return None return name def split_local_functions(code: str) -> LocalFunctions: """Lift a script's trailing `function` blocks into separate files. Returns the script with those lines blanked and the functions in the order they were written. `bailed` is set, and nothing is lifted, when the file is not a plain script with a clean run of function definitions at the end of it. """ stripped = strip_literals(code) raw_lines = code.split("\n") lines = stripped.split("\n") stack: list[str] = [] first_fn: int | None = None starts: list[int] = [] for idx, line in enumerate(lines): body = line.split("%")[0].split("#")[0] depth_at_start = len(stack) top_level_function = False for match in _BLOCK_TOKEN.finditer(body): token = match.group(1) if token in _OPENERS: if token == "function": # Top level means: nothing open, or only the previous # unterminated function - Octave and MATLAB both allow # a function file whose functions have no `end`. if not stack or stack == ["function"]: top_level_function = True if stack == ["function"]: stack.pop() elif first_fn is not None: return LocalFunctions(code, (), "a nested function") if token == "classdef": return LocalFunctions(code, (), "a classdef file") stack.append(token) else: if not stack: return LocalFunctions(code, (), "an unbalanced end") stack.pop() if top_level_function: if first_fn is None: first_fn = idx starts.append(idx) elif first_fn is not None and depth_at_start == 0 and body.strip(): # A statement at the top level after the definitions started - # not a line of a function body, which sits at depth 1. MATLAB # does not allow this either, so leave the file alone and let # the run report whatever it reports. return LocalFunctions(code, (), "code after the function definitions") if first_fn is None: return LocalFunctions(code, (), None) if first_fn == 0 or not any(l.split("%")[0].split("#")[0].strip() for l in lines[:first_fn]): # Nothing above the first function: this is a function file, which # is handled - and explained to the model - elsewhere. return LocalFunctions(code, (), "a function file") if len(starts) > _MAX_LOCAL_FUNCTIONS: return LocalFunctions(code, (), f"more than {_MAX_LOCAL_FUNCTIONS} local functions") bounds = list(zip(starts, starts[1:] + [len(lines)])) functions: list[LocalFunction] = [] seen: set[str] = set() for start, stop in bounds: signature = _FUNCTION_LINE.match(lines[start]) name = _function_name(signature.group(1)) if signature else None if name is None: return LocalFunctions(code, (), "a function signature it could not read") if name in seen: return LocalFunctions(code, (), f"two functions named {name}") seen.add(name) functions.append(LocalFunction( name, start + 1, "\n".join(raw_lines[start:stop]).rstrip() + "\n")) script = raw_lines[:first_fn] + [""] * (len(raw_lines) - first_fn) return LocalFunctions("\n".join(script), tuple(functions), None)