From f40b7bcda316aadcdc08f74c27a0f314508c20ed Mon Sep 17 00:00:00 2001 From: jeditobe1 <13409032+jeditobe1@users.noreply.github.com> Date: Tue, 7 Apr 2026 22:14:33 -0400 Subject: [PATCH 1/3] Add add_record_override() with proper REFR group hierarchy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit REFR and other placed records must live inside the correct cell/ worldspace group structure (WRLD > worldspace > cell block > persistent children) — and alongside parent WRLD/CELL records the engine expects in the hierarchy. The existing add_record() places everything in flat type-0 groups, which causes save-time crashes for REFR overrides. add_record_override(record, source_plugin) finds the group path to the record in the source plugin (collecting parent WRLD before type-1 groups and parent CELL before type-6/8/9 groups), then creates/finds matching groups in the destination and clones the parent records into place. Falls back to flat grouping if the path isn't found. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/esplib/plugin.py | 119 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/src/esplib/plugin.py b/src/esplib/plugin.py index 2ff5415..b3b8416 100644 --- a/src/esplib/plugin.py +++ b/src/esplib/plugin.py @@ -362,6 +362,125 @@ def add_record(self, record: Record, group_signature: Optional[str] = None) -> N self.modified = True + def add_record_override(self, record: Record, + source_plugin: 'Plugin') -> None: + """Add an override record, placing it in the same group hierarchy + as the source record in the source plugin. + + This is essential for REFR, ACHR, and other placed records that + must be inside the correct cell/worldspace group structure. + Falls back to flat add_record if the source group path can't + be found. + """ + # Find the group path to this record in the source plugin + path = self._find_group_path(source_plugin.groups, + record.form_id.value) + if not path: + # Fallback: use flat grouping + self.add_record(record) + return + + # Register in indexes (same as add_record) + if record.form_id.value == 0: + record.form_id = self.get_next_form_id() + if record.form_id.file_index == self._LOCAL_SENTINEL: + self._new_records.append(record) + if self._game_registry: + if record.version == 44: + record.version = self._RECORD_VERSIONS.get( + self._game_registry.game_id, 44) + if record.schema is None: + schema = self._game_registry.get(record.signature) + if schema is not None: + record.bind_schema(schema) + self.records.append(record) + self._form_id_index[record.form_id.value] = record + if record.editor_id: + self._editor_id_index[record.editor_id.lower()] = record + if record.signature not in self._signature_index: + self._signature_index[record.signature] = [] + self._signature_index[record.signature].append(record) + + # Walk/create the group hierarchy, adding parent records as needed + # (e.g., WRLD and CELL records that the engine requires alongside + # their child groups) + current_groups = self.groups + for step in path: + group_type = step['group_type'] + label = step['label'] + parent_record = step.get('parent_record') + + target = None + for g in current_groups: + if isinstance(g, GroupRecord) and \ + g.group_type == group_type and g.label == label: + target = g + break + + if target is None: + # Add parent record (WRLD, CELL) before its child group + # if one exists and hasn't been added yet + if parent_record is not None: + pr_fid = parent_record.form_id.value + already_added = any( + hasattr(r, 'form_id') and r.form_id.value == pr_fid + for r in current_groups) + if not already_added: + # Clone minimal override of the parent record + pr_copy = parent_record.copy() + current_groups.append(pr_copy) + + target = GroupRecord(group_type, label) + if current_groups is self.groups: + self.groups.append(target) + else: + current_groups.append(target) + current_groups = target.records + + # Place the record in the deepest group + current_groups.append(record) + self.modified = True + + @staticmethod + def _find_group_path(groups, target_form_id, + _path=None) -> list: + """Find the group hierarchy path to a record by FormID. + + Returns a list of dicts: + {'group_type': int, 'label': ..., 'parent_record': Record|None} + The parent_record is a non-group record that immediately precedes + a child group at the same level (e.g., WRLD before type-1 groups, + CELL before type-6/8/9 groups). These must be included in override + plugins for the engine to correctly process the group. + """ + if _path is None: + _path = [] + # Track non-group records at this level that may be parents + last_record = None + for g in groups: + if not isinstance(g, GroupRecord): + if hasattr(g, 'form_id'): + if g.form_id.value == target_form_id: + return _path # found the target + last_record = g + continue + # For child groups (type 1,6,8,9), the preceding record + # at this level is the parent (WRLD, CELL) + parent_rec = None + if g.group_type in (1, 6, 8, 9): + parent_rec = last_record + + new_path = _path + [{ + 'group_type': g.group_type, + 'label': g.label, + 'parent_record': parent_rec, + }] + result = Plugin._find_group_path( + g.records, target_form_id, new_path) + if result is not None and len(result) >= len(new_path): + return result + return [] + def remove_record(self, record: Record) -> bool: with self._lock: if record not in self.records: From aa23050bf03c719476d001d7f27a1da38891d545 Mon Sep 17 00:00:00 2001 From: jeditobe1 <13409032+jeditobe1@users.noreply.github.com> Date: Fri, 10 Apr 2026 19:36:35 -0400 Subject: [PATCH 2/3] Translate strings and remap FormIDs on parent-record clones in add_record_override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When add_record_override clones a parent CELL/WRLD record to host the override REFR's group hierarchy, copy.copy() brings every subrecord along verbatim — including FULL string IDs and FormID subrecords (LTMP, XLCN, XEZN, XOWN, etc.) that point into the source plugin's tables and master list. Skyrim does NOT field-merge CELL overrides, so the cloned CELL fully replaces vanilla and the engine reads stale bytes when resolving names and references. Symptoms before this fix: - In a non-localized destination the FULL bytes rendered as garbage windows-1252 text ("Xa") for any cascade walking into the cloned cell (REFR -> XTEL -> dest REFR -> parent CELL -> FULL). - In a localized destination the FULL ID resolved against our own empty/wrong table. - LTMP/XLCN/etc. with master indices pointing at the source plugin's self-index aliased to a different plugin after master-list reordering, half-loading the cell at runtime (skybox only) on any transition into the clone. Surfaced on Skyrim Randomizer Phase 2.7 door gating: LTMP on DLC1DimHollowCrypt01 was 0x02006AE9 (master 2 = Dawnguard self in Dawnguard.esm), should remap to master 1 in a destination pack with master list [Skyrim, Dawnguard, SkyrimRandomizer]. Fixes for the parent-clone path in add_record_override: 1) String localization: branch on destination localization. - Non-localized destination: call _delocalize_strings to resolve the source's string ID via the source plugin's strings table and write the resolved text as inline cp1252 bytes. Mirrors the existing copy_record delocalize step. - Localized destination: call new _localize_strings to resolve the source's string ID, allocate a fresh ID via StringTable.allocate_id (max+1, starting at 1), and write the new 4-byte ID + register the text in our table. The destination string_tables is lazy-initialized as a fresh StringTableManager if not already present. 2) FormID remapping: call _remap_subrecord_formids on the parent clone before localization fixups, and remap the cloned parent's own FormID through remap_formid (the override record itself was already getting this treatment further down; parent clones were missing it). 3) Extend _FORMID_SUBRECORD_SIGS_FALLBACK with CELL FormID subrecord signatures (LTMP, XLCN, XEZN, XCMO, XCAS, XCCM, XCIM, XCWT, XOWN). Without them, CELL records hitting the fallback path (no schema bound) still miss their FormID subrecords. Adds: - Plugin._LOCALIZED_STRING_TABLE_TYPE: signature -> table-type map. Verified against vanilla Skyrim.esm: FULL/SHRT/ITXT live in .STRINGS, DESC/NNAM in .DLSTRINGS. Note that MESG ITXT lives in .STRINGS not .ILSTRINGS despite the extension naming. - Plugin._localize_strings: inverse of _delocalize_strings. - StringTable.allocate_id: simple max(keys)+1 allocator, starts at 1. Verified end-to-end via test_door_lock_esplib.py fixture in the randomizer repo, both flavors: - Flavor A (non-localized destination): cloned BFB01 CELL FULL bytes become b'Bleak Falls Temple\x00'. - Flavor B (localized destination): cloned BFB01 CELL FULL bytes become a 4-byte string ID, sidecar contains the text at that ID. In-game (Skyrim SE): both flavors display "Bleak Falls Temple" correctly on the BFB main entrance prompt and on the deep interior REFR 0x37603 (BFB01 -> exterior reverse door) which was previously unfixable under any workaround. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/esplib/plugin.py | 120 +++++++++++++++++++++++++++++++++++++++++- src/esplib/strings.py | 4 ++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/src/esplib/plugin.py b/src/esplib/plugin.py index b3b8416..f989176 100644 --- a/src/esplib/plugin.py +++ b/src/esplib/plugin.py @@ -426,8 +426,52 @@ def add_record_override(self, record: Record, hasattr(r, 'form_id') and r.form_id.value == pr_fid for r in current_groups) if not already_added: - # Clone minimal override of the parent record pr_copy = parent_record.copy() + # Remap subrecord FormIDs on the cloned parent + # (LTMP, XLCN, XEZN, XCMO, XCAS, XCCM, XCIM, + # XCWT, XOWN on CELLs; WNAM/SNAM/ZNAM etc. on + # WRLDs). Without this, any FormID subrecord + # with a master index pointing at the source's + # self-index carries that raw index through to + # the destination, where it now aliases a + # different master. The engine half-loads the + # cell (skybox only) at runtime because e.g. + # its LTMP resolves against the wrong plugin. + if source_plugin is not None: + self._remap_subrecord_formids( + pr_copy, source_plugin) + # Translate cloned parent record's localized + # subrecords (e.g. CELL.FULL). Without this, a + # cloned parent CELL from a localized source + # plugin (Skyrim.esm) carries a 4-byte string + # ID that's meaningless in our destination, + # corrupting the engine's name-resolution + # cascade (door activation prompts, etc.). + # + # Branch on destination localization: + # - non-localized: delocalize -> inline text + # (mirrors copy_record's delocalize step) + # - localized: re-localize -> new local + # string ID + entry in our own string table + if source_plugin is not None: + if not self.is_localized: + ps = source_plugin.plugin_set + orig_norm = None + if ps is not None: + orig_norm = source_plugin.normalize_form_id( + parent_record.form_id) + self._delocalize_strings( + pr_copy, source_plugin, ps, + chain_form_id=orig_norm) + else: + self._localize_strings(pr_copy, source_plugin) + # Also remap its own FormID from the source's + # master ordering to ours (same treatment the + # override record itself receives below). + remapped_pr_fid = self.remap_formid( + pr_copy.form_id.value, source_plugin) + if remapped_pr_fid != pr_copy.form_id.value: + pr_copy.form_id = FormID(remapped_pr_fid) current_groups.append(pr_copy) target = GroupRecord(group_type, label) @@ -717,6 +761,19 @@ def remap_formid(self, form_id: int, # Subrecord signatures that contain localized string IDs _LOCALIZED_STRING_SIGS = {'FULL', 'SHRT', 'DESC', 'NNAM', 'ITXT'} + # Maps a localized subrecord signature to which of the three string + # tables it lives in. Verified against vanilla Skyrim.esm. Note that + # MESG ITXT lives in .STRINGS (not .ILSTRINGS, despite what the + # extension suggests). .ILSTRINGS is almost exclusively used for + # INFO RNAM (dialogue responses). + _LOCALIZED_STRING_TABLE_TYPE = { + 'FULL': 'strings', + 'SHRT': 'strings', + 'ITXT': 'strings', + 'DESC': 'dlstrings', + 'NNAM': 'dlstrings', + } + # Fallback set for records without a schema. _FORMID_SUBRECORD_SIGS_FALLBACK = frozenset({ 'LNAM', 'KWDA', 'RNAM', 'PNAM', 'DOFT', 'FTST', 'TPLT', @@ -725,6 +782,14 @@ def remap_formid(self, form_id: int, 'DFTF', 'MPAI', 'TIND', 'TINC', 'HEAD', 'NAM8', 'MODL', 'EITM', 'BAMT', 'BIDS', 'ETYP', 'NAM4', 'NAM5', 'YNAM', 'CRDT', 'EFID', 'ATKR', 'HCLF', 'DPLT', 'SOFT', + # CELL subrecord FormIDs (all 4-byte single refs). Without + # these, cloning a parent CELL for REFR override leaves its + # lighting/location/zone/music/acoustic/climate/image-space/ + # water references pointing at wrong master indices after + # master-list reordering, which causes the engine to half- + # load the cell (skybox only) at runtime. + 'LTMP', 'XLCN', 'XEZN', 'XCMO', 'XCAS', 'XCCM', 'XCIM', + 'XCWT', 'XOWN', }) def copy_record(self, record: 'Record', @@ -990,6 +1055,59 @@ def _delocalize_strings(self, record: 'Record', sr.data = bytearray(b'\x00') sr.modified = True + def _localize_strings(self, record: 'Record', + source: Optional['Plugin']) -> None: + """Convert source-plugin string IDs to fresh local string IDs. + + Inverse of _delocalize_strings. Used when copying records from a + localized source plugin into a localized destination plugin: the + source's 4-byte string IDs are meaningless in our string table, + so we resolve each one to text via the source's strings table, + allocate a new ID in our own table, and write the new ID into + the subrecord. + + Requires self.is_localized == True. Lazy-initializes + self.string_tables if not already set up. + """ + from .strings import StringTable, StringTableManager + if source is None or not source.is_localized: + return + if self.string_tables is None: + self.string_tables = StringTableManager() + mgr = self.string_tables + + for sr in record.subrecords: + if sr.signature not in self._LOCALIZED_STRING_SIGS: + continue + if sr.size != 4: + continue + src_sid = sr.get_uint32() + if src_sid == 0: + continue + text = source.resolve_string(src_sid) + if text is None: + continue + text = text.rstrip('\x00') + + # Pick the destination table for this signature + attr = self._LOCALIZED_STRING_TABLE_TYPE.get(sr.signature) + if attr is None: + continue + table = getattr(mgr, attr) + if table is None: + ttype = { + 'strings': StringTable.STRINGS, + 'dlstrings': StringTable.DLSTRINGS, + 'ilstrings': StringTable.ILSTRINGS, + }[attr] + table = StringTable(ttype) + setattr(mgr, attr, table) + + new_sid = table.allocate_id() + table.set(new_sid, text) + sr.data = bytearray(struct.pack(' Optional[str]: def set(self, string_id: int, value: str) -> None: self.strings[string_id] = value + def allocate_id(self) -> int: + """Return a fresh string ID (max existing + 1, starting at 1).""" + return (max(self.strings.keys()) + 1) if self.strings else 1 + def remove(self, string_id: int) -> bool: if string_id in self.strings: del self.strings[string_id] From 7f724b4e61827abdc5875c096b30ba239720b6d7 Mon Sep 17 00:00:00 2001 From: jeditobe1 <13409032+jeditobe1@users.noreply.github.com> Date: Fri, 10 Apr 2026 19:59:25 -0400 Subject: [PATCH 3/3] Add clone_for_override + remap record FormID in add_record_override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Plugin.clone_for_override(source_record, source_plugin) as a documented public alternative to caller-side hand-rolled REFR clones. It deep-copies the source record and remaps master indices in subrecord FormIDs (NAME, XTEL, KWDA, VMAD, etc.) from the source plugin's master ordering to ours, but leaves the record's own FormID in source-plugin space so add_record_override can use it to look up the source group hierarchy. Also updates add_record_override to remap the record's own FormID from source-plugin space to destination-plugin space after the group path lookup completes. This was previously a silent latent bug — when overriding a record from a non-Skyrim master into a plugin where that master's index differs (e.g. Dawnguard.esm at index 0 in Dawnguard.esm itself, but at index 1 in a multi-master content pack), the override's own FormID kept the source-side master index and ended up pointing into the wrong slot of the destination's master list. Skyrim.esm-only callers were never affected because index 0 maps to index 0 in any master list that includes it. Both changes are backward compatible: clone_for_override is a new method, and the FormID remap inside add_record_override is a no-op when source and destination master orderings already align (which is the only state existing callers exercised). Co-Authored-By: Claude Opus 4.6 (1M context) --- src/esplib/plugin.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/esplib/plugin.py b/src/esplib/plugin.py index f989176..2f1933e 100644 --- a/src/esplib/plugin.py +++ b/src/esplib/plugin.py @@ -362,6 +362,25 @@ def add_record(self, record: Record, group_signature: Optional[str] = None) -> N self.modified = True + def clone_for_override(self, source_record: Record, + source_plugin: 'Plugin') -> Record: + """Clone a record from another plugin in preparation for an override. + + Deep-copies the source record and remaps master indices in + subrecord FormIDs (NAME, XTEL, KWDA, VMAD, etc.) from the source + plugin's master ordering to ours. The record's own FormID is + left in source-plugin space — add_record_override remaps it + after using it to find the source group hierarchy. + + Does NOT add the cloned record to the plugin — the caller is + expected to do any further setup (e.g. attaching XLOC via + write_form_id) and then call add_record_override(clone, + source_plugin) to place it in the right group hierarchy. + """ + new_record = source_record.copy() + self._remap_subrecord_formids(new_record, source_plugin) + return new_record + def add_record_override(self, record: Record, source_plugin: 'Plugin') -> None: """Add an override record, placing it in the same group hierarchy @@ -372,7 +391,10 @@ def add_record_override(self, record: Record, Falls back to flat add_record if the source group path can't be found. """ - # Find the group path to this record in the source plugin + # Find the group path to this record in the source plugin. + # Path lookup uses the source-plugin FormID; remap to our master + # ordering happens after the lookup so callers don't have to + # juggle two FormIDs. path = self._find_group_path(source_plugin.groups, record.form_id.value) if not path: @@ -380,6 +402,15 @@ def add_record_override(self, record: Record, self.add_record(record) return + # Remap the record's own FormID from the source's master ordering + # to ours (e.g. Dawnguard.esm at index 0 in source -> index 1 + # here). Subrecord FormIDs are remapped separately by + # clone_for_override before this call. + if record.form_id.value != 0: + remapped = self.remap_formid(record.form_id.value, source_plugin) + if remapped != record.form_id.value: + record.form_id = FormID(remapped) + # Register in indexes (same as add_record) if record.form_id.value == 0: record.form_id = self.get_next_form_id()