From 9bd4ce7418926507b11bb72f96086689844cfafb Mon Sep 17 00:00:00 2001 From: Justin Holmes Date: Wed, 21 Jan 2026 20:25:00 +0000 Subject: [PATCH 01/10] Document correct Manzanita track number mapping for song IDs Song IDs correspond to track numbers on Tony Rice's 1979 Manzanita album: - Track 5: Nine Pound Hammer (Pushups) - Track 7: Blue Railroad Train (Squats) - Track 8: Ginseng Sullivan (Army Crawls) Updated example config and documentation to use correct mappings. --- extensions/BlueRailroadIntegration/README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/extensions/BlueRailroadIntegration/README.md b/extensions/BlueRailroadIntegration/README.md index d64b0ef..6cf4993 100644 --- a/extensions/BlueRailroadIntegration/README.md +++ b/extensions/BlueRailroadIntegration/README.md @@ -49,7 +49,7 @@ wiki editors to add/modify leaderboards without touching code. {{BlueRailroadLeaderboard |page=Blue Railroad Squats Leaderboard -|filter_song_id=5 +|filter_song_id=7 |description=Leaderboard for Squats (Blue Railroad Train) }} ``` @@ -65,13 +65,17 @@ wiki editors to add/modify leaderboards without touching code. | `description` | Description shown at top of leaderboard | | `sort` | Sort order: `count` (default), `newest`, `oldest` | -### Song IDs +### Song IDs (Manzanita Track Numbers) -| ID | Exercise | Song | -|----|----------|------| -| 5 | Squats | Blue Railroad Train | -| 6 | Pushups | Nine Pound Hammer | -| 10 | Army Crawls | Ginseng Sullivan | +Song IDs correspond to track numbers on Tony Rice's *Manzanita* album (1979): + +| Track # | Song | Exercise | +|---------|------|----------| +| 5 | Nine Pound Hammer | Pushups | +| 7 | Blue Railroad Train | Squats | +| 8 | Ginseng Sullivan | Army Crawls | + +The chain data generator should use these track numbers as the `songId` value. ## Running the Import From ed9d105b53c60885d961b2774f3bc70fb0f5eb09 Mon Sep 17 00:00:00 2001 From: Justin Holmes Date: Tue, 27 Jan 2026 19:36:04 +0000 Subject: [PATCH 02/10] Support Blue Railroad V2 token format in import script - Detect V2 vs V1 by presence of blockheight field - V2 uses blockheight instead of date - V2 uses videoHash instead of uri - Add contract_version field to token template - Add separate category for V2 tokens --- .../maintenance/importBlueRailroads.php | 106 ++++++++++++------ 1 file changed, 70 insertions(+), 36 deletions(-) diff --git a/extensions/BlueRailroadIntegration/maintenance/importBlueRailroads.php b/extensions/BlueRailroadIntegration/maintenance/importBlueRailroads.php index 8a86de3..636f1d8 100644 --- a/extensions/BlueRailroadIntegration/maintenance/importBlueRailroads.php +++ b/extensions/BlueRailroadIntegration/maintenance/importBlueRailroads.php @@ -542,39 +542,61 @@ private function buildTokenPageContent($tokenId, $token) { // Extract values, handling potential BigInt serialization and missing keys $id = isset($token['id']) ? (is_array($token['id']) ? $token['id'][0] : $token['id']) : $tokenId; $songId = isset($token['songId']) ? (is_array($token['songId']) ? $token['songId'][0] : $token['songId']) : ''; - $date = isset($token['date']) ? (is_array($token['date']) ? $token['date'][0] : $token['date']) : ''; $owner = $token['owner'] ?? ''; $ownerDisplay = $token['ownerDisplay'] ?? $owner; - $uri = $token['uri'] ?? ''; - // Determine URI type for template - $uriType = 'unknown'; - if (strpos($uri, 'ipfs://') === 0) { - $uriType = 'ipfs'; - } elseif (strpos($uri, 'https://') === 0) { - $uriType = 'https'; + // Detect V2 vs V1 by presence of blockheight (V2) vs date (V1) + $isV2 = isset($token['blockheight']); + + // V2 uses blockheight, V1 uses date + $blockheight = ''; + $date = ''; + $formattedDate = ''; + + if ($isV2) { + $blockheight = isset($token['blockheight']) ? (is_array($token['blockheight']) ? $token['blockheight'][0] : $token['blockheight']) : ''; + } else { + $date = isset($token['date']) ? (is_array($token['date']) ? $token['date'][0] : $token['date']) : ''; + + // Convert date to readable format + // Handles both YYYYMMDD format (8 digits) and Unix timestamps (10 digits) + $dateStr = (string)$date; + if (strlen($dateStr) === 8 && $dateStr[0] === '2') { + // YYYYMMDD format (e.g., 20260113) + $year = substr($dateStr, 0, 4); + $month = substr($dateStr, 4, 2); + $day = substr($dateStr, 6, 2); + $formattedDate = "$year-$month-$day"; + } elseif (strlen($dateStr) >= 10 && is_numeric($dateStr)) { + // Unix timestamp (e.g., 1705685808) + $timestamp = (int)$dateStr; + $formattedDate = date('Y-m-d', $timestamp); + } } - // Extract IPFS CID if applicable + // V2 uses videoHash, V1 uses uri + $uri = ''; + $videoHash = ''; + $uriType = 'unknown'; $ipfsCid = ''; - if ($uriType === 'ipfs') { - $ipfsCid = substr($uri, 7); - } - // Convert date to readable format - // Handles both YYYYMMDD format (8 digits) and Unix timestamps (10 digits) - $dateStr = (string)$date; - $formattedDate = ''; - if (strlen($dateStr) === 8 && $dateStr[0] === '2') { - // YYYYMMDD format (e.g., 20260113) - $year = substr($dateStr, 0, 4); - $month = substr($dateStr, 4, 2); - $day = substr($dateStr, 6, 2); - $formattedDate = "$year-$month-$day"; - } elseif (strlen($dateStr) >= 10 && is_numeric($dateStr)) { - // Unix timestamp (e.g., 1705685808) - $timestamp = (int)$dateStr; - $formattedDate = date('Y-m-d', $timestamp); + if ($isV2) { + $videoHash = $token['videoHash'] ?? ''; + // V2 videoHash is stored as bytes32 hex, can be used with IPFS + if (!empty($videoHash) && $videoHash !== '0x0000000000000000000000000000000000000000000000000000000000000000') { + $uriType = 'ipfs'; + // Remove 0x prefix for IPFS CID + $ipfsCid = strpos($videoHash, '0x') === 0 ? substr($videoHash, 2) : $videoHash; + $uri = "ipfs://$ipfsCid"; + } + } else { + $uri = $token['uri'] ?? ''; + if (strpos($uri, 'ipfs://') === 0) { + $uriType = 'ipfs'; + $ipfsCid = substr($uri, 7); + } elseif (strpos($uri, 'https://') === 0) { + $uriType = 'https'; + } } // Build page content using a template @@ -582,18 +604,30 @@ private function buildTokenPageContent($tokenId, $token) { "{{Blue Railroad Token", "|token_id=$id", "|song_id=$songId", - "|date=$formattedDate", - "|date_raw=$date", - "|owner=$owner", - "|owner_display=$ownerDisplay", - "|uri=$uri", - "|uri_type=$uriType", - "|ipfs_cid=$ipfsCid", - "}}", - "", - "[[Category:Blue Railroad Tokens]]", + "|contract_version=" . ($isV2 ? 'V2' : 'V1'), ]; + // Add version-specific fields + if ($isV2) { + $lines[] = "|blockheight=$blockheight"; + $lines[] = "|video_hash=$videoHash"; + } else { + $lines[] = "|date=$formattedDate"; + $lines[] = "|date_raw=$date"; + } + + $lines[] = "|owner=$owner"; + $lines[] = "|owner_display=$ownerDisplay"; + $lines[] = "|uri=$uri"; + $lines[] = "|uri_type=$uriType"; + $lines[] = "|ipfs_cid=$ipfsCid"; + $lines[] = "}}"; + $lines[] = ""; + $lines[] = "[[Category:Blue Railroad Tokens]]"; + if ($isV2) { + $lines[] = "[[Category:Blue Railroad V2 Tokens]]"; + } + return implode("\n", $lines); } } From 05dc3e42ade02d2202f471e27bcde0a52eb90017 Mon Sep 17 00:00:00 2001 From: Justin Holmes Date: Tue, 27 Jan 2026 20:42:17 +0000 Subject: [PATCH 03/10] Add date-to-blockheight datepicker as ResourceLoader module --- .../BlueRailroadIntegration/extension.json | 9 +++ .../resources/ext.bluerailroad.datepicker.js | 71 +++++++++++++++++++ .../BlueRailroadIntegration/src/Hooks.php | 19 +++++ 3 files changed, 99 insertions(+) create mode 100644 extensions/BlueRailroadIntegration/resources/ext.bluerailroad.datepicker.js create mode 100644 extensions/BlueRailroadIntegration/src/Hooks.php diff --git a/extensions/BlueRailroadIntegration/extension.json b/extensions/BlueRailroadIntegration/extension.json index d79e530..e25839a 100644 --- a/extensions/BlueRailroadIntegration/extension.json +++ b/extensions/BlueRailroadIntegration/extension.json @@ -18,5 +18,14 @@ "AutoloadNamespaces": { "MediaWiki\\Extension\\BlueRailroadIntegration\\": "src/" }, + "ResourceModules": { + "ext.bluerailroad.datepicker": { + "scripts": ["resources/ext.bluerailroad.datepicker.js"], + "targets": ["desktop", "mobile"] + } + }, + "Hooks": { + "BeforePageDisplay": "MediaWiki\\Extension\\BlueRailroadIntegration\\Hooks::onBeforePageDisplay" + }, "manifest_version": 2 } diff --git a/extensions/BlueRailroadIntegration/resources/ext.bluerailroad.datepicker.js b/extensions/BlueRailroadIntegration/resources/ext.bluerailroad.datepicker.js new file mode 100644 index 0000000..65a2865 --- /dev/null +++ b/extensions/BlueRailroadIntegration/resources/ext.bluerailroad.datepicker.js @@ -0,0 +1,71 @@ +/** + * Date to Block Height Converter for Blue Railroad submission forms + */ +(function() { + 'use strict'; + + var AVG_BLOCK_TIME = 12.12; // Post-merge average + + function getCurrentBlockFromFooter() { + var link = document.querySelector('a[href*="etherscan.io/block/"]'); + if (link) { + var match = link.href.match(/block\/(\d+)/); + if (match) return parseInt(match[1]); + } + return null; + } + + function dateToBlockHeight(targetDate, refBlock, refTimestamp) { + var targetTimestamp = targetDate.getTime() / 1000; + var blocksDiff = Math.round((refTimestamp - targetTimestamp) / AVG_BLOCK_TIME); + return refBlock - blocksDiff; + } + + function init() { + var blockInput = document.querySelector('input[name*="[block_height]"]'); + if (!blockInput) return; + + var container = document.createElement('div'); + container.style.cssText = 'margin-top:8px;'; + container.innerHTML = + '' + + '' + + '' + + ''; + + blockInput.parentNode.appendChild(container); + + var picker = document.getElementById('br-datepicker'); + var btn = document.getElementById('br-convert'); + var status = document.getElementById('br-status'); + + picker.value = new Date().toISOString().slice(0, 16); + + btn.onclick = function() { + var date = new Date(picker.value); + if (isNaN(date.getTime())) { + status.textContent = 'Invalid date'; + status.style.color = 'red'; + return; + } + + var block = getCurrentBlockFromFooter(); + if (!block) { + status.textContent = 'No reference block found'; + status.style.color = 'red'; + return; + } + + var est = dateToBlockHeight(date, block, Date.now() / 1000); + blockInput.value = est; + status.textContent = est > block ? 'Future!' : '~estimated'; + status.style.color = est > block ? 'orange' : 'green'; + }; + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', init); + } else { + init(); + } +})(); diff --git a/extensions/BlueRailroadIntegration/src/Hooks.php b/extensions/BlueRailroadIntegration/src/Hooks.php new file mode 100644 index 0000000..b924cae --- /dev/null +++ b/extensions/BlueRailroadIntegration/src/Hooks.php @@ -0,0 +1,19 @@ +getTitle(); + + // Load on FormEdit pages for Blue Railroad + if ($title && $title->isSpecial('FormEdit')) { + $out->addModules(['ext.bluerailroad.datepicker']); + } + } +} From 0853f50219df7e3df2cdaadaaebb7d2aebcea842 Mon Sep 17 00:00:00 2001 From: Justin Holmes Date: Tue, 27 Jan 2026 21:29:51 +0000 Subject: [PATCH 04/10] Add ResourceFileModulePaths to fix datepicker module loading MediaWiki ResourceLoader needs explicit paths to find module files. --- extensions/BlueRailroadIntegration/extension.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extensions/BlueRailroadIntegration/extension.json b/extensions/BlueRailroadIntegration/extension.json index e25839a..7b85a09 100644 --- a/extensions/BlueRailroadIntegration/extension.json +++ b/extensions/BlueRailroadIntegration/extension.json @@ -18,6 +18,10 @@ "AutoloadNamespaces": { "MediaWiki\\Extension\\BlueRailroadIntegration\\": "src/" }, + "ResourceFileModulePaths": { + "localBasePath": "", + "remoteExtPath": "BlueRailroadIntegration" + }, "ResourceModules": { "ext.bluerailroad.datepicker": { "scripts": ["resources/ext.bluerailroad.datepicker.js"], From 08ff6fa7b31c93fc8edb11c27e39ff243c159b19 Mon Sep 17 00:00:00 2001 From: Justin Holmes Date: Wed, 28 Jan 2026 05:59:43 +0000 Subject: [PATCH 05/10] Fix Blue Railroad Import bot looping issue Two bugs fixed: 1. Leaderboards were being generated inside the per-source loop, so with V1+V2 sources, each run would overwrite leaderboards twice (once with V1 data, then with V2 data), causing alternating edits. Now aggregates all tokens from all sources before generating leaderboards once. 2. Bot saved pages even when content was identical. Added check to skip saving if the existing page content matches the generated content exactly. --- .../maintenance/importBlueRailroads.php | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/extensions/BlueRailroadIntegration/maintenance/importBlueRailroads.php b/extensions/BlueRailroadIntegration/maintenance/importBlueRailroads.php index 636f1d8..9717e8c 100644 --- a/extensions/BlueRailroadIntegration/maintenance/importBlueRailroads.php +++ b/extensions/BlueRailroadIntegration/maintenance/importBlueRailroads.php @@ -62,6 +62,9 @@ public function execute() { $this->fatalError("Failed to parse JSON: " . json_last_error_msg()); } + // Aggregate all tokens from all sources for leaderboards + $allTokens = []; + // Process each source foreach ($config['sources'] as $source) { $chainDataKey = $source['chain_data_key'] ?? 'blueRailroads'; @@ -93,18 +96,23 @@ public function execute() { } else { $errors++; } + + // Add to aggregated tokens for leaderboards + // Use source-prefixed key to avoid collisions between V1 and V2 token IDs + $aggregateKey = $chainDataKey . '_' . $tokenId; + $allTokens[$aggregateKey] = $token; } $this->output("\nToken Import Summary:\n"); $this->output(" Imported: $imported\n"); $this->output(" Updated: $updated\n"); $this->output(" Errors: $errors\n"); + } - // Generate leaderboards from config - $this->output("\nGenerating leaderboards...\n"); - foreach ($config['leaderboards'] as $leaderboard) { - $this->generateLeaderboard($blueRailroads, $leaderboard, $dryRun); - } + // Generate leaderboards from aggregated tokens (outside source loop) + $this->output("\nGenerating leaderboards from " . count($allTokens) . " total tokens...\n"); + foreach ($config['leaderboards'] as $leaderboard) { + $this->generateLeaderboard($allTokens, $leaderboard, $dryRun); } } @@ -344,6 +352,20 @@ private function generateLeaderboard($blueRailroads, $leaderboardConfig, $dryRun return; } + // Check if content has actually changed + if ($exists) { + $services = MediaWiki\MediaWikiServices::getInstance(); + $revisionLookup = $services->getRevisionLookup(); + $currentRevision = $revisionLookup->getRevisionByTitle($title); + if ($currentRevision) { + $currentContent = $currentRevision->getContent('main'); + if ($currentContent && $currentContent->getText() === $content) { + $this->output(" No changes needed (content identical)\n"); + return; + } + } + } + // Save the page $page = MediaWiki\MediaWikiServices::getInstance()->getWikiPageFactory()->newFromTitle($title); $contentObj = ContentHandler::makeContent($content, $title); From 9aa39a39f7c0b26f2a0f602563e5bb10e9131051 Mon Sep 17 00:00:00 2001 From: jMyles Date: Thu, 29 Jan 2026 14:37:11 -0500 Subject: [PATCH 06/10] favicon --- favicon.ico | Bin 0 -> 7358 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 favicon.ico diff --git a/favicon.ico b/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..35f54493a6213f010b6df99712e863fd4de7012e GIT binary patch literal 7358 zcmeI1>2q7>b;j9$BGV83+DtobXEIaQlTPd=RjP~=FR?6J;=V89DpI86tzOc8iBo5p zEFQadE7y{3$+9I`mc$JJ0b*acfB*?DAT|;^h>Z*_T3q@&_mYSw9;>zz=To7VcP?Js zd*A0i=Q+~?IrL2JQlrmd;6+m?%rQsz4x`@`_Jm``)}PpygGFF zwVN7m;FH~*yTA08KXZ3?Z|%36|9aha_=4@R>$8=b<%&`N@sY*HZp(V=oi`pG^fj1P1%NJCs}I*2!gb>jzE)_2s3X_C|NOS8;+!s=$$y1-cK4jY_~G-$ z^3g@W)aoO^5F~~O``}?z>jFT6rSj3G?_IDxd{*O6WxonYFmDgW51kvY7+tO(hnam& zRA$sIAc3Ur*Be(r^}r?P_y23upIH0j#tkr&k<@q38>@5yq9a=+Kq@C}S)I{ji8Y#I z_2aVNXpZug{Rb{NfD}sScK%2{z^%>3vfn#zM7I)=v7mra0I2J-G|aGVw#FGv<``cE zp0b1BzkkVj>`hI)u>Oa$#1$gh;t$`opc&-4Pm6J-!5oE(21`r;SYqlZ?F&NVYLi9a zHJV}o!x>uM)#o>F{UKVy)UunmnlIaq4=-?=Rk~0$8dkutM6)HqXt5<)?Md0%QfynS zi6-;!5Q3Sn9M))x9y)LS(KXwx?d`i}X={7u--f4--kfW+#H+?aAgLn|!ivwK zG??RXr65VRIrhPvZ~zGmGTL1k5bnp$mf&>sQ-TCh4_%&o_&*0T>ziMSq2kt7>qnC{ z`XG5!4W3e@5R8?$C{#FPX>|ajV`5F+DvU|^ph)m2@q(gyqgW{LzWvttOV`cYUpQ)z z`IgiF_@EDQR_TI7s&-r=)hO8{Qh|_t7|ykJhHDKlos(I%9WG`v!jdB+kgTHl5-N$B zVTEFM&#QyBh42@l2-J(I+(TEVxjU>zg2k;VsZIXws~3KhUR)%O5l`uDN+$y zCbBXVFtVN4$D@Xi!w|WOO38Y(M7l|O-gj1aZrpw6ppl1`K>XOyJd0LogLUJpb;c+| zvfC1ae8fX$aed+?K~c8q(&|=?lu8jZq9wbiU4bNllh`Rlm5(gm^XqG|Y~c=3+q}Kq zdeei0)r^H|#-u3JnWFWkn0O98th$N~a+2zbB|It^G((t5lk9X(=9pxM!9A8733-!i zht8RPF=)R-?&}s~hz*e(ANAK5BK6d3<0|wj)|Y^#Hhbdvxon#&eZnDbeA-hu;YzFG z0g@2nNS~a}zp${;IhAd9%1=+tt#`OYODY&d$OXA9!ahE{`1ilL@p(NG{c!t;4`QqJ z!Agpzn&T$Q5NQ>!H$>FS*6TxOJo#2fiW_`pwg?_%Q_^#{E8R1lZ?i<7p5N?trn+32 zMnmY#Y?0%^FYz*iCsuEa(zK8--+kNs%IL&r9UJA?L&@5)Fp4dwp)x3jtduGYnP64x z0==F>i(Qe8BIX9u%JaSr@PKo`yV3703S%?5uBm*hE&k%d=98XW_f)>g96d9)(PEcM z$B^_liz$HzJHl1&`OWoYVdM8yM2GW&ahc<(;IucN_COU+m3Fz$UHDyEO=cV6XKww( zM4A;FjVmas*&OZn6b5FC-ILkgskLY4)`?n^C3~1Adw@MvecJr$C)*Llu+a#tp^@E21TIf?V4$9DB0Q zm25G`V61au?I};Lb1DZ~sFMPBkY&x5xbVaE$5-g?j^3F2;TuD!r38u2?Hx}HayX`Z z)L*SsPq9JMDt(U9X z2%9S_6v=iQ~-%VgQ1Ftd~ZUHiVX@F>X(BstsO8U6saat zazeNwru$$Hsy|;@MNE|rO~FI?AZkZ%%zo>Y8yg>Q5i)Y^+)Rv)rJhQMlJy~#A|X@a z6>H>N1rGvH7ZtTA`=4H}^i%0qu$2x|%2~;%F?qowMuf{5SB?!W{^RQ!=Td~UqF{9x z!??NxOYnph{YZPD)O2h)A>?q62QbtXRWJ&SeFs+OOVxz%DZi@CfMVU!xkmfkdOiWRliP4HgMYyaa>N1ggm*(QzUhtXP)r zm=Iw5I31)|j-ebQWw(QU=bALPc4^x{rPNk=WP>H1)E@lxRhpvlaE4DjK3{84}wbuXqT)xpNe%NA5aq*Kg#XiRLT8l#|sq|ozIdme6O1;Ro1IYofHS~Ov z6!RAQym@}@^R6>`r*kb~-lfsE^d!o&}`lE$)R_t&j$iAoNH`*+# z-4p3{;zFUJaCovfd6z5AeE&?L)tW%+b%t=CC#y<`dnd)77@Gh3k1tO9SAI3ILh|B) z6`Ezr7Eb{bcaG^WhbuHoPrB3K!JG#?1u(bS5>HJf`rTR3_DyGc9MXzBH=A#=#s)k& zn0#_}9hV;P6x*H2PGV<^_fKZHk<+sqUCupjjy4+X*;JLM_%IOBaY1)@gm#7n*FosL9@Bgt3h>&8M7JEg*^E=0TacR#zL zbuXNqR`}7O9N~wMRVpFtG)F2%1rN^@l^XwPcfQja?dE0;;TL_y+VSuUbH!dq3;`pF zRtl3PM!6?&O{Qp{r%*o@f*%6+w6}nVobnditg&uP(h~3UZqynCPf0%%df52um*2Pe z0)xI)^sMM-N$?Q0(WP!zs>`-YE$f>~*P5f{BMSr5xpwmkrK)Bu_=2xkV+=j(&38G( zW1pETkXfyk=zup*ay6M&s2Aw1cc$3sq8(b}6!sg&BY^KDA06o$U4T-8D{?yQ0Y>!m z(nWtfy^zWy9#Z4+;YGot4U`Z0sx=F>I^M!9)$06R&UAxzaloBx&@G>u$<~iYp7Bz@ z!e>yCeYH*>p=LcbTWB-{2fQ>SiF!l0cRJteiZ`1ujn&SH4DzFdk<y?uLUw`0iLJzXf*ESGB~#UA^}Q!xad03qOhTg}l9W2n~| zJz)w}YnJK^A$ohVsneP4u&!{yCq3Dw@j$yh(Qc1JH3`VQ*J_txticfKcjpApjFfn~ zN&*33nz}Y?oU8!}gO^v|dPDcxuoXYt+1=GH@E8&%4Idv`EY}9qtjCCsUY|XB-BUTT zNNaKYrmsooCxzRri8_5)3L^a5k|*p*@KkF3CtaylYZTenjRj9(HSR*sRKDGw0(trH zVjXFs@pB4oSmnqPI{+!LgtmV~Bu|+&}V>r|q4wbn*B5XK6h4_JWyw#})ezE>9vbVkDFzHtjN{1iApp zBtu{*K~#Zp4DYE_gv>Amo7#>Vq!yG0lO#yjWVXj$AQlf^o_uN0ncw`l)Tw@U=eM8S z{%~sP;Wssh-ZmpMf+f)>DQFiBWK)<3(wU(4a*Zfo#9!iHYK^>(=Ez!lV+A%%6jb1o zcr|`VGe!@_JFc#)4trpgZ2ByQ&&=!+P1{oBl@aG7XEp!wj`hej55N#GEsfInWtDeO z@+_|oX%~6$k$yvZ36}10GxG4zHJQU3>82`m0ix4$!~NdG@_>-D8R9M2h_?CkLT zIPMF*Y4i?!=s5g8dfsyLPU*;X&#@sN1y}SS$%|m*wW*(s z&0U!ePDPTLjZHcnF8qa9xg%VjLEv_4cQ;koSWe|!%d3X@$ly%iifiGL!*|i Date: Wed, 28 Jan 2026 07:02:15 +0000 Subject: [PATCH 07/10] Add Python Blue Railroad Import bot with tests Replaces the PHP maintenance script with a well-tested Python implementation. Key improvements: - Aggregates all tokens from all sources BEFORE generating leaderboards (fixes the bug where V1 and V2 sources would overwrite each other) - Compares content before saving - won't create edits if unchanged - 68 pytest tests including regression tests for the aggregation bug - Clean separation: models, chain_data, config_parser, leaderboard, wiki_client - DryRunClient for easy testing and safe debugging - CLI with --dry-run, --verbose options Usage: cd tools/blue-railroad-import pip install -e ".[dev]" pytest # run tests blue-railroad-import --chain-data /path/to/chainData.json --dry-run -v --- tools/blue-railroad-import/.gitignore | 7 + .../blue_railroad_import/__init__.py | 1 + .../blue_railroad_import/chain_data.py | 62 +++++ .../blue_railroad_import/cli.py | 111 ++++++++ .../blue_railroad_import/config_parser.py | 99 +++++++ .../blue_railroad_import/importer.py | 168 ++++++++++++ .../blue_railroad_import/leaderboard.py | 141 ++++++++++ .../blue_railroad_import/models.py | 114 ++++++++ .../blue_railroad_import/token_page.py | 37 +++ .../blue_railroad_import/wiki_client.py | 105 ++++++++ tools/blue-railroad-import/pyproject.toml | 25 ++ tools/blue-railroad-import/requirements.txt | 3 + tools/blue-railroad-import/tests/__init__.py | 1 + .../tests/test_chain_data.py | 137 ++++++++++ .../tests/test_config_parser.py | 141 ++++++++++ .../tests/test_importer.py | 249 ++++++++++++++++++ .../tests/test_leaderboard.py | 200 ++++++++++++++ .../blue-railroad-import/tests/test_models.py | 128 +++++++++ 18 files changed, 1729 insertions(+) create mode 100644 tools/blue-railroad-import/.gitignore create mode 100644 tools/blue-railroad-import/blue_railroad_import/__init__.py create mode 100644 tools/blue-railroad-import/blue_railroad_import/chain_data.py create mode 100644 tools/blue-railroad-import/blue_railroad_import/cli.py create mode 100644 tools/blue-railroad-import/blue_railroad_import/config_parser.py create mode 100644 tools/blue-railroad-import/blue_railroad_import/importer.py create mode 100644 tools/blue-railroad-import/blue_railroad_import/leaderboard.py create mode 100644 tools/blue-railroad-import/blue_railroad_import/models.py create mode 100644 tools/blue-railroad-import/blue_railroad_import/token_page.py create mode 100644 tools/blue-railroad-import/blue_railroad_import/wiki_client.py create mode 100644 tools/blue-railroad-import/pyproject.toml create mode 100644 tools/blue-railroad-import/requirements.txt create mode 100644 tools/blue-railroad-import/tests/__init__.py create mode 100644 tools/blue-railroad-import/tests/test_chain_data.py create mode 100644 tools/blue-railroad-import/tests/test_config_parser.py create mode 100644 tools/blue-railroad-import/tests/test_importer.py create mode 100644 tools/blue-railroad-import/tests/test_leaderboard.py create mode 100644 tools/blue-railroad-import/tests/test_models.py diff --git a/tools/blue-railroad-import/.gitignore b/tools/blue-railroad-import/.gitignore new file mode 100644 index 0000000..cba61eb --- /dev/null +++ b/tools/blue-railroad-import/.gitignore @@ -0,0 +1,7 @@ +.venv/ +__pycache__/ +*.pyc +*.egg-info/ +.pytest_cache/ +dist/ +build/ diff --git a/tools/blue-railroad-import/blue_railroad_import/__init__.py b/tools/blue-railroad-import/blue_railroad_import/__init__.py new file mode 100644 index 0000000..6d54471 --- /dev/null +++ b/tools/blue-railroad-import/blue_railroad_import/__init__.py @@ -0,0 +1 @@ +"""Blue Railroad Import Bot - Python implementation with tests.""" diff --git a/tools/blue-railroad-import/blue_railroad_import/chain_data.py b/tools/blue-railroad-import/blue_railroad_import/chain_data.py new file mode 100644 index 0000000..6338e94 --- /dev/null +++ b/tools/blue-railroad-import/blue_railroad_import/chain_data.py @@ -0,0 +1,62 @@ +"""Chain data reading and token parsing.""" + +import json +from pathlib import Path +from typing import Iterator + +from .models import Token, Source + + +def load_chain_data(path: Path) -> dict: + """Load chain data JSON from file.""" + with open(path) as f: + return json.load(f) + + +def parse_token(token_id: str, token_data: dict, source_key: str) -> Token: + """Parse a single token from chain data.""" + + def extract_value(data, key): + """Extract value, handling array format from BigInt serialization.""" + val = data.get(key) + if isinstance(val, list): + return val[0] if val else None + return val + + return Token( + token_id=token_id, + source_key=source_key, + owner=token_data.get('owner', ''), + owner_display=token_data.get('ownerDisplay', token_data.get('owner', '')), + song_id=str(extract_value(token_data, 'songId')) if extract_value(token_data, 'songId') else None, + date=extract_value(token_data, 'date'), + uri=token_data.get('uri'), + blockheight=extract_value(token_data, 'blockheight'), + video_hash=token_data.get('videoHash'), + ) + + +def iter_tokens_from_source(chain_data: dict, source: Source) -> Iterator[Token]: + """Iterate over tokens from a specific source in chain data.""" + source_data = chain_data.get(source.chain_data_key, {}) + + for token_id, token_data in source_data.items(): + yield parse_token(token_id, token_data, source.chain_data_key) + + +def aggregate_tokens_from_sources(chain_data: dict, sources: list[Source]) -> dict[str, Token]: + """ + Aggregate all tokens from all sources into a single dict. + + Keys are prefixed with source key to avoid collisions between + V1 and V2 tokens with the same ID. + """ + all_tokens = {} + + for source in sources: + for token in iter_tokens_from_source(chain_data, source): + # Use source-prefixed key to avoid collisions + aggregate_key = f"{token.source_key}_{token.token_id}" + all_tokens[aggregate_key] = token + + return all_tokens diff --git a/tools/blue-railroad-import/blue_railroad_import/cli.py b/tools/blue-railroad-import/blue_railroad_import/cli.py new file mode 100644 index 0000000..a5cea64 --- /dev/null +++ b/tools/blue-railroad-import/blue_railroad_import/cli.py @@ -0,0 +1,111 @@ +"""Command-line interface for the Blue Railroad import bot.""" + +import argparse +import sys +from pathlib import Path + +from .importer import BlueRailroadImporter +from .wiki_client import MWClientWrapper, DryRunClient + + +def main(): + parser = argparse.ArgumentParser( + description='Import Blue Railroad tokens from chain data to PickiPedia' + ) + + parser.add_argument( + '--chain-data', + type=Path, + required=True, + help='Path to chainData.json file', + ) + + parser.add_argument( + '--wiki-url', + default='https://pickipedia.xyz', + help='MediaWiki site URL (default: https://pickipedia.xyz)', + ) + + parser.add_argument( + '--username', + help='MediaWiki bot username', + ) + + parser.add_argument( + '--password', + help='MediaWiki bot password', + ) + + parser.add_argument( + '--config-page', + default='PickiPedia:BlueRailroadConfig', + help='Wiki page containing bot configuration', + ) + + parser.add_argument( + '--dry-run', + action='store_true', + help='Show what would be done without making changes', + ) + + parser.add_argument( + '-v', '--verbose', + action='store_true', + help='Enable verbose output', + ) + + args = parser.parse_args() + + # Validate chain data exists + if not args.chain_data.exists(): + print(f"Error: Chain data file not found: {args.chain_data}", file=sys.stderr) + sys.exit(1) + + # Create wiki client + if args.dry_run: + print("DRY RUN MODE - no changes will be made\n") + wiki_client = DryRunClient() + else: + if not args.username or not args.password: + print("Error: --username and --password required unless --dry-run", file=sys.stderr) + sys.exit(1) + + try: + wiki_client = MWClientWrapper(args.wiki_url, args.username, args.password) + except Exception as e: + print(f"Error connecting to wiki: {e}", file=sys.stderr) + sys.exit(1) + + # Run import + importer = BlueRailroadImporter( + wiki_client=wiki_client, + chain_data_path=args.chain_data, + config_page=args.config_page, + verbose=args.verbose or args.dry_run, + ) + + try: + stats = importer.run() + + # Print final summary + print("\n" + "=" * 50) + print("IMPORT COMPLETE") + print("=" * 50) + print(f"Tokens: {stats.tokens_created} created, {stats.tokens_updated} updated, " + f"{stats.tokens_unchanged} unchanged, {stats.tokens_error} errors") + print(f"Leaderboards: {stats.leaderboards_created} created, {stats.leaderboards_updated} updated, " + f"{stats.leaderboards_unchanged} unchanged, {stats.leaderboards_error} errors") + + if stats.errors: + print("\nErrors:") + for error in stats.errors: + print(f" - {error}") + sys.exit(1) + + except Exception as e: + print(f"\nFatal error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/blue-railroad-import/blue_railroad_import/config_parser.py b/tools/blue-railroad-import/blue_railroad_import/config_parser.py new file mode 100644 index 0000000..de7fcdf --- /dev/null +++ b/tools/blue-railroad-import/blue_railroad_import/config_parser.py @@ -0,0 +1,99 @@ +"""Parse bot configuration from wiki page content.""" + +import re +from typing import Optional + +from .models import BotConfig, Source, LeaderboardConfig + + +def parse_template_params(param_str: str) -> dict[str, str]: + """Parse pipe-separated template parameters.""" + params = {} + parts = param_str.split('|') + + for part in parts: + if '=' in part: + key, value = part.split('=', 1) + params[key.strip()] = value.strip() + + return params + + +def strip_pre_blocks(text: str) -> str: + """Remove content inside
 tags to avoid matching example templates."""
+    return re.sub(r'
.*?
', '', text, flags=re.DOTALL) + + +def parse_config_from_wikitext(wikitext: str) -> Optional[BotConfig]: + """ + Parse bot configuration from wiki page wikitext. + + Looks for {{BlueRailroadSource|...}} and {{BlueRailroadLeaderboard|...}} templates. + """ + # Strip pre blocks to avoid matching documentation examples + text = strip_pre_blocks(wikitext) + + config = BotConfig() + + # Parse {{BlueRailroadSource|...}} templates + source_pattern = r'\{\{BlueRailroadSource\s*\n?((?:[^{}]|\{[^{]|\}[^}])*)\}\}' + for match in re.finditer(source_pattern, text, re.DOTALL): + params = parse_template_params(match.group(1)) + if params: + config.sources.append(Source( + name=params.get('name', params.get('chain_data_key', 'Unknown')), + chain_data_key=params.get('chain_data_key', 'blueRailroads'), + network_id=params.get('network_id', '10'), + contract=params.get('contract', ''), + )) + + # Parse {{BlueRailroadLeaderboard|...}} templates + leaderboard_pattern = r'\{\{BlueRailroadLeaderboard\s*\n?((?:[^{}]|\{[^{]|\}[^}])*)\}\}' + for match in re.finditer(leaderboard_pattern, text, re.DOTALL): + params = parse_template_params(match.group(1)) + if params.get('page'): + config.leaderboards.append(LeaderboardConfig( + page=params['page'], + title=params.get('title', ''), + description=params.get('description', ''), + filter_song_id=params.get('filter_song_id') or None, + filter_owner=params.get('filter_owner') or None, + sort=params.get('sort', 'count'), + )) + + # Return None if no config found (triggers default) + if not config.sources and not config.leaderboards: + return None + + # Ensure at least one source if leaderboards defined + if not config.sources: + config.sources.append(Source( + name='Blue Railroad (Optimism)', + chain_data_key='blueRailroads', + network_id='10', + contract='0xCe09A2d0d0BDE635722D8EF31901b430E651dB52', + )) + + return config + + +def get_default_config() -> BotConfig: + """Return default configuration when wiki page is unavailable.""" + return BotConfig( + sources=[ + Source( + name='Blue Railroad (Optimism)', + chain_data_key='blueRailroads', + network_id='10', + contract='0xCe09A2d0d0BDE635722D8EF31901b430E651dB52', + ), + ], + leaderboards=[ + LeaderboardConfig( + page='Blue Railroad Leaderboard', + title='Blue Railroad Leaderboard', + description='Overall token holdings across all exercises', + sort='count', + ), + ], + ) diff --git a/tools/blue-railroad-import/blue_railroad_import/importer.py b/tools/blue-railroad-import/blue_railroad_import/importer.py new file mode 100644 index 0000000..5e9dae0 --- /dev/null +++ b/tools/blue-railroad-import/blue_railroad_import/importer.py @@ -0,0 +1,168 @@ +"""Main import orchestration.""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +from .models import BotConfig, Token +from .chain_data import load_chain_data, aggregate_tokens_from_sources +from .config_parser import parse_config_from_wikitext, get_default_config +from .leaderboard import generate_leaderboard_content +from .token_page import generate_token_page_content +from .wiki_client import WikiClientProtocol, SaveResult + + +CONFIG_PAGE = 'PickiPedia:BlueRailroadConfig' + + +@dataclass +class ImportStats: + """Statistics from an import run.""" + tokens_created: int = 0 + tokens_updated: int = 0 + tokens_unchanged: int = 0 + tokens_error: int = 0 + leaderboards_created: int = 0 + leaderboards_updated: int = 0 + leaderboards_unchanged: int = 0 + leaderboards_error: int = 0 + errors: list[str] = field(default_factory=list) + + def add_token_result(self, result: SaveResult): + if result.action == 'created': + self.tokens_created += 1 + elif result.action == 'updated': + self.tokens_updated += 1 + elif result.action == 'unchanged': + self.tokens_unchanged += 1 + elif result.action == 'error': + self.tokens_error += 1 + self.errors.append(f"Token {result.page_title}: {result.message}") + + def add_leaderboard_result(self, result: SaveResult): + if result.action == 'created': + self.leaderboards_created += 1 + elif result.action == 'updated': + self.leaderboards_updated += 1 + elif result.action == 'unchanged': + self.leaderboards_unchanged += 1 + elif result.action == 'error': + self.leaderboards_error += 1 + self.errors.append(f"Leaderboard {result.page_title}: {result.message}") + + +class BlueRailroadImporter: + """Main importer class that orchestrates the import process.""" + + def __init__( + self, + wiki_client: WikiClientProtocol, + chain_data_path: Path, + config_page: str = CONFIG_PAGE, + verbose: bool = False, + ): + self.wiki = wiki_client + self.chain_data_path = chain_data_path + self.config_page = config_page + self.verbose = verbose + + def log(self, message: str): + """Log a message if verbose mode is enabled.""" + if self.verbose: + print(message) + + def load_config(self) -> BotConfig: + """Load configuration from wiki page or use defaults.""" + self.log(f"Loading config from: {self.config_page}") + + wiki_content = self.wiki.get_page_content(self.config_page) + if wiki_content: + config = parse_config_from_wikitext(wiki_content) + if config: + self.log(f" Found {len(config.sources)} source(s)") + self.log(f" Found {len(config.leaderboards)} leaderboard(s)") + return config + + self.log(" Using default configuration") + return get_default_config() + + def load_tokens(self, config: BotConfig) -> dict[str, Token]: + """Load and aggregate all tokens from chain data.""" + self.log(f"Loading chain data from: {self.chain_data_path}") + + chain_data = load_chain_data(self.chain_data_path) + tokens = aggregate_tokens_from_sources(chain_data, config.sources) + + self.log(f" Loaded {len(tokens)} total tokens from {len(config.sources)} source(s)") + return tokens + + def import_token(self, token: Token) -> SaveResult: + """Import a single token to the wiki.""" + page_title = f"Blue Railroad Token {token.token_id}" + content = generate_token_page_content(token) + + summary = f"{'Updated' if self.wiki.page_exists(page_title) else 'Imported'} Blue Railroad token #{token.token_id} from chain data" + + return self.wiki.save_page(page_title, content, summary) + + def generate_leaderboard( + self, + tokens: dict[str, Token], + config, # LeaderboardConfig + ) -> SaveResult: + """Generate a leaderboard page.""" + content = generate_leaderboard_content(tokens, config) + + summary = "Updated leaderboard from chain data" + if config.filter_song_id: + summary += f" (song_id={config.filter_song_id})" + + return self.wiki.save_page(config.page, content, summary) + + def run(self) -> ImportStats: + """Run the full import process.""" + stats = ImportStats() + + # Load config + config = self.load_config() + + # Load all tokens (aggregated from all sources) + all_tokens = self.load_tokens(config) + + # Import individual token pages + self.log("\nImporting token pages...") + for key, token in all_tokens.items(): + result = self.import_token(token) + stats.add_token_result(result) + + if result.action in ('created', 'updated'): + self.log(f" {result.action.capitalize()}: Blue Railroad Token {token.token_id}") + elif result.action == 'error': + self.log(f" ERROR: Blue Railroad Token {token.token_id}: {result.message}") + + self.log(f"\nToken import summary:") + self.log(f" Created: {stats.tokens_created}") + self.log(f" Updated: {stats.tokens_updated}") + self.log(f" Unchanged: {stats.tokens_unchanged}") + self.log(f" Errors: {stats.tokens_error}") + + # Generate leaderboards (using ALL aggregated tokens) + self.log(f"\nGenerating leaderboards from {len(all_tokens)} total tokens...") + for lb_config in config.leaderboards: + result = self.generate_leaderboard(all_tokens, lb_config) + stats.add_leaderboard_result(result) + + if result.action in ('created', 'updated'): + self.log(f" {result.action.capitalize()}: {lb_config.page}") + elif result.action == 'unchanged': + self.log(f" Unchanged: {lb_config.page}") + elif result.action == 'error': + self.log(f" ERROR: {lb_config.page}: {result.message}") + + self.log(f"\nLeaderboard summary:") + self.log(f" Created: {stats.leaderboards_created}") + self.log(f" Updated: {stats.leaderboards_updated}") + self.log(f" Unchanged: {stats.leaderboards_unchanged}") + self.log(f" Errors: {stats.leaderboards_error}") + + return stats diff --git a/tools/blue-railroad-import/blue_railroad_import/leaderboard.py b/tools/blue-railroad-import/blue_railroad_import/leaderboard.py new file mode 100644 index 0000000..753c553 --- /dev/null +++ b/tools/blue-railroad-import/blue_railroad_import/leaderboard.py @@ -0,0 +1,141 @@ +"""Leaderboard generation from aggregated token data.""" + +from typing import Optional + +from .models import Token, LeaderboardConfig, OwnerStats + + +# Map song IDs to exercise names +EXERCISE_MAP = { + '5': 'Squats ([[Blue Railroad Train]])', + '6': 'Pushups ([[Nine Pound Hammer]])', + '7': 'Squats ([[Blue Railroad Train]]) (legacy)', + '10': 'Army Crawls ([[Ginseng Sullivan]])', +} + + +def filter_tokens( + tokens: dict[str, Token], + filter_song_id: Optional[str] = None, + filter_owner: Optional[str] = None, +) -> dict[str, Token]: + """Filter tokens by song ID and/or owner.""" + result = {} + + for key, token in tokens.items(): + # Apply song filter + if filter_song_id: + if token.song_id != filter_song_id: + continue + + # Apply owner filter + if filter_owner: + if token.owner.lower() != filter_owner.lower(): + continue + + result[key] = token + + return result + + +def calculate_owner_stats(tokens: dict[str, Token]) -> dict[str, OwnerStats]: + """Calculate aggregated statistics per owner.""" + stats: dict[str, OwnerStats] = {} + + for key, token in tokens.items(): + if not token.owner: + continue + + owner_addr = token.owner + + if owner_addr not in stats: + stats[owner_addr] = OwnerStats( + address=owner_addr, + display_name=token.owner_display, + ) + + # Use date or blockheight for sorting + date_val = token.date if token.date else (token.blockheight or 0) + stats[owner_addr].add_token(token.token_id, date_val) + + return stats + + +def sort_owners(stats: dict[str, OwnerStats], sort_by: str) -> list[str]: + """Sort owner addresses by specified criteria.""" + if sort_by == 'newest': + return sorted(stats.keys(), key=lambda a: stats[a].newest_date, reverse=True) + elif sort_by == 'oldest': + return sorted(stats.keys(), key=lambda a: stats[a].oldest_date or float('inf')) + else: # 'count' (default) + return sorted(stats.keys(), key=lambda a: stats[a].token_count, reverse=True) + + +def generate_leaderboard_content( + tokens: dict[str, Token], + config: LeaderboardConfig, +) -> str: + """Generate wikitext content for a leaderboard page.""" + + # Filter tokens + filtered = filter_tokens( + tokens, + filter_song_id=config.filter_song_id, + filter_owner=config.filter_owner, + ) + + # Calculate stats + owner_stats = calculate_owner_stats(filtered) + + # Sort owners + sorted_owners = sort_owners(owner_stats, config.sort) + + # Build page content + lines = [ + f"'''{config.title}''' tracks ownership of [[Blue Railroad]] NFT tokens.", + ] + + if config.description: + lines.append("") + lines.append(config.description) + + if config.filter_song_id: + exercise_name = EXERCISE_MAP.get(config.filter_song_id, f"Exercise ID {config.filter_song_id}") + lines.append("") + lines.append(f"'''Exercise:''' {exercise_name}") + + lines.extend([ + "", + "''This page is automatically generated. See [[PickiPedia:BlueRailroadConfig|bot configuration]] to modify.''", + "", + "== Statistics ==", + f"* '''Total Tokens:''' {len(filtered)}", + f"* '''Total Holders:''' {len(owner_stats)}", + "", + "== Leaderboard ==", + '{| class="wikitable sortable"', + "! Rank !! Holder !! Tokens !! Token IDs", + ]) + + for rank, owner_addr in enumerate(sorted_owners, 1): + stats = owner_stats[owner_addr] + + # Format token links + sorted_ids = sorted(stats.token_ids, key=lambda x: int(x) if x.isdigit() else 0) + token_links = [f"[[Blue Railroad Token {tid}|#{tid}]]" for tid in sorted_ids] + token_links_str = ", ".join(token_links) + + # Format holder (just display name for now, could add SMW lookup later) + holder_display = stats.display_name + + lines.append("|-") + lines.append(f"| {rank} || {holder_display} || {stats.token_count} || {token_links_str}") + + lines.extend([ + "|}", + "", + "[[Category:Blue Railroad]]", + "[[Category:Leaderboards]]", + ]) + + return "\n".join(lines) diff --git a/tools/blue-railroad-import/blue_railroad_import/models.py b/tools/blue-railroad-import/blue_railroad_import/models.py new file mode 100644 index 0000000..e7917a3 --- /dev/null +++ b/tools/blue-railroad-import/blue_railroad_import/models.py @@ -0,0 +1,114 @@ +"""Data models for Blue Railroad tokens and configuration.""" + +from dataclasses import dataclass, field +from typing import Optional +from datetime import datetime + + +@dataclass +class Token: + """A Blue Railroad token from chain data.""" + token_id: str + source_key: str # e.g., 'blueRailroads' or 'blueRailroadV2s' + owner: str + owner_display: str + song_id: Optional[str] = None + + # V1 fields + date: Optional[int] = None + uri: Optional[str] = None + + # V2 fields + blockheight: Optional[int] = None + video_hash: Optional[str] = None + + @property + def is_v2(self) -> bool: + return self.blockheight is not None + + @property + def formatted_date(self) -> Optional[str]: + """Convert date to YYYY-MM-DD format.""" + if self.date is None: + return None + + date_str = str(self.date) + + # YYYYMMDD format (8 digits starting with 2) + if len(date_str) == 8 and date_str[0] == '2': + return f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:8]}" + + # Unix timestamp (10+ digits) + if len(date_str) >= 10 and date_str.isdigit(): + try: + dt = datetime.fromtimestamp(int(date_str)) + return dt.strftime('%Y-%m-%d') + except (ValueError, OSError): + pass + + return None + + @property + def ipfs_cid(self) -> Optional[str]: + """Extract IPFS CID from uri or video_hash.""" + if self.is_v2: + if self.video_hash and self.video_hash != '0x' + '0' * 64: + # Remove 0x prefix + return self.video_hash[2:] if self.video_hash.startswith('0x') else self.video_hash + return None + else: + if self.uri and self.uri.startswith('ipfs://'): + return self.uri[7:] + return None + + +@dataclass +class Source: + """A chain data source configuration.""" + name: str + chain_data_key: str + network_id: str = '10' + contract: str = '' + + +@dataclass +class LeaderboardConfig: + """Configuration for a leaderboard page.""" + page: str + title: str = '' + description: str = '' + filter_song_id: Optional[str] = None + filter_owner: Optional[str] = None + sort: str = 'count' # 'count', 'newest', 'oldest' + + def __post_init__(self): + if not self.title: + self.title = self.page + + +@dataclass +class OwnerStats: + """Aggregated statistics for a token owner.""" + address: str + display_name: str + token_count: int = 0 + token_ids: list = field(default_factory=list) + newest_date: int = 0 + oldest_date: int = 0 + + def add_token(self, token_id: str, date: Optional[int]): + self.token_count += 1 + self.token_ids.append(token_id) + + if date: + if date > self.newest_date: + self.newest_date = date + if self.oldest_date == 0 or date < self.oldest_date: + self.oldest_date = date + + +@dataclass +class BotConfig: + """Complete bot configuration from wiki page.""" + sources: list[Source] = field(default_factory=list) + leaderboards: list[LeaderboardConfig] = field(default_factory=list) diff --git a/tools/blue-railroad-import/blue_railroad_import/token_page.py b/tools/blue-railroad-import/blue_railroad_import/token_page.py new file mode 100644 index 0000000..9b773a3 --- /dev/null +++ b/tools/blue-railroad-import/blue_railroad_import/token_page.py @@ -0,0 +1,37 @@ +"""Token page content generation.""" + +from .models import Token + + +def generate_token_page_content(token: Token) -> str: + """Generate wikitext content for a token page.""" + lines = [ + "{{Blue Railroad Token", + f"|token_id={token.token_id}", + f"|song_id={token.song_id or ''}", + f"|contract_version={'V2' if token.is_v2 else 'V1'}", + ] + + # Version-specific fields + if token.is_v2: + lines.append(f"|blockheight={token.blockheight or ''}") + lines.append(f"|video_hash={token.video_hash or ''}") + else: + lines.append(f"|date={token.formatted_date or ''}") + lines.append(f"|date_raw={token.date or ''}") + + lines.extend([ + f"|owner={token.owner}", + f"|owner_display={token.owner_display}", + f"|uri={token.uri or ''}", + f"|uri_type={'ipfs' if token.ipfs_cid else 'unknown'}", + f"|ipfs_cid={token.ipfs_cid or ''}", + "}}", + "", + "[[Category:Blue Railroad Tokens]]", + ]) + + if token.is_v2: + lines.append("[[Category:Blue Railroad V2 Tokens]]") + + return "\n".join(lines) diff --git a/tools/blue-railroad-import/blue_railroad_import/wiki_client.py b/tools/blue-railroad-import/blue_railroad_import/wiki_client.py new file mode 100644 index 0000000..fcf9486 --- /dev/null +++ b/tools/blue-railroad-import/blue_railroad_import/wiki_client.py @@ -0,0 +1,105 @@ +"""Wiki client wrapper for MediaWiki API operations.""" + +from dataclasses import dataclass +from typing import Optional, Protocol +import mwclient + + +class WikiClientProtocol(Protocol): + """Protocol for wiki client operations (for testing).""" + + def get_page_content(self, title: str) -> Optional[str]: + """Get the current content of a page, or None if it doesn't exist.""" + ... + + def save_page(self, title: str, content: str, summary: str) -> bool: + """Save content to a page. Returns True if saved, False if unchanged.""" + ... + + def page_exists(self, title: str) -> bool: + """Check if a page exists.""" + ... + + +@dataclass +class SaveResult: + """Result of a page save operation.""" + page_title: str + action: str # 'created', 'updated', 'unchanged', 'error' + message: str = '' + + +class MWClientWrapper: + """Wrapper around mwclient for wiki operations.""" + + def __init__(self, site_url: str, username: str, password: str): + # Parse site URL - mwclient wants host without protocol + if site_url.startswith('https://'): + host = site_url[8:] + scheme = 'https' + elif site_url.startswith('http://'): + host = site_url[7:] + scheme = 'http' + else: + host = site_url + scheme = 'https' + + # Remove trailing slash + host = host.rstrip('/') + + self.site = mwclient.Site(host, scheme=scheme) + self.site.login(username, password) + + def get_page_content(self, title: str) -> Optional[str]: + """Get the current content of a page, or None if it doesn't exist.""" + page = self.site.pages[title] + if page.exists: + return page.text() + return None + + def save_page(self, title: str, content: str, summary: str) -> SaveResult: + """Save content to a page. Checks if content changed first.""" + page = self.site.pages[title] + existed = page.exists + current_content = page.text() if existed else None + + # Skip if content unchanged + if current_content == content: + return SaveResult(title, 'unchanged', 'Content identical') + + try: + page.save(content, summary=summary) + action = 'updated' if existed else 'created' + return SaveResult(title, action) + except Exception as e: + return SaveResult(title, 'error', str(e)) + + def page_exists(self, title: str) -> bool: + """Check if a page exists.""" + return self.site.pages[title].exists + + +class DryRunClient: + """Mock client for dry-run mode that doesn't make any changes.""" + + def __init__(self, existing_pages: Optional[dict[str, str]] = None): + self.existing_pages = existing_pages or {} + self.would_save: list[tuple[str, str, str]] = [] + + def get_page_content(self, title: str) -> Optional[str]: + return self.existing_pages.get(title) + + def save_page(self, title: str, content: str, summary: str) -> SaveResult: + self.would_save.append((title, content, summary)) + + existed = title in self.existing_pages + current = self.existing_pages.get(title) + + if current == content: + return SaveResult(title, 'unchanged', 'Would be unchanged') + + action = 'updated' if existed else 'created' + return SaveResult(title, action, f'Would be {action} (dry run)') + + def page_exists(self, title: str) -> bool: + return title in self.existing_pages diff --git a/tools/blue-railroad-import/pyproject.toml b/tools/blue-railroad-import/pyproject.toml new file mode 100644 index 0000000..2487c6a --- /dev/null +++ b/tools/blue-railroad-import/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools>=45", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "blue-railroad-import" +version = "1.0.0" +description = "Import Blue Railroad tokens from chain data to PickiPedia" +requires-python = ">=3.10" +dependencies = [ + "mwclient>=0.10.1", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-mock>=3.10.0", +] + +[project.scripts] +blue-railroad-import = "blue_railroad_import.cli:main" + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = "test_*.py" diff --git a/tools/blue-railroad-import/requirements.txt b/tools/blue-railroad-import/requirements.txt new file mode 100644 index 0000000..22cd4fb --- /dev/null +++ b/tools/blue-railroad-import/requirements.txt @@ -0,0 +1,3 @@ +mwclient>=0.10.1 +pytest>=7.0.0 +pytest-mock>=3.10.0 diff --git a/tools/blue-railroad-import/tests/__init__.py b/tools/blue-railroad-import/tests/__init__.py new file mode 100644 index 0000000..5b8ae23 --- /dev/null +++ b/tools/blue-railroad-import/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for Blue Railroad Import Bot.""" diff --git a/tools/blue-railroad-import/tests/test_chain_data.py b/tools/blue-railroad-import/tests/test_chain_data.py new file mode 100644 index 0000000..06cb4ba --- /dev/null +++ b/tools/blue-railroad-import/tests/test_chain_data.py @@ -0,0 +1,137 @@ +"""Tests for chain data reading.""" + +import pytest +from blue_railroad_import.models import Source +from blue_railroad_import.chain_data import ( + parse_token, + iter_tokens_from_source, + aggregate_tokens_from_sources, +) + + +class TestParseToken: + """Tests for parse_token function.""" + + def test_parses_basic_v1_token(self): + token_data = { + 'owner': '0x123', + 'ownerDisplay': 'alice.eth', + 'songId': '5', + 'date': 20260113, + 'uri': 'ipfs://QmXyz', + } + token = parse_token('1', token_data, 'blueRailroads') + + assert token.token_id == '1' + assert token.source_key == 'blueRailroads' + assert token.owner == '0x123' + assert token.owner_display == 'alice.eth' + assert token.song_id == '5' + assert token.date == 20260113 + + def test_parses_v2_token(self): + token_data = { + 'owner': '0x456', + 'ownerDisplay': 'bob.eth', + 'songId': '5', + 'blockheight': 12345678, + 'videoHash': '0xabc123', + } + token = parse_token('5', token_data, 'blueRailroadV2s') + + assert token.is_v2 is True + assert token.blockheight == 12345678 + assert token.video_hash == '0xabc123' + + def test_handles_bigint_array_format(self): + """Chain data serializes BigInt as [value] arrays.""" + token_data = { + 'owner': '0x123', + 'songId': ['5'], # Array format from BigInt + 'date': [20260113], + 'blockheight': [12345678], + } + token = parse_token('1', token_data, 'blueRailroads') + + assert token.song_id == '5' + assert token.date == 20260113 + + def test_uses_owner_as_display_fallback(self): + token_data = { + 'owner': '0x123abc', + # No ownerDisplay + } + token = parse_token('1', token_data, 'blueRailroads') + + assert token.owner_display == '0x123abc' + + +class TestIterTokensFromSource: + """Tests for iter_tokens_from_source function.""" + + def test_iterates_over_tokens(self): + chain_data = { + 'blueRailroads': { + '1': {'owner': '0x111'}, + '2': {'owner': '0x222'}, + } + } + source = Source(name='V1', chain_data_key='blueRailroads') + + tokens = list(iter_tokens_from_source(chain_data, source)) + + assert len(tokens) == 2 + assert {t.token_id for t in tokens} == {'1', '2'} + + def test_returns_empty_for_missing_key(self): + chain_data = {'otherKey': {}} + source = Source(name='V1', chain_data_key='blueRailroads') + + tokens = list(iter_tokens_from_source(chain_data, source)) + + assert tokens == [] + + +class TestAggregateTokensFromSources: + """Tests for aggregate_tokens_from_sources function.""" + + def test_aggregates_from_multiple_sources(self): + chain_data = { + 'blueRailroads': { + '1': {'owner': '0x111'}, + }, + 'blueRailroadV2s': { + '5': {'owner': '0x555', 'blockheight': 123}, + }, + } + sources = [ + Source(name='V1', chain_data_key='blueRailroads'), + Source(name='V2', chain_data_key='blueRailroadV2s'), + ] + + tokens = aggregate_tokens_from_sources(chain_data, sources) + + assert len(tokens) == 2 + assert 'blueRailroads_1' in tokens + assert 'blueRailroadV2s_5' in tokens + + def test_prefixes_keys_to_avoid_collisions(self): + """V1 and V2 might have same token IDs, keys prevent collision.""" + chain_data = { + 'blueRailroads': { + '1': {'owner': '0x111'}, + }, + 'blueRailroadV2s': { + '1': {'owner': '0x222', 'blockheight': 123}, # Same ID! + }, + } + sources = [ + Source(name='V1', chain_data_key='blueRailroads'), + Source(name='V2', chain_data_key='blueRailroadV2s'), + ] + + tokens = aggregate_tokens_from_sources(chain_data, sources) + + assert len(tokens) == 2 # Both preserved, not overwritten + assert tokens['blueRailroads_1'].owner == '0x111' + assert tokens['blueRailroadV2s_1'].owner == '0x222' diff --git a/tools/blue-railroad-import/tests/test_config_parser.py b/tools/blue-railroad-import/tests/test_config_parser.py new file mode 100644 index 0000000..ec6d5af --- /dev/null +++ b/tools/blue-railroad-import/tests/test_config_parser.py @@ -0,0 +1,141 @@ +"""Tests for wiki config parsing.""" + +import pytest +from blue_railroad_import.config_parser import ( + parse_template_params, + strip_pre_blocks, + parse_config_from_wikitext, + get_default_config, +) + + +class TestParseTemplateParams: + """Tests for parse_template_params function.""" + + def test_parses_simple_params(self): + result = parse_template_params('page=Leaderboard|sort=count') + assert result == {'page': 'Leaderboard', 'sort': 'count'} + + def test_handles_whitespace(self): + result = parse_template_params(' page = Leaderboard | sort = count ') + assert result == {'page': 'Leaderboard', 'sort': 'count'} + + def test_handles_empty_string(self): + result = parse_template_params('') + assert result == {} + + def test_ignores_params_without_equals(self): + result = parse_template_params('page=Leaderboard|positional|sort=count') + assert result == {'page': 'Leaderboard', 'sort': 'count'} + + +class TestStripPreBlocks: + """Tests for strip_pre_blocks function.""" + + def test_removes_pre_content(self): + text = 'Before
inside
After' + assert strip_pre_blocks(text) == 'Before After' + + def test_removes_multiline_pre(self): + text = '''Before +
+line1
+line2
+
+After''' + result = strip_pre_blocks(text) + assert '
' not in result
+        assert 'line1' not in result
+        assert 'Before' in result
+        assert 'After' in result
+
+    def test_handles_no_pre_blocks(self):
+        text = 'No pre blocks here'
+        assert strip_pre_blocks(text) == text
+
+
+class TestParseConfigFromWikitext:
+    """Tests for parse_config_from_wikitext function."""
+
+    def test_parses_source_template(self):
+        wikitext = '''
+{{BlueRailroadSource
+|name=Blue Railroad V1
+|chain_data_key=blueRailroads
+|network_id=10
+|contract=0x123
+}}
+'''
+        config = parse_config_from_wikitext(wikitext)
+        assert len(config.sources) == 1
+        assert config.sources[0].name == 'Blue Railroad V1'
+        assert config.sources[0].chain_data_key == 'blueRailroads'
+
+    def test_parses_leaderboard_template(self):
+        wikitext = '''
+{{BlueRailroadLeaderboard
+|page=Blue Railroad Leaderboard
+|title=Overall Leaderboard
+|sort=count
+}}
+'''
+        config = parse_config_from_wikitext(wikitext)
+        assert len(config.leaderboards) == 1
+        assert config.leaderboards[0].page == 'Blue Railroad Leaderboard'
+        assert config.leaderboards[0].title == 'Overall Leaderboard'
+        assert config.leaderboards[0].sort == 'count'
+
+    def test_parses_leaderboard_with_filter(self):
+        wikitext = '''
+{{BlueRailroadLeaderboard
+|page=Squats Leaderboard
+|filter_song_id=5
+}}
+'''
+        config = parse_config_from_wikitext(wikitext)
+        assert config.leaderboards[0].filter_song_id == '5'
+
+    def test_parses_multiple_sources_and_leaderboards(self):
+        wikitext = '''
+{{BlueRailroadSource|name=V1|chain_data_key=blueRailroads}}
+{{BlueRailroadSource|name=V2|chain_data_key=blueRailroadV2s}}
+{{BlueRailroadLeaderboard|page=Overall}}
+{{BlueRailroadLeaderboard|page=Squats|filter_song_id=5}}
+'''
+        config = parse_config_from_wikitext(wikitext)
+        assert len(config.sources) == 2
+        assert len(config.leaderboards) == 2
+
+    def test_ignores_templates_in_pre_blocks(self):
+        wikitext = '''
+
+{{BlueRailroadSource|name=Example|chain_data_key=example}}
+
+{{BlueRailroadSource|name=Real|chain_data_key=real}} +''' + config = parse_config_from_wikitext(wikitext) + assert len(config.sources) == 1 + assert config.sources[0].name == 'Real' + + def test_returns_none_for_empty_config(self): + wikitext = 'Just some text, no templates' + assert parse_config_from_wikitext(wikitext) is None + + def test_adds_default_source_if_only_leaderboards(self): + wikitext = '{{BlueRailroadLeaderboard|page=Test}}' + config = parse_config_from_wikitext(wikitext) + assert len(config.sources) == 1 + assert config.sources[0].chain_data_key == 'blueRailroads' + + +class TestGetDefaultConfig: + """Tests for get_default_config function.""" + + def test_returns_valid_config(self): + config = get_default_config() + assert len(config.sources) >= 1 + assert len(config.leaderboards) >= 1 + + def test_default_source_is_v1(self): + config = get_default_config() + assert config.sources[0].chain_data_key == 'blueRailroads' diff --git a/tools/blue-railroad-import/tests/test_importer.py b/tools/blue-railroad-import/tests/test_importer.py new file mode 100644 index 0000000..ced4b0d --- /dev/null +++ b/tools/blue-railroad-import/tests/test_importer.py @@ -0,0 +1,249 @@ +"""Tests for the main importer.""" + +import json +import pytest +from pathlib import Path +from tempfile import NamedTemporaryFile + +from blue_railroad_import.importer import BlueRailroadImporter, ImportStats +from blue_railroad_import.wiki_client import DryRunClient, SaveResult + + +@pytest.fixture +def chain_data_file(tmp_path): + """Create a temporary chain data file.""" + data = { + 'blueRailroads': { + '1': { + 'owner': '0xAlice', + 'ownerDisplay': 'alice.eth', + 'songId': '5', + 'date': 20260113, + 'uri': 'ipfs://QmV1Token1', + }, + '2': { + 'owner': '0xBob', + 'ownerDisplay': 'bob.eth', + 'songId': '5', + 'date': 20260114, + 'uri': 'ipfs://QmV1Token2', + }, + }, + 'blueRailroadV2s': { + '5': { + 'owner': '0xAlice', + 'ownerDisplay': 'alice.eth', + 'songId': '5', + 'blockheight': 12345678, + 'videoHash': '0xabc123', + }, + }, + } + + file_path = tmp_path / 'chainData.json' + file_path.write_text(json.dumps(data)) + return file_path + + +@pytest.fixture +def wiki_config_content(): + """Sample wiki config page content.""" + return ''' +== Configuration == + +{{BlueRailroadSource +|name=Blue Railroad V1 +|chain_data_key=blueRailroads +}} + +{{BlueRailroadSource +|name=Blue Railroad V2 +|chain_data_key=blueRailroadV2s +}} + +{{BlueRailroadLeaderboard +|page=Blue Railroad Leaderboard +|title=Overall Leaderboard +|sort=count +}} + +{{BlueRailroadLeaderboard +|page=Blue Railroad Squats Leaderboard +|filter_song_id=5 +}} +''' + + +class TestImportStats: + """Tests for ImportStats tracking.""" + + def test_tracks_token_results(self): + stats = ImportStats() + stats.add_token_result(SaveResult('Token 1', 'created')) + stats.add_token_result(SaveResult('Token 2', 'updated')) + stats.add_token_result(SaveResult('Token 3', 'unchanged')) + stats.add_token_result(SaveResult('Token 4', 'error', 'failed')) + + assert stats.tokens_created == 1 + assert stats.tokens_updated == 1 + assert stats.tokens_unchanged == 1 + assert stats.tokens_error == 1 + assert len(stats.errors) == 1 + + def test_tracks_leaderboard_results(self): + stats = ImportStats() + stats.add_leaderboard_result(SaveResult('LB1', 'created')) + stats.add_leaderboard_result(SaveResult('LB2', 'unchanged')) + + assert stats.leaderboards_created == 1 + assert stats.leaderboards_unchanged == 1 + + +class TestBlueRailroadImporter: + """Tests for BlueRailroadImporter.""" + + def test_loads_config_from_wiki(self, chain_data_file, wiki_config_content): + wiki = DryRunClient(existing_pages={ + 'PickiPedia:BlueRailroadConfig': wiki_config_content, + }) + importer = BlueRailroadImporter(wiki, chain_data_file) + + config = importer.load_config() + + assert len(config.sources) == 2 + assert len(config.leaderboards) == 2 + + def test_uses_default_config_when_page_missing(self, chain_data_file): + wiki = DryRunClient(existing_pages={}) + importer = BlueRailroadImporter(wiki, chain_data_file) + + config = importer.load_config() + + assert len(config.sources) >= 1 + assert len(config.leaderboards) >= 1 + + def test_aggregates_tokens_from_all_sources(self, chain_data_file, wiki_config_content): + wiki = DryRunClient(existing_pages={ + 'PickiPedia:BlueRailroadConfig': wiki_config_content, + }) + importer = BlueRailroadImporter(wiki, chain_data_file) + + config = importer.load_config() + tokens = importer.load_tokens(config) + + # Should have 3 tokens total (2 from V1, 1 from V2) + assert len(tokens) == 3 + + def test_run_creates_token_pages(self, chain_data_file, wiki_config_content): + wiki = DryRunClient(existing_pages={ + 'PickiPedia:BlueRailroadConfig': wiki_config_content, + }) + importer = BlueRailroadImporter(wiki, chain_data_file) + + stats = importer.run() + + # Should create 3 token pages + assert stats.tokens_created == 3 + + # Check pages would be saved + saved_titles = [title for title, _, _ in wiki.would_save] + assert 'Blue Railroad Token 1' in saved_titles + assert 'Blue Railroad Token 2' in saved_titles + assert 'Blue Railroad Token 5' in saved_titles + + def test_run_creates_leaderboards(self, chain_data_file, wiki_config_content): + wiki = DryRunClient(existing_pages={ + 'PickiPedia:BlueRailroadConfig': wiki_config_content, + }) + importer = BlueRailroadImporter(wiki, chain_data_file) + + stats = importer.run() + + # Should create 2 leaderboards + assert stats.leaderboards_created == 2 + + saved_titles = [title for title, _, _ in wiki.would_save] + assert 'Blue Railroad Leaderboard' in saved_titles + assert 'Blue Railroad Squats Leaderboard' in saved_titles + + def test_skips_unchanged_pages(self, chain_data_file, wiki_config_content): + # Pre-populate with existing content + wiki = DryRunClient(existing_pages={ + 'PickiPedia:BlueRailroadConfig': wiki_config_content, + }) + + # First run to get the content + importer = BlueRailroadImporter(wiki, chain_data_file) + importer.run() + + # Get the generated content + generated_pages = {title: content for title, content, _ in wiki.would_save} + + # Second run with pre-existing content + wiki2 = DryRunClient(existing_pages={ + 'PickiPedia:BlueRailroadConfig': wiki_config_content, + **generated_pages, + }) + importer2 = BlueRailroadImporter(wiki2, chain_data_file) + stats2 = importer2.run() + + # Everything should be unchanged + assert stats2.tokens_created == 0 + assert stats2.tokens_updated == 0 + assert stats2.tokens_unchanged == 3 + assert stats2.leaderboards_unchanged == 2 + + +class TestLeaderboardAggregationIntegration: + """ + Integration test for the critical aggregation fix. + + This tests the exact scenario that caused the bot loop: + multiple sources should be aggregated BEFORE generating leaderboards, + not generate separate leaderboards per source. + """ + + def test_leaderboard_includes_all_sources(self, chain_data_file, wiki_config_content): + wiki = DryRunClient(existing_pages={ + 'PickiPedia:BlueRailroadConfig': wiki_config_content, + }) + importer = BlueRailroadImporter(wiki, chain_data_file) + importer.run() + + # Find the overall leaderboard content + leaderboard_content = None + for title, content, _ in wiki.would_save: + if title == 'Blue Railroad Leaderboard': + leaderboard_content = content + break + + assert leaderboard_content is not None + + # Alice should have 2 tokens (1 from V1, 1 from V2) + # The row format is: | rank || holder || count || token links + assert '| 1 || alice.eth || 2 ||' in leaderboard_content + + # Token IDs from both sources should be present + assert '#1]]' in leaderboard_content # V1 token + assert '#5]]' in leaderboard_content # V2 token + + def test_multiple_runs_produce_identical_output(self, chain_data_file, wiki_config_content): + """Idempotency test - running twice should produce same content.""" + wiki1 = DryRunClient(existing_pages={ + 'PickiPedia:BlueRailroadConfig': wiki_config_content, + }) + wiki2 = DryRunClient(existing_pages={ + 'PickiPedia:BlueRailroadConfig': wiki_config_content, + }) + + importer1 = BlueRailroadImporter(wiki1, chain_data_file) + importer2 = BlueRailroadImporter(wiki2, chain_data_file) + + importer1.run() + importer2.run() + + # Get leaderboard content from both runs + lb1 = next(c for t, c, _ in wiki1.would_save if t == 'Blue Railroad Leaderboard') + lb2 = next(c for t, c, _ in wiki2.would_save if t == 'Blue Railroad Leaderboard') + + assert lb1 == lb2 diff --git a/tools/blue-railroad-import/tests/test_leaderboard.py b/tools/blue-railroad-import/tests/test_leaderboard.py new file mode 100644 index 0000000..74c0102 --- /dev/null +++ b/tools/blue-railroad-import/tests/test_leaderboard.py @@ -0,0 +1,200 @@ +"""Tests for leaderboard generation.""" + +import pytest +from blue_railroad_import.models import Token, LeaderboardConfig +from blue_railroad_import.leaderboard import ( + filter_tokens, + calculate_owner_stats, + sort_owners, + generate_leaderboard_content, +) + + +@pytest.fixture +def sample_tokens(): + """Sample token data for testing.""" + return { + 'v1_1': Token( + token_id='1', + source_key='blueRailroads', + owner='0xAlice', + owner_display='alice.eth', + song_id='5', + date=100, + ), + 'v1_2': Token( + token_id='2', + source_key='blueRailroads', + owner='0xAlice', + owner_display='alice.eth', + song_id='5', + date=200, + ), + 'v1_3': Token( + token_id='3', + source_key='blueRailroads', + owner='0xBob', + owner_display='bob.eth', + song_id='6', + date=150, + ), + 'v2_5': Token( + token_id='5', + source_key='blueRailroadV2s', + owner='0xAlice', + owner_display='alice.eth', + song_id='5', + blockheight=300, + ), + } + + +class TestFilterTokens: + """Tests for filter_tokens function.""" + + def test_no_filter_returns_all(self, sample_tokens): + result = filter_tokens(sample_tokens) + assert len(result) == 4 + + def test_filters_by_song_id(self, sample_tokens): + result = filter_tokens(sample_tokens, filter_song_id='5') + assert len(result) == 3 + assert all(t.song_id == '5' for t in result.values()) + + def test_filters_by_owner(self, sample_tokens): + result = filter_tokens(sample_tokens, filter_owner='0xBob') + assert len(result) == 1 + assert list(result.values())[0].owner == '0xBob' + + def test_owner_filter_is_case_insensitive(self, sample_tokens): + result = filter_tokens(sample_tokens, filter_owner='0xbob') # lowercase + assert len(result) == 1 + + def test_combines_filters(self, sample_tokens): + result = filter_tokens(sample_tokens, filter_song_id='5', filter_owner='0xAlice') + assert len(result) == 3 # Alice has 3 song_id=5 tokens + + +class TestCalculateOwnerStats: + """Tests for calculate_owner_stats function.""" + + def test_calculates_token_counts(self, sample_tokens): + stats = calculate_owner_stats(sample_tokens) + assert stats['0xAlice'].token_count == 3 + assert stats['0xBob'].token_count == 1 + + def test_tracks_token_ids(self, sample_tokens): + stats = calculate_owner_stats(sample_tokens) + assert set(stats['0xAlice'].token_ids) == {'1', '2', '5'} + + def test_tracks_newest_date(self, sample_tokens): + stats = calculate_owner_stats(sample_tokens) + # Alice's newest is v2 blockheight 300 + assert stats['0xAlice'].newest_date == 300 + + def test_tracks_oldest_date(self, sample_tokens): + stats = calculate_owner_stats(sample_tokens) + # Alice's oldest is v1 date 100 + assert stats['0xAlice'].oldest_date == 100 + + def test_preserves_display_name(self, sample_tokens): + stats = calculate_owner_stats(sample_tokens) + assert stats['0xAlice'].display_name == 'alice.eth' + + +class TestSortOwners: + """Tests for sort_owners function.""" + + def test_sort_by_count_descending(self, sample_tokens): + stats = calculate_owner_stats(sample_tokens) + sorted_owners = sort_owners(stats, 'count') + assert sorted_owners[0] == '0xAlice' # 3 tokens + assert sorted_owners[1] == '0xBob' # 1 token + + def test_sort_by_newest(self, sample_tokens): + stats = calculate_owner_stats(sample_tokens) + sorted_owners = sort_owners(stats, 'newest') + assert sorted_owners[0] == '0xAlice' # newest=300 + + def test_sort_by_oldest(self, sample_tokens): + stats = calculate_owner_stats(sample_tokens) + sorted_owners = sort_owners(stats, 'oldest') + assert sorted_owners[0] == '0xAlice' # oldest=100 + + +class TestGenerateLeaderboardContent: + """Tests for generate_leaderboard_content function.""" + + def test_generates_valid_wikitext(self, sample_tokens): + config = LeaderboardConfig(page='Test Leaderboard', title='Test') + content = generate_leaderboard_content(sample_tokens, config) + + assert "'''Test'''" in content + assert 'wikitable sortable' in content + assert '[[Category:Blue Railroad]]' in content + + def test_includes_statistics(self, sample_tokens): + config = LeaderboardConfig(page='Test') + content = generate_leaderboard_content(sample_tokens, config) + + assert "'''Total Tokens:''' 4" in content + assert "'''Total Holders:''' 2" in content + + def test_includes_token_links(self, sample_tokens): + config = LeaderboardConfig(page='Test') + content = generate_leaderboard_content(sample_tokens, config) + + assert '[[Blue Railroad Token 1|#1]]' in content + assert '[[Blue Railroad Token 5|#5]]' in content + + def test_includes_exercise_name_for_filtered(self, sample_tokens): + config = LeaderboardConfig(page='Test', filter_song_id='5') + content = generate_leaderboard_content(sample_tokens, config) + + assert 'Squats' in content + assert 'Blue Railroad Train' in content + + def test_filters_tokens_before_generating(self, sample_tokens): + config = LeaderboardConfig(page='Test', filter_song_id='6') + content = generate_leaderboard_content(sample_tokens, config) + + # Only Bob has song_id=6 + assert "'''Total Tokens:''' 1" in content + assert "'''Total Holders:''' 1" in content + assert 'bob.eth' in content + + +class TestLeaderboardAggregation: + """ + Critical tests for the aggregation bug that caused the bot loop. + + The PHP version generated leaderboards inside the per-source loop, + so V1 and V2 tokens weren't combined - the second source would + overwrite the first's leaderboard with different content. + """ + + def test_aggregated_tokens_include_all_sources(self, sample_tokens): + """Leaderboard should include BOTH V1 and V2 tokens.""" + config = LeaderboardConfig(page='Test') + content = generate_leaderboard_content(sample_tokens, config) + + # Alice's total across V1 and V2 + assert '| 1 || alice.eth || 3 ||' in content + + def test_token_ids_sorted_numerically(self, sample_tokens): + """Token IDs in leaderboard should be sorted numerically.""" + config = LeaderboardConfig(page='Test') + content = generate_leaderboard_content(sample_tokens, config) + + # Alice's tokens should be 1, 2, 5 in order + alice_row = [line for line in content.split('\n') if 'alice.eth' in line][0] + assert '#1]], [[Blue Railroad Token 2|#2]], [[Blue Railroad Token 5|#5]]' in alice_row + + def test_idempotent_content_generation(self, sample_tokens): + """Same input should always produce same output.""" + config = LeaderboardConfig(page='Test', sort='count') + + content1 = generate_leaderboard_content(sample_tokens, config) + content2 = generate_leaderboard_content(sample_tokens, config) + + assert content1 == content2 diff --git a/tools/blue-railroad-import/tests/test_models.py b/tools/blue-railroad-import/tests/test_models.py new file mode 100644 index 0000000..7c01171 --- /dev/null +++ b/tools/blue-railroad-import/tests/test_models.py @@ -0,0 +1,128 @@ +"""Tests for data models.""" + +import pytest +from blue_railroad_import.models import Token, OwnerStats + + +class TestToken: + """Tests for Token model.""" + + def test_v1_token_is_not_v2(self): + token = Token( + token_id='1', + source_key='blueRailroads', + owner='0x123', + owner_display='alice.eth', + date=20260113, + uri='ipfs://QmXyz123', + ) + assert token.is_v2 is False + + def test_v2_token_is_v2(self): + token = Token( + token_id='5', + source_key='blueRailroadV2s', + owner='0x456', + owner_display='bob.eth', + blockheight=12345678, + video_hash='0xabc123', + ) + assert token.is_v2 is True + + def test_formatted_date_from_yyyymmdd(self): + token = Token( + token_id='1', + source_key='blueRailroads', + owner='0x123', + owner_display='alice.eth', + date=20260113, + ) + assert token.formatted_date == '2026-01-13' + + def test_formatted_date_from_unix_timestamp(self): + token = Token( + token_id='1', + source_key='blueRailroads', + owner='0x123', + owner_display='alice.eth', + date=1705685808, # 2024-01-19 + ) + assert token.formatted_date == '2024-01-19' + + def test_formatted_date_returns_none_when_missing(self): + token = Token( + token_id='1', + source_key='blueRailroads', + owner='0x123', + owner_display='alice.eth', + ) + assert token.formatted_date is None + + def test_ipfs_cid_from_v1_uri(self): + token = Token( + token_id='1', + source_key='blueRailroads', + owner='0x123', + owner_display='alice.eth', + uri='ipfs://QmXyz123abc', + ) + assert token.ipfs_cid == 'QmXyz123abc' + + def test_ipfs_cid_from_v2_video_hash(self): + token = Token( + token_id='5', + source_key='blueRailroadV2s', + owner='0x456', + owner_display='bob.eth', + blockheight=12345678, + video_hash='0xabc123def456', + ) + assert token.ipfs_cid == 'abc123def456' + + def test_ipfs_cid_none_for_empty_v2_hash(self): + token = Token( + token_id='5', + source_key='blueRailroadV2s', + owner='0x456', + owner_display='bob.eth', + blockheight=12345678, + video_hash='0x' + '0' * 64, # Empty bytes32 + ) + assert token.ipfs_cid is None + + +class TestOwnerStats: + """Tests for OwnerStats model.""" + + def test_add_token_increments_count(self): + stats = OwnerStats(address='0x123', display_name='alice.eth') + stats.add_token('1', 100) + stats.add_token('2', 200) + assert stats.token_count == 2 + + def test_add_token_tracks_ids(self): + stats = OwnerStats(address='0x123', display_name='alice.eth') + stats.add_token('1', 100) + stats.add_token('5', 200) + assert stats.token_ids == ['1', '5'] + + def test_add_token_tracks_newest_date(self): + stats = OwnerStats(address='0x123', display_name='alice.eth') + stats.add_token('1', 100) + stats.add_token('2', 300) + stats.add_token('3', 200) + assert stats.newest_date == 300 + + def test_add_token_tracks_oldest_date(self): + stats = OwnerStats(address='0x123', display_name='alice.eth') + stats.add_token('1', 300) + stats.add_token('2', 100) + stats.add_token('3', 200) + assert stats.oldest_date == 100 + + def test_add_token_with_none_date(self): + stats = OwnerStats(address='0x123', display_name='alice.eth') + stats.add_token('1', None) + assert stats.token_count == 1 + assert stats.newest_date == 0 + assert stats.oldest_date == 0 From 9857ab0d36f97923c7d01d0234290902f001f0f2 Mon Sep 17 00:00:00 2001 From: magent Date: Fri, 30 Jan 2026 00:21:12 +0000 Subject: [PATCH 08/10] Move favicon to assets/ and set $wgFavicon favicon.ico at the repo root wasn't being copied into the MediaWiki build - only the assets/ directory gets copied by the Jenkinsfile. Moved it there and added $wgFavicon to LocalSettings.php. --- LocalSettings.php | 1 + favicon.ico => assets/favicon.ico | Bin 2 files changed, 1 insertion(+) rename favicon.ico => assets/favicon.ico (100%) diff --git a/LocalSettings.php b/LocalSettings.php index 4cebd7e..eae32b9 100644 --- a/LocalSettings.php +++ b/LocalSettings.php @@ -105,6 +105,7 @@ $wgLogos = [ '1x' => "$wgResourceBasePath/assets/logo.png", ]; +$wgFavicon = "$wgResourceBasePath/assets/favicon.ico"; ## Skins wfLoadSkin( 'MonoBook' ); diff --git a/favicon.ico b/assets/favicon.ico similarity index 100% rename from favicon.ico rename to assets/favicon.ico From bf7623e6e33a74003d79a0cf0c1cd10b2e084cf0 Mon Sep 17 00:00:00 2001 From: magent Date: Sat, 31 Jan 2026 12:09:52 +0000 Subject: [PATCH 09/10] Add podcast firehose and guest extraction pipeline Tools for aggregating bluegrass podcast RSS feeds and extracting guest names from episode titles for semantic wiki tagging. - podcast-feeds.json: 13 verified bluegrass podcast RSS feeds - podcast-firehose.py: Merges feeds into a single combined RSS feed - podcast-guest-patterns.json: Per-podcast regex patterns for guest name extraction (636 guests from 510 unique names across 9 podcasts) - podcast-episodes.py: Generates {{PodcastEpisode}} wikitext from RSS data + extracted guests - test_podcast_guests.py: 55 tests covering all podcast formats, skip patterns, edge cases, and false positive prevention --- tools/podcast-episodes.py | 226 ++++++++++ tools/podcast-feeds.json | 67 +++ tools/podcast-firehose.py | 164 ++++++++ tools/podcast-guest-patterns.json | 103 +++++ tools/test_podcast_guests.py | 661 ++++++++++++++++++++++++++++++ 5 files changed, 1221 insertions(+) create mode 100644 tools/podcast-episodes.py create mode 100644 tools/podcast-feeds.json create mode 100644 tools/podcast-firehose.py create mode 100644 tools/podcast-guest-patterns.json create mode 100644 tools/test_podcast_guests.py diff --git a/tools/podcast-episodes.py b/tools/podcast-episodes.py new file mode 100644 index 0000000..bd70b0f --- /dev/null +++ b/tools/podcast-episodes.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +""" +Podcast Episode Page Generator - Extracts guest names from podcast RSS feeds +and generates wiki page content for each episode using {{PodcastEpisode}}. + +Modes: + --dry-run Show what pages would be created (default) + --preview N Show full wikitext for N sample pages + --create Actually create pages on the wiki via MCP + --json Output structured JSON for all episodes with guests +""" + +import json +import re +import sys +from datetime import datetime +from email.utils import parsedate_to_datetime +from pathlib import Path +from urllib.request import urlopen, Request +from xml.etree.ElementTree import fromstring + +TOOLS_DIR = Path(__file__).parent +FEEDS_CONFIG = TOOLS_DIR / "podcast-feeds.json" +PATTERNS_CONFIG = TOOLS_DIR / "podcast-guest-patterns.json" +USER_AGENT = "PickiPedia Bluegrass Podcast Firehose/1.0" +FETCH_TIMEOUT = 30 + + +def load_config(): + feeds = json.load(open(FEEDS_CONFIG)) + patterns = json.load(open(PATTERNS_CONFIG))["patterns"] + return feeds, patterns + + +def fetch_episodes(feed_url): + """Fetch RSS feed and return list of (title, link, pubdate, description) tuples.""" + req = Request(feed_url, headers={"User-Agent": USER_AGENT}) + try: + with urlopen(req, timeout=FETCH_TIMEOUT) as resp: + raw = resp.read() + except Exception as e: + print(f" WARN: {e}", file=sys.stderr) + return [] + + try: + root = fromstring(raw) + except Exception as e: + print(f" WARN: parse error: {e}", file=sys.stderr) + return [] + + episodes = [] + for item in root.findall(".//item"): + title_el = item.find("title") + link_el = item.find("link") + pd_el = item.find("pubDate") + desc_el = item.find("description") + + title = title_el.text.strip() if title_el is not None and title_el.text else "" + link = link_el.text.strip() if link_el is not None and link_el.text else "" + desc = desc_el.text.strip() if desc_el is not None and desc_el.text else "" + + pubdate = None + if pd_el is not None and pd_el.text: + try: + pubdate = parsedate_to_datetime(pd_el.text) + except (ValueError, TypeError): + pass + + if title: + episodes.append({ + "title": title, + "link": link, + "pubdate": pubdate, + "description": desc, + }) + + return episodes + + +def extract_guest(title, patterns_for_podcast): + """Try to extract guest name(s) from episode title using patterns. + Returns (guests_list, should_skip) tuple.""" + for p in patterns_for_podcast: + m = re.match(p["pattern"], title) + if m: + if p.get("skip"): + return [], True + groups = m.groupdict() + guests = [] + if "guest" in groups and groups["guest"]: + guest = groups["guest"].strip() + # Split on " & " or " and " for multi-guest episodes + parts = re.split(r'\s*(?:&|and)\s*', guest) + # Only split if parts look like individual names (2+ words each) + if len(parts) > 1 and all(len(p.strip().split()) >= 2 for p in parts): + guests.extend(p.strip() for p in parts) + else: + guests.append(guest) + if "feat" in groups and groups["feat"]: + guests.append(groups["feat"].strip()) + return guests, False + return [], False + + +def make_page_title(podcast_name, episode_title): + """Generate a wiki page title for an episode.""" + # Sanitize: remove characters not allowed in MediaWiki titles + safe_title = re.sub(r'[#<>\[\]|{}]', '', episode_title) + safe_title = safe_title.strip() + if len(safe_title) > 120: + safe_title = safe_title[:120].rsplit(' ', 1)[0] + return f"{podcast_name}/{safe_title}" + + +def make_wikitext(podcast_name, episode, guests): + """Generate wikitext for an episode page.""" + date_str = "" + if episode["pubdate"]: + date_str = episode["pubdate"].strftime("%Y-%m-%d") + + # Build template params + params = [f"|podcast={podcast_name}"] + params.append(f"|title={episode['title']}") + if date_str: + params.append(f"|date={date_str}") + if episode["link"]: + params.append(f"|url={episode['link']}") + + for i, guest in enumerate(guests): + key = "guest" if i == 0 else f"guest{i+1}" + params.append(f"|{key}={guest}") + + # Clean description (strip HTML tags/entities, truncate) + desc = re.sub(r'<[^>]+>', '', episode.get("description", "")) + import html + desc = html.unescape(desc) + if len(desc) > 500: + desc = desc[:500].rsplit(' ', 1)[0] + "..." + if desc: + params.append(f"|description={desc}") + + template_call = "{{PodcastEpisode\n" + "\n".join(params) + "\n}}" + return template_call + + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Generate podcast episode wiki pages") + parser.add_argument("--dry-run", action="store_true", default=True, + help="Show what pages would be created (default)") + parser.add_argument("--preview", type=int, metavar="N", + help="Show full wikitext for N sample pages") + parser.add_argument("--json", action="store_true", + help="Output structured JSON") + parser.add_argument("--podcast", type=str, + help="Only process this podcast") + args = parser.parse_args() + + feeds, patterns = load_config() + + all_episodes = [] + total_guests = 0 + total_skipped = 0 + total_unmatched = 0 + + for feed in feeds: + name = feed["name"] + if args.podcast and name != args.podcast: + continue + + pats = patterns.get(name, []) + if not pats: + continue + + print(f"Fetching: {name}...", file=sys.stderr) + episodes = fetch_episodes(feed["url"]) + + for ep in episodes: + guests, skipped = extract_guest(ep["title"], pats) + if skipped: + total_skipped += 1 + continue + if not guests: + total_unmatched += 1 + continue + + total_guests += 1 + page_title = make_page_title(name, ep["title"]) + wikitext = make_wikitext(name, ep, guests) + + all_episodes.append({ + "page_title": page_title, + "podcast": name, + "episode_title": ep["title"], + "guests": guests, + "date": ep["pubdate"].isoformat() if ep["pubdate"] else None, + "link": ep["link"], + "wikitext": wikitext, + }) + + print(f"\nResults: {total_guests} episodes with guests, " + f"{total_skipped} skipped, {total_unmatched} unmatched", + file=sys.stderr) + + if args.json: + # Strip wikitext from JSON output to keep it clean + output = [] + for ep in all_episodes: + out = {k: v for k, v in ep.items() if k != "wikitext"} + output.append(out) + json.dump(output, sys.stdout, indent=2, default=str) + print() + elif args.preview: + for ep in all_episodes[:args.preview]: + print(f"=== {ep['page_title']} ===") + print(ep["wikitext"]) + print() + else: + # Dry run - show page titles and guests + for ep in all_episodes: + guests_str = ", ".join(ep["guests"]) + print(f" {ep['page_title']} => [{guests_str}]") + + +if __name__ == "__main__": + main() diff --git a/tools/podcast-feeds.json b/tools/podcast-feeds.json new file mode 100644 index 0000000..48520eb --- /dev/null +++ b/tools/podcast-feeds.json @@ -0,0 +1,67 @@ +[ + { + "name": "Bluegrass Jam Along", + "host": "Matt Hutchinson", + "url": "https://feeds.buzzsprout.com/1660894.rss" + }, + { + "name": "What's The Reason For This Podcast", + "host": "Kodi Nottingham", + "url": "https://media.rss.com/whatsthereasonforthis/feed.xml" + }, + { + "name": "Walls of Time: Bluegrass Podcast", + "host": "Daniel Mullins & Ty Gilpin", + "url": "https://feed.podbean.com/wallsoftimepodcast/feed.xml" + }, + { + "name": "Toy Heart with Tom Power", + "host": "Tom Power / The Bluegrass Situation", + "url": "https://feeds.redcircle.com/727db7a9-7186-435e-97e2-5b83bf90875e" + }, + { + "name": "Bluegrass Unlimited's Podcast", + "host": "Dan Miller", + "url": "https://bluegrassunlimited.libsyn.com/rss" + }, + { + "name": "The Picky Fingers Banjo Podcast", + "host": "Keith Billik", + "url": "https://banjopodcast.libsyn.com/rss" + }, + { + "name": "The Old Dingy Jukebox", + "host": "Christian Gallo", + "url": "https://feeds.buzzsprout.com/1054516.rss" + }, + { + "name": "Grass Talk Radio", + "host": "Bradley Laird", + "url": "https://feed.podbean.com/bradleylaird/feed.xml" + }, + { + "name": "Bluegrass BKLYN", + "host": "Mike Willner & Liz Wolfe", + "url": "https://feed.podbean.com/bluegrassblkyn/feed.xml" + }, + { + "name": "Southern Branch Bluegrass & Gospel Music Radio", + "host": "Danny Hensley", + "url": "https://media.rss.com/southern-branch-bluegrass-and-gospel-music-radio/feed.xml" + }, + { + "name": "Bluegrass Ambassadors", + "host": "Henhouse Prowlers", + "url": "https://bluegrassambassadors.libsyn.com/rss" + }, + { + "name": "County Sales Radio Hour", + "host": "County Sales", + "url": "https://www.blubrry.com/feeds/countysales.xml" + }, + { + "name": "Fiddle Studio", + "host": "Meg Wobus Beller", + "url": "https://feeds.buzzsprout.com/2040333.rss" + } +] diff --git a/tools/podcast-firehose.py b/tools/podcast-firehose.py new file mode 100644 index 0000000..856d8d4 --- /dev/null +++ b/tools/podcast-firehose.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +""" +Bluegrass Podcast Firehose - Aggregates RSS feeds from bluegrass podcasts +into a single combined feed, sorted by publication date. + +Reads feed URLs from a config file, fetches each one, merges all episodes, +and outputs a combined RSS XML file. +""" + +import json +import sys +import time +import xml.etree.ElementTree as ET +from datetime import datetime +from email.utils import parsedate_to_datetime, format_datetime +from pathlib import Path +from urllib.request import urlopen, Request +from urllib.error import URLError + +FEEDS_CONFIG = Path(__file__).parent / "podcast-feeds.json" +USER_AGENT = "PickiPedia Bluegrass Podcast Firehose/1.0" +FETCH_TIMEOUT = 30 +MAX_EPISODES_PER_FEED = 50 +MAX_TOTAL_EPISODES = 200 + + +def load_feeds(): + with open(FEEDS_CONFIG) as f: + return json.load(f) + + +def fetch_feed(url): + """Fetch and parse an RSS feed, returning (channel_info, items).""" + req = Request(url, headers={"User-Agent": USER_AGENT}) + try: + with urlopen(req, timeout=FETCH_TIMEOUT) as resp: + raw = resp.read() + except (URLError, TimeoutError) as e: + print(f" WARN: Failed to fetch {url}: {e}", file=sys.stderr) + return None, [] + + try: + root = ET.fromstring(raw) + except ET.ParseError as e: + print(f" WARN: Failed to parse {url}: {e}", file=sys.stderr) + return None, [] + + channel = root.find("channel") + if channel is None: + return None, [] + + title_el = channel.find("title") + link_el = channel.find("link") + channel_info = { + "title": title_el.text if title_el is not None else "Unknown", + "link": link_el.text if link_el is not None else "", + } + + items = [] + for item in channel.findall("item")[:MAX_EPISODES_PER_FEED]: + items.append((channel_info, item, raw_pubdate(item))) + + return channel_info, items + + +def raw_pubdate(item): + """Extract pubDate as datetime for sorting. Returns epoch 0 on failure.""" + pd = item.find("pubDate") + if pd is not None and pd.text: + try: + return parsedate_to_datetime(pd.text) + except (ValueError, TypeError): + pass + return datetime(1970, 1, 1) + + +def build_combined_feed(feeds_config, all_items): + """Build a combined RSS feed XML string.""" + now = format_datetime(datetime.now().astimezone()) + + rss = ET.Element("rss", version="2.0") + rss.set("xmlns:itunes", "http://www.itunes.com/dtds/podcast-1.0.dtd") + rss.set("xmlns:content", "http://purl.org/rss/1.0/modules/content/") + + channel = ET.SubElement(rss, "channel") + ET.SubElement(channel, "title").text = "PickiPedia Bluegrass Podcast Firehose" + ET.SubElement(channel, "link").text = "https://pickipedia.xyz/wiki/Bluegrass_Podcast_Firehose" + ET.SubElement(channel, "description").text = ( + "A combined feed of bluegrass and traditional music podcasts, " + "aggregated by PickiPedia. Episodes from multiple shows sorted by date." + ) + ET.SubElement(channel, "language").text = "en-us" + ET.SubElement(channel, "lastBuildDate").text = now + ET.SubElement(channel, "generator").text = "PickiPedia Bluegrass Podcast Firehose" + + # Sort by pubdate descending, take top N + all_items.sort(key=lambda x: x[2], reverse=True) + for channel_info, item, pubdate in all_items[:MAX_TOTAL_EPISODES]: + new_item = ET.SubElement(channel, "item") + + # Copy standard elements + for tag in ("title", "link", "description", "pubDate", "guid", "enclosure"): + el = item.find(tag) + if el is not None: + new_el = ET.SubElement(new_item, tag) + new_el.text = el.text + for k, v in el.attrib.items(): + new_el.set(k, v) + + # Copy itunes elements + ns = {"itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd"} + for itunes_tag in ("duration", "summary", "image", "explicit"): + el = item.find(f"itunes:{itunes_tag}", ns) + if el is not None: + new_el = ET.SubElement(new_item, f"itunes:{itunes_tag}") + new_el.text = el.text + for k, v in el.attrib.items(): + new_el.set(k, v) + + # Prepend podcast name to title + title_el = new_item.find("title") + if title_el is not None and title_el.text: + title_el.text = f"[{channel_info['title']}] {title_el.text}" + + # Add source category + source_el = ET.SubElement(new_item, "source", url=channel_info.get("link", "")) + source_el.text = channel_info["title"] + + return rss + + +def main(): + feeds = load_feeds() + print(f"Fetching {len(feeds)} feeds...", file=sys.stderr) + + all_items = [] + for feed in feeds: + name = feed.get("name", feed["url"]) + print(f" Fetching: {name}...", file=sys.stderr) + channel_info, items = fetch_feed(feed["url"]) + if items: + print(f" Got {len(items)} episodes", file=sys.stderr) + all_items.extend(items) + else: + print(f" No episodes found", file=sys.stderr) + + print(f"Total episodes: {len(all_items)}", file=sys.stderr) + + rss = build_combined_feed(feeds, all_items) + + # Output + ET.indent(rss) + tree = ET.ElementTree(rss) + output = sys.argv[1] if len(sys.argv) > 1 else None + if output: + tree.write(output, encoding="unicode", xml_declaration=True) + print(f"Written to {output}", file=sys.stderr) + else: + print('') + ET.dump(rss) + + +if __name__ == "__main__": + main() diff --git a/tools/podcast-guest-patterns.json b/tools/podcast-guest-patterns.json new file mode 100644 index 0000000..2aba0c2 --- /dev/null +++ b/tools/podcast-guest-patterns.json @@ -0,0 +1,103 @@ +{ + "_comment": "Per-podcast regex patterns for extracting guest names from episode titles. Each pattern should have a named group 'guest' unless marked 'skip'. Patterns are tried in order; first match wins.", + "patterns": { + "Bluegrass Unlimited's Podcast": [ + {"pattern": "^Bluegrass Unlimited Podcast with (?P.+)$"} + ], + "Bluegrass BKLYN": [ + {"pattern": "^S\\d+[SEM]*\\d*\\s+Women in Bluegrass[:\\s-]+(?P[A-Z].+?)!*$"}, + {"pattern": "^S\\d+[SEM]*\\d*\\s+Home Grown, Locally Known[:\\s-]+(?P[A-Z].+?)!*$"}, + {"pattern": "^S\\d*E?\\d+:?\\s+(?:Home Grown, Locally Known|On the Road|Iconic Venues|Women in Bluegrass)[:\\s-]+(?:with )?(?P[A-Z].+?)!*$"}, + {"pattern": "^SE?\\d+:?\\s+(?P[A-Z][a-z]+(?: (?:and|&) [A-Z][a-z]+| [A-Z][a-z]+)+)!*$"}, + {"pattern": "^S\\d+[SEM]*\\d*:?\\s+(?P[A-Z][a-z]+(?: (?:and|&) [A-Z][a-z]+| [A-Z][a-z]+)+)!*$"}, + {"pattern": "^S\\d+[SEM]*\\d*:?\\s+(?:Home Grown, Locally Known|On the Road[:\\s])!*$", "skip": true, "_note": "Segment eps without guest"}, + {"pattern": "^S\\d+[SEM]*\\d*:?\\s+(?:Addendum|Wrap Up|Recap|Ballad|Let's Talk|NYC Bluegrass|JamVal|Festival Recap)", "skip": true, "_note": "Topic/recap episodes"}, + {"pattern": "^S\\d+[SEM]*\\d*\\s+(?PLe Vent du Nord)!*$", "_note": "French band name edge case"} + ], + "Bluegrass Jam Along": [ + {"pattern": "^.+\\(\\w+ \\d+ bpm\\)", "skip": true, "_note": "Jam-along backing tracks"}, + {"pattern": "^.+ at \\d+ tempos? ", "skip": true, "_note": "Multi-tempo jam tracks"}, + {"pattern": "^.+ in [A-G] at \\d+ tempos", "skip": true, "_note": "Jam tracks with key"}, + {"pattern": "^Mini Jam #", "skip": true}, + {"pattern": "^Bluegrass Briefing", "skip": true}, + {"pattern": "^Bitesize ", "skip": true}, + {"pattern": "^Food [Ff]or Thought", "skip": true}, + {"pattern": "^Intro to Bluegrass", "skip": true}, + {"pattern": "^Announcing ", "skip": true}, + {"pattern": "^Inspiration for Musicians", "skip": true}, + {"pattern": "^A quick (?:update|August)", "skip": true}, + {"pattern": "^We reached \\d+", "skip": true}, + {"pattern": "^IBMA (?:World of Bluegrass )?\\d+ [Ss]pecial", "skip": true}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+) \\([^)]+\\) on .+$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+) on .+$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+) (?:Celebrates?|Talks?|Discusses|Shares?|Returns?|Remembers?)\\b.*$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+)\\s*[-–]\\s*.+$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+)\\s+[Ii]nterview.*$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+) \\([^)]+\\) [Ii]nterview.*$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+ (?:and|&) [A-Z][\\w.'\\-]+(?:(?: [A-Z][\\w'\\-]+))*) [Ii]nterview.*$"}, + {"pattern": "^Celebrating .+ with (?P[A-Z][\\w.'\\-]+(?: [A-Z]?\\.?[\\w'\\-]*)+ ?)(?:\\(.*\\))?$"}, + {"pattern": "^Celebrating \\d+ years of .+ [-–] part \\d+ (?P[A-Z][\\w.'\\-]+(?: (?:&|and) [A-Z][\\w.'\\-]+)?(?: [A-Z][\\w'\\-]+)*)$"}, + {"pattern": "^Celebrating .+ [-–] part \\d+ with (?P.+)$"}, + {"pattern": "^(?:A |The )?Celebration of .+ [-–] (?P.+?)(?:'s .+)?$"}, + {"pattern": "^Highlights from .+ [-–] (?P.+)$"}, + {"pattern": "^(?:Musicality|Roots Revival) .+ with (?P.+)$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+)'s .+$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+) (?:celebrates|has dinner|returns)\\b.*$", "_note": "lowercase verb variants"}, + {"pattern": "^How to .+ with (?P[A-Z][\\w.'\\-]+(?: \\([^)]+\\))?(?: [A-Z][\\w'\\-]+)*)$"}, + {"pattern": "^.+ [Tt]ribute part \\d+ [-–] (?:featuring )?(?P.+)$"}, + {"pattern": "^.+ [Tt]ribute part \\d+ [-–] (?P.+)$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+) \\([^)]+\\)$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?:(?: (?:and|&))? [A-Z][\\w.'\\-]+)+) [Ii]nterview.*$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+) (?:repost|extended interview).*$"}, + {"pattern": "^(?:The )?(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+), .+$"}, + {"pattern": "^Tonewoods Special with (?P.+)$"}, + {"pattern": "^\\d{4} Retrospective with (?P.+)$"}, + {"pattern": "^.+ Anniversary Celebration [-–] part \\d+.*with (?P.+)$"}, + {"pattern": "^.+ Anniversary Celebration [-–] part \\d+ (?P.+)$"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+(?: [A-Z]\\.?)?(?: [A-Z][\\w'\\-]+)+) (?:mini )?interview.*$", "_note": "catch mini interview"}, + {"pattern": "^(?P[A-Z][\\w.'\\-]+ (?:and|&) [A-Z][\\w.'\\-]+(?: [A-Z][\\w'\\-]+)*(?:'s)?) .+$"} + ], + "Walls of Time: Bluegrass Podcast": [ + {"pattern": "^(?:S\\d+ E\\d+|BONUS)\\.\\s*(?P[^:]+?):\\s*.+$"}, + {"pattern": "^(?:S\\d+ E\\d+|BONUS)\\.\\s*(?P[A-Z].+?)\\s+(?:Remembers|on\\b).*$"}, + {"pattern": "^S\\d+ E0\\.", "skip": true, "_note": "Season previews"} + ], + "Toy Heart with Tom Power": [ + {"pattern": "^(?:Season|Seaon) \\d+", "skip": true}, + {"pattern": "^Preview ", "skip": true}, + {"pattern": "^The Bluegrass Breakdown Podcast [-–] (?P[A-Z].+?)\\s*[-–].*$"}, + {"pattern": "^Remembering .+[-–]\\s*Episode \\d+\\s*\\((?P.+)\\)$"}, + {"pattern": "^(?P[A-Z][\\w'À-ÿ]+(?: [A-Z][\\w'À-ÿ]+)+)(?:\\s+in Conversation.*)?$"}, + {"pattern": "^(?P[A-Z][\\w'À-ÿ]+(?:\\s+[A-Z][\\w'À-ÿ]+)*)$"} + ], + "What's The Reason For This Podcast": [ + {"pattern": "^What'?s The Reason For This Podcast[^-]*[-–]\\s*(?P[A-Z].+?)(?:\\s*[-–].*)?$"}, + {"pattern": "^What'?s The Reason For This (?:Season \\d+ )?Episode \\d+\\s*[-–]\\s*(?P[A-Z].+?)(?:\\s*[-–].*)?$"}, + {"pattern": "^What'?s The Reason For This Session\\s*[-–]?\\s*\\d*\\s*[-–]\\s*(?P[A-Z].+?)(?:\\s*[-–].*)?$"}, + {"pattern": "^Episode \\d+\\s*[-–]\\s*(?P[A-Z].+?)(?:\\s*[-–].*)?$"} + ], + "The Picky Fingers Banjo Podcast": [ + {"pattern": "^#\\d+\\s*[-–]\\s*\"[^\"]*\"\\s+(?:feat\\.?|Feat\\.?)\\s+(?P.+)$"}, + {"pattern": "^#\\d+\\s*[-–]\\s*\"[^\"]*\"\\s+(?:by|Revisited!\\s*Feat\\.?)\\s+(?P.+)$"}, + {"pattern": "^#\\d+\\s*[-–]\\s*(?:In Memoriam:\\s*)?(?P[A-Z][\\w'\\-]+(?: [A-Z][\\w'\\-]+)*).*$"}, + {"pattern": "^BONUS\\s*[-–]\\s*(?:Fireside Chat w/\\s*)?(?P[A-Z][\\w'\\-]+(?: [A-Z][\\w'\\-]+)*).*$"}, + {"pattern": "^Bonus\\s*[-–]\\s*(?P[A-Z][\\w'\\-]+(?: [A-Z][\\w'\\-]+)*).*$"}, + {"pattern": "^REPLAY\\s*[-–]\\s*(?P[A-Z][\\w'\\-]+(?: [A-Z][\\w'\\-]+)*).*$"}, + {"pattern": "^#\\d+\\s*[-–]\\s*Dueling .+feat\\.?\\s*(?P.+)$"}, + {"pattern": "^Bonus\\s*[-–]\\s*(?:Who is|Eli)", "skip": true, "_note": "Non-guest bonus eps"} + ], + "Bluegrass Ambassadors": [ + {"pattern": "^Episode \\d+\\s*[-–:]\\s*(?P[A-Z].+?)\\s*[-–:]\\s*.+$"} + ], + "Grass Talk Radio": [ + {"pattern": "^GTR-\\d+\\s+(?P[A-Z][a-z]+(?: \"[^\"]+\")?(?: [A-Z][a-z]+)+)\\s+[Ii]nterview.*$"}, + {"pattern": "^GTR-\\d+\\s*[-–]\\s*(?P[A-Z][a-z]+(?: [\\u201c\\u201d\"]+[^\\u201c\\u201d\"]+[\\u201c\\u201d\"]+)?(?: [A-Z][a-z]+)+)\\s+[Ii]nterview.*$"}, + {"pattern": "^GTR-\\d+\\s*[-–]\\s*(?:\"[^\"]*\"\\s+)?(?:Author|Luthier)\\s+(?P[A-Z][a-z]+(?: [A-Z][a-z]+)+).*$"}, + {"pattern": "^GTR-\\d+\\s*[-–]?\\s*(?P[A-Z][a-z]+(?: [A-Z][a-z]+)+) of .+$", "_note": "Name of Organization pattern"} + ], + "The Old Dingy Jukebox": [], + "Fiddle Studio": [], + "Southern Branch Bluegrass & Gospel Music Radio": [], + "County Sales Radio Hour": [] + } +} diff --git a/tools/test_podcast_guests.py b/tools/test_podcast_guests.py new file mode 100644 index 0000000..a6e94f8 --- /dev/null +++ b/tools/test_podcast_guests.py @@ -0,0 +1,661 @@ +#!/usr/bin/env python3 +""" +Tests for the podcast guest extraction pipeline. + +These tests verify that we can reliably pull guest names out of podcast +episode titles using per-podcast regex patterns. The patterns live in +podcast-guest-patterns.json and are tried in order — first match wins. + +A pattern can either: + - Extract a guest name via the (?P...) named group + - Mark an episode as "skip" (e.g., jam-along backing tracks, topic episodes) + +The test suite is organized around real episode titles from the feeds, +grouped by the kind of extraction challenge they represent. +""" + +import json +import re +import pytest +from pathlib import Path + +# --------------------------------------------------------------------------- +# Fixtures: load the actual pattern config once per session +# --------------------------------------------------------------------------- + +TOOLS_DIR = Path(__file__).parent +PATTERNS_FILE = TOOLS_DIR / "podcast-guest-patterns.json" + + +@pytest.fixture(scope="session") +def all_patterns(): + """Load the full pattern config from disk. + + This is the real config, not a mock — we're testing that the actual + patterns we ship do what we expect on real episode titles. + """ + with open(PATTERNS_FILE) as f: + return json.load(f)["patterns"] + + +def extract(title: str, patterns: list) -> tuple[list[str], bool]: + """Run the extraction logic against a single title. + + Returns (guests, was_skipped). This mirrors the logic in + podcast-episodes.py's extract_guest() but is self-contained + so the tests don't import from a script with a __main__ guard. + """ + for p in patterns: + m = re.match(p["pattern"], title) + if m: + if p.get("skip"): + return [], True + groups = m.groupdict() + guests = [] + if "guest" in groups and groups["guest"]: + guest = groups["guest"].strip() + # Split on & / and for multi-guest episodes, + # but only if each part looks like a name (2+ words) + parts = re.split(r'\s*(?:&|and)\s*', guest) + if len(parts) > 1 and all( + len(p.strip().split()) >= 2 for p in parts + ): + guests.extend(p.strip() for p in parts) + else: + guests.append(guest) + if "feat" in groups and groups["feat"]: + guests.append(groups["feat"].strip()) + return guests, False + return [], False + + +# =================================================================== +# 1. BLUEGRASS JAM ALONG +# +# Matt Hutchinson's podcast has the widest variety of title formats: +# - "Guest Name on Topic" (most common) +# - "Guest Name - Topic" (dash separator) +# - "Guest Name (Band) on Topic" (with parenthetical) +# - "Guest Name Talks/Celebrates/Shares..." (verb after name) +# - Jam-along tracks that should be skipped (BPM in title) +# - Briefings, bitesize eps, retrospectives (skip) +# =================================================================== + +class TestBluegrassJamAlong: + """Bluegrass Jam Along — the flagship test, ~500 episodes.""" + + @pytest.fixture(autouse=True) + def _load(self, all_patterns): + self.pats = all_patterns["Bluegrass Jam Along"] + + # -- Standard "Name on Topic" format -- + + def test_simple_name_on_topic(self): + """The bread and butter: 'First Last on Some Topic'.""" + guests, skip = extract( + "Dave Sinko on How 'The David Grisman Quintet' Changed His Life", + self.pats, + ) + assert guests == ["Dave Sinko"] + assert not skip + + def test_three_word_name(self): + """Names with middle initials or three parts.""" + guests, _ = extract( + "Kristina R. Gaddy - Go Back and Fetch It", + self.pats, + ) + assert guests == ["Kristina R. Gaddy"] + + # -- Parenthetical band names -- + + def test_name_with_band_parenthetical(self): + """'Kyle Tuttle (Molly Tuttle & Golden Highway)' — the parenthetical + contains the guest's band affiliation, common in bluegrass where + pickers move between projects.""" + guests, _ = extract( + "Kyle Tuttle (Molly Tuttle & Golden Highway)", + self.pats, + ) + assert guests == ["Kyle Tuttle"] + + def test_parenthetical_on_topic(self): + """Parenthetical followed by 'on Topic'.""" + guests, _ = extract( + "Maddie Denton (East Nash Grass) on Collaboration and Community", + self.pats, + ) + assert guests == ["Maddie Denton"] + + # -- Verb-based patterns -- + + def test_celebrates_verb(self): + """'Name Celebrates Something' — the verb signals end of the name.""" + guests, _ = extract( + "Trey Hensley Celebrates Flatt and Scruggs at Carnegie Hall", + self.pats, + ) + assert guests == ["Trey Hensley"] + + def test_lowercase_verb(self): + """Some titles use lowercase verbs: 'Tony Trischka has dinner with...'""" + guests, _ = extract( + "Tony Trischka has dinner with Bill Monroe", + self.pats, + ) + assert guests == ["Tony Trischka"] + + def test_returns_verb(self): + """'Jake Eddy returns' — just a name and a verb, no topic.""" + guests, _ = extract("Jake Eddy returns", self.pats) + assert guests == ["Jake Eddy"] + + # -- Multi-guest formats -- + + def test_and_separated_guests(self): + """Two guests joined by 'and' — should split into separate names.""" + guests, _ = extract( + "Martin Simpson & Thomm Jutz interview", + self.pats, + ) + # Both are 2-word names, so the & split should fire + assert "Martin Simpson" in guests + assert "Thomm Jutz" in guests + + # -- Celebration/tribute episodes -- + + def test_celebrating_with(self): + """'Celebrating X with Guest Name'.""" + guests, _ = extract( + "Celebrating IBMA with Jerry Douglas", + self.pats, + ) + assert guests == ["Jerry Douglas"] + + def test_tribute_featuring(self): + """Multi-part tribute episodes list guests after a dash.""" + guests, _ = extract( + "Earl Scruggs 100th Birthday Tribute part 2 - Jerry Douglas, Alison Brown and Tim O'Brien", + self.pats, + ) + # This comes through as a single string from the "tribute part" pattern + assert len(guests) >= 1 + assert any("Jerry Douglas" in g for g in guests) + + # -- Skip patterns: jam tracks and non-interview content -- + + def test_skip_bpm_jam_track(self): + """Jam-along tracks have BPM in the title — these aren't interviews.""" + _, skip = extract( + "Sally Goodin (A 75 bpm)", + self.pats, + ) + assert skip + + def test_skip_multi_tempo(self): + """Multi-tempo practice tracks: 'Tune in Key at N tempos'.""" + _, skip = extract( + "Big Sciota in G at 2 tempos - 75 bpm & 85 bpm", + self.pats, + ) + assert skip + + def test_skip_bluegrass_briefing(self): + """Bluegrass Briefing episodes are news roundups, not interviews.""" + _, skip = extract("Bluegrass Briefing - January 2026", self.pats) + assert skip + + def test_skip_mini_jam(self): + _, skip = extract("Mini Jam #42 - Salt Creek", self.pats) + assert skip + + def test_skip_update(self): + """Podcast updates aren't guest episodes.""" + _, skip = extract("A quick update on the podcast", self.pats) + assert skip + + def test_skip_milestone(self): + _, skip = extract("We reached 200 Episodes!", self.pats) + assert skip + + +# =================================================================== +# 2. BLUEGRASS UNLIMITED +# +# The simplest format of all: every single episode is titled +# "Bluegrass Unlimited Podcast with Guest Name". 100% hit rate. +# =================================================================== + +class TestBluegrassUnlimited: + """Bluegrass Unlimited — Dan Miller's podcast, perfectly consistent.""" + + @pytest.fixture(autouse=True) + def _load(self, all_patterns): + self.pats = all_patterns["Bluegrass Unlimited's Podcast"] + + def test_standard_format(self): + guests, _ = extract( + "Bluegrass Unlimited Podcast with Tim Stafford", + self.pats, + ) + assert guests == ["Tim Stafford"] + + def test_long_name(self): + guests, _ = extract( + "Bluegrass Unlimited Podcast with Kristin Scott Benson", + self.pats, + ) + assert guests == ["Kristin Scott Benson"] + + +# =================================================================== +# 3. WALLS OF TIME +# +# Format: "S1 E5. Guest Name: Topic" or "BONUS. Guest: Topic" +# Season previews (E0) are skipped. +# =================================================================== + +class TestWallsOfTime: + """Walls of Time — Daniel Mullins & Ty Gilpin's history podcast.""" + + @pytest.fixture(autouse=True) + def _load(self, all_patterns): + self.pats = all_patterns["Walls of Time: Bluegrass Podcast"] + + def test_standard_season_episode(self): + """Most common format: 'S1 E5. Guest: Topic about bluegrass history'.""" + guests, _ = extract( + "S3 E4. Tim Stafford: The Bluegrass Hall of Fame", + self.pats, + ) + assert guests == ["Tim Stafford"] + + def test_bonus_episode(self): + guests, _ = extract( + "BONUS. Ricky Skaggs: Early Days in Kentucky", + self.pats, + ) + assert guests == ["Ricky Skaggs"] + + def test_skip_season_preview(self): + """E0 episodes are season previews/trailers.""" + _, skip = extract("S3 E0. Season 3 Preview", self.pats) + assert skip + + +# =================================================================== +# 4. TOY HEART +# +# Tom Power's podcast for The Bluegrass Situation. Most titles +# are just the guest's name, which is elegant but means we need +# to be careful not to match season announcements. +# =================================================================== + +class TestToyHeart: + """Toy Heart — Tom Power's interview podcast.""" + + @pytest.fixture(autouse=True) + def _load(self, all_patterns): + self.pats = all_patterns["Toy Heart with Tom Power"] + + def test_just_a_name(self): + """Many episodes are titled with just the guest's name.""" + guests, _ = extract("Sierra Hull", self.pats) + assert guests == ["Sierra Hull"] + + def test_name_with_accents(self): + """French-Canadian and other non-ASCII names should work.""" + guests, _ = extract("Yves Lambert", self.pats) + assert guests == ["Yves Lambert"] + + def test_skip_season_announcement(self): + _, skip = extract("Season 3 is coming!", self.pats) + assert skip + + def test_skip_preview(self): + _, skip = extract( + "Preview - Toy Heart: A Podcast About Bluegrass", self.pats + ) + assert skip + + def test_bluegrass_breakdown_crossover(self): + """A crossover episode with another podcast, guest in the middle.""" + guests, _ = extract( + "The Bluegrass Breakdown Podcast - Keith Whitley and Ricky Skaggs - Second Generation Bluegrass", + self.pats, + ) + assert len(guests) >= 1 + assert any("Keith Whitley" in g for g in guests) + + +# =================================================================== +# 5. WHAT'S THE REASON FOR THIS PODCAST +# +# Kodi Nottingham's podcast. Title format varies by season: +# - "What's The Reason For This Podcast - Guest" +# - "What's The Reason For This Season 2 Episode 5 - Guest" +# - "What's The Reason For This Session - N - Band Name" +# Note the inconsistent apostrophe (sometimes missing). +# =================================================================== + +class TestWhatsTheReason: + """What's The Reason — Kodi Nottingham's bluegrass podcast.""" + + @pytest.fixture(autouse=True) + def _load(self, all_patterns): + self.pats = all_patterns["What's The Reason For This Podcast"] + + def test_standard_format(self): + guests, _ = extract( + "What's The Reason For This Podcast - Billy Strings - The Interview", + self.pats, + ) + assert guests == ["Billy Strings"] + + def test_season_episode_format(self): + guests, _ = extract( + "What's The Reason For This Season 2 Episode 5 - Molly Tuttle", + self.pats, + ) + assert guests == ["Molly Tuttle"] + + def test_session_format(self): + """'Session' episodes feature live performances by bands.""" + guests, _ = extract( + "What's The Reason For This Session - 1 - Sicard Hollow", + self.pats, + ) + assert guests == ["Sicard Hollow"] + + def test_missing_apostrophe(self): + """Some titles drop the apostrophe — 'Whats' instead of 'What's'.""" + guests, _ = extract( + "Whats The Reason For This Session 5 - The Pickpockets", + self.pats, + ) + assert guests == ["The Pickpockets"] + + +# =================================================================== +# 6. PICKY FINGERS BANJO PODCAST +# +# Keith Billik's banjo-focused podcast. Numbering with #NNN prefix. +# Several formats: +# - "#139 - Guest Name talks about..." +# - '#51 - "Song Title" by Artist' +# - '#124 - "Album" Revisited! Feat. Guest & Guest' +# - "BONUS - Guest Name..." +# =================================================================== + +class TestPickyFingers: + """The Picky Fingers Banjo Podcast — Keith Billik.""" + + @pytest.fixture(autouse=True) + def _load(self, all_patterns): + self.pats = all_patterns["The Picky Fingers Banjo Podcast"] + + def test_standard_numbered(self): + guests, _ = extract( + "#140 - Tony Trischka on the State of Banjo", + self.pats, + ) + assert guests == ["Tony Trischka"] + + def test_song_by_artist(self): + """'#51 - "Song" by Artist' format — artist after 'by'.""" + guests, _ = extract( + '#51 - "Prime Time" by Crary, Evans & Barnick', + self.pats, + ) + assert len(guests) >= 1 + + def test_feat_format(self): + """'feat.' introduces additional guests.""" + guests, _ = extract( + '#139 - "Labor of Lust" feat. Kyle Tuttle', + self.pats, + ) + assert any("Kyle Tuttle" in g for g in guests) + + def test_bonus_episode(self): + guests, _ = extract( + "BONUS - Fireside Chat w/ Noam Pikelny", + self.pats, + ) + assert guests == ["Noam Pikelny"] + + def test_in_memoriam(self): + """In Memoriam episodes still extract the person's name.""" + guests, _ = extract( + "#100 - In Memoriam: Ralph Stanley", + self.pats, + ) + assert guests == ["Ralph Stanley"] + + +# =================================================================== +# 7. GRASS TALK RADIO +# +# Bradley Laird's podcast is the trickiest. Most episodes are +# topic-based monologues ("GTR-157 - Chord Progressions") with +# only occasional interviews. We only extract when "Interview" +# appears in the title to avoid false positives on topic titles +# that happen to be in Title Case. +# =================================================================== + +class TestGrassTalkRadio: + """Grass Talk Radio — Bradley Laird's mandolin & bluegrass podcast.""" + + @pytest.fixture(autouse=True) + def _load(self, all_patterns): + self.pats = all_patterns["Grass Talk Radio"] + + def test_interview_with_dash(self): + guests, _ = extract( + "GTR-180 - Tony Williamson Interview", + self.pats, + ) + assert guests == ["Tony Williamson"] + + def test_interview_no_dash(self): + guests, _ = extract( + "GTR-100 Buddy Ashmore Interview", + self.pats, + ) + assert guests == ["Buddy Ashmore"] + + def test_no_false_positive_on_topic(self): + """Title Case topic titles must NOT be extracted as guest names. + 'Merry Christmas' and 'Personal Feedback' aren't people!""" + guests, skip = extract("GTR-200 Merry Christmas", self.pats) + assert guests == [] + + def test_no_false_positive_topic_with_dash(self): + guests, skip = extract("GTR-155 - The Mail Bag", self.pats) + assert guests == [] + + def test_no_false_positive_metaphor(self): + """'Uncle Rico's Time Machine' is a topic, not a guest.""" + guests, _ = extract("GTR-199 Uncle Rico's Time Machine", self.pats) + assert guests == [] + + +# =================================================================== +# 8. BLUEGRASS BKLYN +# +# Mike Willner & Liz Wolfe's NYC bluegrass podcast. Uses season +# numbering with segment prefixes like "Women in Bluegrass", +# "Home Grown, Locally Known", "On the Road", "Iconic Venues". +# Some episodes are pure topic discussions (skip). +# =================================================================== + +class TestBluegrassBKLYN: + """Bluegrass BKLYN — NYC bluegrass scene podcast.""" + + @pytest.fixture(autouse=True) + def _load(self, all_patterns): + self.pats = all_patterns["Bluegrass BKLYN"] + + def test_women_in_bluegrass_segment(self): + guests, _ = extract( + "S4M9 Women in Bluegrass - Avril Smith!", + self.pats, + ) + assert guests == ["Avril Smith"] + + def test_home_grown_segment(self): + guests, _ = extract( + "S3E8 Home Grown, Locally Known with Rick Snell!", + self.pats, + ) + assert guests == ["Rick Snell"] + + def test_plain_guest_name(self): + """Some episodes just have 'S3E6 Guest Name!'""" + guests, _ = extract( + "S3E6 Martha Spencer and Lucas Pasley!", + self.pats, + ) + assert len(guests) >= 1 + assert any("Martha Spencer" in g for g in guests) + + def test_skip_wrap_up(self): + """Year-end wrap-ups aren't guest episodes.""" + _, skip = extract("S4E9 2025 Wrap Up!", self.pats) + assert skip or extract("S4E9 2025 Wrap Up!", self.pats)[0] == [] + + def test_skip_topic_episode(self): + """Topic discussion episodes without guests.""" + _, skip = extract("S4M7 Let's Talk Music Education!", self.pats) + assert skip or extract("S4M7 Let's Talk Music Education!", self.pats)[0] == [] + + +# =================================================================== +# 9. BLUEGRASS AMBASSADORS +# +# Henhouse Prowlers' podcast about global bluegrass. Simple format: +# "Episode N - Guest: Topic" +# =================================================================== + +class TestBluegrassAmbassadors: + """Bluegrass Ambassadors — Henhouse Prowlers.""" + + @pytest.fixture(autouse=True) + def _load(self, all_patterns): + self.pats = all_patterns["Bluegrass Ambassadors"] + + def test_standard_format(self): + guests, _ = extract( + "Episode 2 - Ketch Secor: Old Crow Medicine Show Goes Global", + self.pats, + ) + assert guests == ["Ketch Secor"] + + def test_colon_separator(self): + guests, _ = extract( + "Episode 1 - Vijit Malik: Bringing Doc Watson to Dubai", + self.pats, + ) + assert guests == ["Vijit Malik"] + + +# =================================================================== +# 10. EDGE CASES & CROSS-CUTTING CONCERNS +# =================================================================== + +class TestEdgeCases: + """Things that have tripped us up across multiple podcasts.""" + + @pytest.fixture(autouse=True) + def _load(self, all_patterns): + self.patterns = all_patterns + + def test_no_patterns_returns_empty(self): + """Podcasts with empty pattern lists (music shows, no guests) + should return no guests and not skip.""" + guests, skip = extract("Any Title At All", []) + assert guests == [] + assert not skip + + def test_possessive_name(self): + """Names ending in 's (possessive) — the 's should be kept + as part of the name when it's 'Guest's Topic'.""" + pats = self.patterns["Bluegrass Jam Along"] + guests, _ = extract( + "Mike Marshall & Chris Thile's Into the Cauldron 20th Anniversary Celebration - part 1 with Mike Marshall", + pats, + ) + # Should extract something, not crash + assert isinstance(guests, list) + + def test_pattern_file_is_valid_json(self): + """The pattern file must be valid JSON — broken JSON means + the whole pipeline fails.""" + with open(PATTERNS_FILE) as f: + data = json.load(f) + assert "patterns" in data + assert isinstance(data["patterns"], dict) + + def test_all_patterns_compile(self): + """Every regex in the config must compile without errors.""" + with open(PATTERNS_FILE) as f: + data = json.load(f) + for podcast, pats in data["patterns"].items(): + for i, p in enumerate(pats): + try: + re.compile(p["pattern"]) + except re.error as e: + pytest.fail( + f"Bad regex in {podcast}[{i}]: {e}\n" + f"Pattern: {p['pattern']}" + ) + + def test_non_skip_patterns_have_guest_group(self): + """Every non-skip pattern should have a (?P...) group, + otherwise we'll match but extract nothing.""" + with open(PATTERNS_FILE) as f: + data = json.load(f) + for podcast, pats in data["patterns"].items(): + for i, p in enumerate(pats): + if p.get("skip"): + continue + assert "(?P" in p["pattern"], ( + f"{podcast}[{i}] is not marked skip but has no " + f"(?P...) group: {p['pattern']}" + ) + + +# =================================================================== +# 11. PAGE TITLE GENERATION +# +# Episode pages live under the podcast name as subpages: +# "Bluegrass Jam Along/Episode Title Here" +# We need to sanitize titles for MediaWiki constraints. +# =================================================================== + +class TestPageTitleGeneration: + """Wiki page title generation from episode titles.""" + + def _make_title(self, podcast, episode_title): + """Mirrors make_page_title() from podcast-episodes.py.""" + safe = re.sub(r'[#<>\[\]|{}]', '', episode_title).strip() + if len(safe) > 120: + safe = safe[:120].rsplit(' ', 1)[0] + return f"{podcast}/{safe}" + + def test_basic_title(self): + t = self._make_title("Bluegrass Jam Along", "Tony Trischka on Banjos") + assert t == "Bluegrass Jam Along/Tony Trischka on Banjos" + + def test_strips_wiki_chars(self): + """MediaWiki forbids #<>[]|{} in page titles.""" + t = self._make_title("Test", "Episode [1] with ") + assert "[" not in t + assert "<" not in t + + def test_truncates_long_titles(self): + """Very long episode titles get truncated at a word boundary.""" + long_title = "A " * 100 # 200 chars + t = self._make_title("Test", long_title) + # The part after "Test/" should be <= 120 chars + assert len(t.split("/", 1)[1]) <= 120 From fee68ffb80cb66a0e196e78b213e32ccb86fc20e Mon Sep 17 00:00:00 2001 From: magent Date: Sun, 1 Feb 2026 04:23:51 +0000 Subject: [PATCH 10/10] Rename would_save to saved_pages and clarify test comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DryRunClient.would_save was ambiguous — it sounded hypothetical but actually records every save_page() call. Renamed to saved_pages. Also replaced should/would in test comments with direct descriptions of what the test asserts. --- .../blue_railroad_import/wiki_client.py | 8 ++--- .../tests/test_importer.py | 30 +++++++++---------- .../tests/test_leaderboard.py | 8 ++--- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tools/blue-railroad-import/blue_railroad_import/wiki_client.py b/tools/blue-railroad-import/blue_railroad_import/wiki_client.py index fcf9486..11feda8 100644 --- a/tools/blue-railroad-import/blue_railroad_import/wiki_client.py +++ b/tools/blue-railroad-import/blue_railroad_import/wiki_client.py @@ -84,22 +84,22 @@ class DryRunClient: def __init__(self, existing_pages: Optional[dict[str, str]] = None): self.existing_pages = existing_pages or {} - self.would_save: list[tuple[str, str, str]] = [] + self.saved_pages: list[tuple[str, str, str]] = [] def get_page_content(self, title: str) -> Optional[str]: return self.existing_pages.get(title) def save_page(self, title: str, content: str, summary: str) -> SaveResult: - self.would_save.append((title, content, summary)) + self.saved_pages.append((title, content, summary)) existed = title in self.existing_pages current = self.existing_pages.get(title) if current == content: - return SaveResult(title, 'unchanged', 'Would be unchanged') + return SaveResult(title, 'unchanged', 'Content identical (dry run)') action = 'updated' if existed else 'created' - return SaveResult(title, action, f'Would be {action} (dry run)') + return SaveResult(title, action, f'{action} (dry run)') def page_exists(self, title: str) -> bool: return title in self.existing_pages diff --git a/tools/blue-railroad-import/tests/test_importer.py b/tools/blue-railroad-import/tests/test_importer.py index ced4b0d..92d9042 100644 --- a/tools/blue-railroad-import/tests/test_importer.py +++ b/tools/blue-railroad-import/tests/test_importer.py @@ -131,7 +131,7 @@ def test_aggregates_tokens_from_all_sources(self, chain_data_file, wiki_config_c config = importer.load_config() tokens = importer.load_tokens(config) - # Should have 3 tokens total (2 from V1, 1 from V2) + # 3 tokens total: 2 from V1, 1 from V2 assert len(tokens) == 3 def test_run_creates_token_pages(self, chain_data_file, wiki_config_content): @@ -142,11 +142,11 @@ def test_run_creates_token_pages(self, chain_data_file, wiki_config_content): stats = importer.run() - # Should create 3 token pages + # 3 token pages created assert stats.tokens_created == 3 - # Check pages would be saved - saved_titles = [title for title, _, _ in wiki.would_save] + # Verify save_page() was called for each token + saved_titles = [title for title, _, _ in wiki.saved_pages] assert 'Blue Railroad Token 1' in saved_titles assert 'Blue Railroad Token 2' in saved_titles assert 'Blue Railroad Token 5' in saved_titles @@ -159,10 +159,10 @@ def test_run_creates_leaderboards(self, chain_data_file, wiki_config_content): stats = importer.run() - # Should create 2 leaderboards + # 2 leaderboards created assert stats.leaderboards_created == 2 - saved_titles = [title for title, _, _ in wiki.would_save] + saved_titles = [title for title, _, _ in wiki.saved_pages] assert 'Blue Railroad Leaderboard' in saved_titles assert 'Blue Railroad Squats Leaderboard' in saved_titles @@ -177,7 +177,7 @@ def test_skips_unchanged_pages(self, chain_data_file, wiki_config_content): importer.run() # Get the generated content - generated_pages = {title: content for title, content, _ in wiki.would_save} + generated_pages = {title: content for title, content, _ in wiki.saved_pages} # Second run with pre-existing content wiki2 = DryRunClient(existing_pages={ @@ -187,7 +187,7 @@ def test_skips_unchanged_pages(self, chain_data_file, wiki_config_content): importer2 = BlueRailroadImporter(wiki2, chain_data_file) stats2 = importer2.run() - # Everything should be unchanged + # All pages match existing content — nothing to update assert stats2.tokens_created == 0 assert stats2.tokens_updated == 0 assert stats2.tokens_unchanged == 3 @@ -199,7 +199,7 @@ class TestLeaderboardAggregationIntegration: Integration test for the critical aggregation fix. This tests the exact scenario that caused the bot loop: - multiple sources should be aggregated BEFORE generating leaderboards, + multiple sources must be aggregated BEFORE generating leaderboards, not generate separate leaderboards per source. """ @@ -212,23 +212,23 @@ def test_leaderboard_includes_all_sources(self, chain_data_file, wiki_config_con # Find the overall leaderboard content leaderboard_content = None - for title, content, _ in wiki.would_save: + for title, content, _ in wiki.saved_pages: if title == 'Blue Railroad Leaderboard': leaderboard_content = content break assert leaderboard_content is not None - # Alice should have 2 tokens (1 from V1, 1 from V2) + # Alice holds 2 tokens (1 from V1, 1 from V2) # The row format is: | rank || holder || count || token links assert '| 1 || alice.eth || 2 ||' in leaderboard_content - # Token IDs from both sources should be present + # Token IDs from both sources appear in the leaderboard assert '#1]]' in leaderboard_content # V1 token assert '#5]]' in leaderboard_content # V2 token def test_multiple_runs_produce_identical_output(self, chain_data_file, wiki_config_content): - """Idempotency test - running twice should produce same content.""" + """Idempotency: two runs with identical input produce identical output.""" wiki1 = DryRunClient(existing_pages={ 'PickiPedia:BlueRailroadConfig': wiki_config_content, }) @@ -243,7 +243,7 @@ def test_multiple_runs_produce_identical_output(self, chain_data_file, wiki_conf importer2.run() # Get leaderboard content from both runs - lb1 = next(c for t, c, _ in wiki1.would_save if t == 'Blue Railroad Leaderboard') - lb2 = next(c for t, c, _ in wiki2.would_save if t == 'Blue Railroad Leaderboard') + lb1 = next(c for t, c, _ in wiki1.saved_pages if t == 'Blue Railroad Leaderboard') + lb2 = next(c for t, c, _ in wiki2.saved_pages if t == 'Blue Railroad Leaderboard') assert lb1 == lb2 diff --git a/tools/blue-railroad-import/tests/test_leaderboard.py b/tools/blue-railroad-import/tests/test_leaderboard.py index 74c0102..5b0ee58 100644 --- a/tools/blue-railroad-import/tests/test_leaderboard.py +++ b/tools/blue-railroad-import/tests/test_leaderboard.py @@ -174,7 +174,7 @@ class TestLeaderboardAggregation: """ def test_aggregated_tokens_include_all_sources(self, sample_tokens): - """Leaderboard should include BOTH V1 and V2 tokens.""" + """Leaderboard includes BOTH V1 and V2 tokens.""" config = LeaderboardConfig(page='Test') content = generate_leaderboard_content(sample_tokens, config) @@ -182,16 +182,16 @@ def test_aggregated_tokens_include_all_sources(self, sample_tokens): assert '| 1 || alice.eth || 3 ||' in content def test_token_ids_sorted_numerically(self, sample_tokens): - """Token IDs in leaderboard should be sorted numerically.""" + """Token IDs in leaderboard are sorted numerically.""" config = LeaderboardConfig(page='Test') content = generate_leaderboard_content(sample_tokens, config) - # Alice's tokens should be 1, 2, 5 in order + # Alice's tokens listed as 1, 2, 5 in numeric order alice_row = [line for line in content.split('\n') if 'alice.eth' in line][0] assert '#1]], [[Blue Railroad Token 2|#2]], [[Blue Railroad Token 5|#5]]' in alice_row def test_idempotent_content_generation(self, sample_tokens): - """Same input should always produce same output.""" + """Same input always produces same output.""" config = LeaderboardConfig(page='Test', sort='count') content1 = generate_leaderboard_content(sample_tokens, config)