Add natural language search for Premium Archive

apps/reader/views.py

@@ -1112,7 +1112,13 @@ def load_single_feed(request, feed_id):
         if user.profile.is_premium:
             user_search = MUserSearch.get_user(user.pk)
             user_search.touch_search_date()
-            stories = feed.find_stories(query, order=order, offset=offset, limit=limit)
+            search_type = "keyword"
+            if user.profile.is_archive:
+                user_search.touch_discover_date()
+                search_type = "hybrid" if user_search.discover_indexed else "keyword"
+            stories = feed.find_stories(
+                query, order=order, offset=offset, limit=limit, search_type=search_type
+            )
         else:
             stories = []
             message = "You must be a premium subscriber to search."
@@ -1455,6 +1461,11 @@ def load_single_feed(request, feed_id):
         message=message,
     )
 
+    if query and user.profile.is_premium:
+        data["semantic_search"] = search_type == "hybrid"
+        if not data["semantic_search"] and user_search and user_search.discover_indexing:
+            data["semantic_indexing"] = True
+
     if include_feeds:
         data["feeds"] = feeds
     if not include_hidden:
@@ -2307,12 +2318,18 @@ def load_river_stories__redis(request):
         if user.profile.is_premium:
             user_search = MUserSearch.get_user(user.pk)
             user_search.touch_search_date()
+            search_type = "keyword"
+            if user.profile.is_archive:
+                user_search.touch_discover_date()
+                search_type = "hybrid" if user_search.discover_indexed else "keyword"
             usersubs = UserSubscription.subs_for_feeds(user.pk, feed_ids=feed_ids, read_filter="all")
             feed_ids = [sub.feed_id for sub in usersubs]
             feed_ids = Feed.exclude_briefing_feeds(feed_ids)
             if infrequent:
                 feed_ids = Feed.low_volume_feeds(feed_ids, stories_per_month=infrequent)
-            stories = Feed.find_feed_stories(feed_ids, query, order=order, offset=offset, limit=limit)
+            stories = Feed.find_feed_stories(
+                feed_ids, query, order=order, offset=offset, limit=limit, search_type=search_type
+            )
             mstories = stories
             unread_feed_story_hashes = UserSubscription.story_hashes(
                 user.pk,
@@ -2670,6 +2687,11 @@ def load_river_stories__redis(request):
         user_profiles=user_profiles,
     )
 
+    if query and user.profile.is_premium:
+        data["semantic_search"] = search_type == "hybrid"
+        if not data["semantic_search"] and user_search and user_search.discover_indexing:
+            data["semantic_indexing"] = True
+
     if include_feeds:
         data["feeds"] = feeds
     if not include_hidden:
@@ -4170,7 +4192,16 @@ def all_classifiers(request):
     from collections import defaultdict
 
     classifiers_by_feed = defaultdict(
-        lambda: {"titles": [], "authors": [], "tags": [], "texts": [], "feeds": [], "urls": [], "prompts": [], "image_prompts": []}
+        lambda: {
+            "titles": [],
+            "authors": [],
+            "tags": [],
+            "texts": [],
+            "feeds": [],
+            "urls": [],
+            "prompts": [],
+            "image_prompts": [],
+        }
     )
 
     # Separate scoped classifiers (global/folder) from feed-scoped ones

apps/rss_feeds/models.py

@@ -2178,12 +2178,24 @@ def get_stories(
         return stories
 
     @classmethod
-    def find_feed_stories(cls, feed_ids, query, order="newest", offset=0, limit=25):
-        story_ids = SearchStory.query(feed_ids=feed_ids, query=query, order=order, offset=offset, limit=limit)
-        stories_db = MStory.objects(story_hash__in=story_ids).order_by(
-            "-story_date" if order == "newest" else "story_date"
-        )
-        stories = cls.format_stories(stories_db)
+    def find_feed_stories(cls, feed_ids, query, order="newest", offset=0, limit=25, search_type="keyword"):
+        if search_type == "hybrid":
+            story_ids = SearchStory.hybrid_query(
+                feed_ids=feed_ids, query=query, order=order, offset=offset, limit=limit
+            )
+        else:
+            story_ids = SearchStory.query(
+                feed_ids=feed_ids, query=query, order=order, offset=offset, limit=limit
+            )
+        stories_db = MStory.objects(story_hash__in=story_ids)
+        if search_type == "hybrid":
+            # Preserve relevance ordering from hybrid_query
+            story_order = {sid: i for i, sid in enumerate(story_ids)}
+            stories = cls.format_stories(stories_db)
+            stories.sort(key=lambda s: story_order.get(s["story_hash"], len(story_ids)))
+        else:
+            stories_db = stories_db.order_by("-story_date" if order == "newest" else "story_date")
+            stories = cls.format_stories(stories_db)
 
         return stories
 
@@ -2543,15 +2555,23 @@ def xls_query_popularity(cls, queries, limit):
         workbook.close()
         return title
 
-    def find_stories(self, query, order="newest", offset=0, limit=25):
-        story_ids = SearchStory.query(
-            feed_ids=[self.pk], query=query, order=order, offset=offset, limit=limit
-        )
-        stories_db = MStory.objects(story_hash__in=story_ids).order_by(
-            "-story_date" if order == "newest" else "story_date"
-        )
-
-        stories = self.format_stories(stories_db, self.pk)
+    def find_stories(self, query, order="newest", offset=0, limit=25, search_type="keyword"):
+        if search_type == "hybrid":
+            story_ids = SearchStory.hybrid_query(
+                feed_ids=[self.pk], query=query, order=order, offset=offset, limit=limit
+            )
+        else:
+            story_ids = SearchStory.query(
+                feed_ids=[self.pk], query=query, order=order, offset=offset, limit=limit
+            )
+        stories_db = MStory.objects(story_hash__in=story_ids)
+        if search_type == "hybrid":
+            story_order = {sid: i for i, sid in enumerate(story_ids)}
+            stories = self.format_stories(stories_db, self.pk)
+            stories.sort(key=lambda s: story_order.get(s["story_hash"], len(story_ids)))
+        else:
+            stories_db = stories_db.order_by("-story_date" if order == "newest" else "story_date")
+            stories = self.format_stories(stories_db, self.pk)
 
         return stories
 

apps/search/models.py

@@ -569,6 +569,110 @@ def query(cls, feed_ids, query, order, offset, limit, strip=False):
 
         return result_ids
 
+    @classmethod
+    def generate_query_embedding(cls, query_text):
+        """Generate a 256-dim projected embedding for a search query string."""
+        model_name = "text-embedding-3-small"
+        client = OpenAI(api_key=settings.OPENAI_API_KEY)
+
+        try:
+            response = client.embeddings.create(model=model_name, input=query_text.lower())
+        except APITimeoutError as e:
+            logging.debug(f" ***> ~FROpenAI API timeout for query embedding: {e}")
+            return None
+        except Exception as e:
+            logging.debug(f" ***> ~FROpenAI API error for query embedding: {e}")
+            return None
+
+        LLMCostTracker.record_embedding(
+            model=model_name,
+            input_tokens=response.usage.total_tokens,
+            feature="story_search_query_embedding",
+            metadata={"query": query_text[:100]},
+        )
+
+        query_embedding = response.data[0].embedding
+        projected_embedding = project_vector(query_embedding)
+
+        return projected_embedding.tolist()
+
+    @classmethod
+    def hybrid_query(cls, feed_ids, query, order, offset, limit, text_weight=0.6, semantic_weight=0.4):
+        """Combine keyword search with semantic vector search for story discovery.
+
+        Returns a list of story_hash IDs ranked by combined relevance score.
+        Falls back to keyword-only results if the semantic component fails.
+        """
+        results = {}
+
+        # Part 1: Keyword search
+        keyword_ids = cls.query(feed_ids, query, order, offset, limit)
+        if keyword_ids:
+            max_rank = len(keyword_ids)
+            for i, story_hash in enumerate(keyword_ids):
+                # Position-based score: first result = 1.0, last = near 0
+                normalized_score = (max_rank - i) / max_rank
+                results[story_hash] = {
+                    "text_score": normalized_score,
+                    "semantic_score": 0,
+                }
+
+        # Part 2: Semantic search
+        try:
+            query_vector = cls.generate_query_embedding(query)
+            if query_vector:
+                semantic_results = DiscoverStory.vector_query(
+                    query_vector,
+                    feed_ids_to_include=feed_ids[:2000],
+                    max_results=limit,
+                    offset=0,
+                )
+                if semantic_results:
+                    max_semantic_score = max(hit["_score"] for hit in semantic_results)
+                    for hit in semantic_results:
+                        story_hash = hit["_id"]
+                        # Normalize semantic score to 0-1 range
+                        semantic_score = (
+                            (hit["_score"] - 1.0) / (max_semantic_score - 1.0)
+                            if max_semantic_score > 1.0
+                            else 0
+                        )
+                        if story_hash in results:
+                            results[story_hash]["semantic_score"] = semantic_score
+                        else:
+                            results[story_hash] = {
+                                "text_score": 0,
+                                "semantic_score": semantic_score,
+                            }
+        except Exception as e:
+            logging.debug(f" ***> ~FRSemantic search failed, using keyword-only: {e}")
+
+        # Combine and rank
+        combined = []
+        for story_hash, scores in results.items():
+            combined_score = (scores["text_score"] * text_weight) + (
+                scores["semantic_score"] * semantic_weight
+            )
+            combined.append((story_hash, combined_score))
+
+        combined.sort(key=lambda x: -x[1])
+
+        result_ids = [story_hash for story_hash, _ in combined]
+
+        logging.info(
+            " ---> ~FG~SNHybrid search ~FCstories~FG for: ~SB%s~SN, ~SB%s~SN keyword + ~SB%s~SN semantic = ~SB%s~SN combined (across %s feed%s)"
+            % (
+                query,
+                len(keyword_ids),
+                len(results) - len(keyword_ids),
+                len(result_ids),
+                len(feed_ids),
+                "s" if len(feed_ids) != 1 else "",
+            )
+        )
+
+        return result_ids
+
     @classmethod
     def query_briefing_custom(cls, feed_ids, phrase, date_start, date_end, limit=10):
         """Search stories by exact phrase within a date range, for briefing custom sections.

clients/ios/Classes/PremiumView.swift

@@ -139,7 +139,8 @@ struct PremiumView: View {
         PremiumFeature(title: "Filter stories by date range", icon: "calendar", iconColor: .pink),
         PremiumFeature(title: "Apply training across a folder", icon: "folder.fill", iconColor: .mint),
         PremiumFeature(title: "Apply training globally", icon: "globe", iconColor: .indigo),
-        PremiumFeature(title: "Web Feeds from any website, even without RSS", icon: "web-feed-100", iconColor: .purple, isCustomImage: true)
+        PremiumFeature(title: "Web Feeds from any website, even without RSS", icon: "web-feed-100", iconColor: .purple, isCustomImage: true),
+        PremiumFeature(title: "Natural language search by near terms", icon: "magnifyingglass.circle.fill", iconColor: .teal)
     ]
 
     @available(iOS 15.0, *)
@@ -149,7 +150,8 @@ struct PremiumView: View {
             PremiumFeature(title: "Follow up to 10,000 sites", icon: "square.stack.3d.up.fill", iconColor: .green),
             PremiumFeature(title: "All feeds fetched every 5-15 minutes", icon: "bolt.fill", iconColor: PremiumColors.proOrange),
             PremiumFeature(title: "Train stories with regular expressions", icon: "textformat.abc", iconColor: .yellow),
-            PremiumFeature(title: "Priority support", icon: "headphones", iconColor: .yellow)
+            PremiumFeature(title: "Priority support", icon: "headphones", iconColor: .yellow),
+            PremiumFeature(title: "Natural language filters", icon: "text.magnifyingglass", iconColor: .gray, isUpcoming: true)
         ]
     }
 

media/css/pages/welcome.css

@@ -1789,6 +1789,12 @@ a.NB-fgi img {
     /* indigo-purple */
 }
 
+.NB-pricing-tier-archive .NB-pricing-tier-features li.NB-15 .NB-pricing-bullet {
+    background-image: url('/media/embed/icons/nouns/search-filled.svg');
+    filter: invert(55%) sepia(75%) saturate(1500%) hue-rotate(140deg) brightness(95%) contrast(95%);
+    /* teal */
+}
+
 /* Pro tier bullets with vivid colors */
 .NB-pricing-tier-pro .NB-pricing-tier-features li.NB-1 .NB-pricing-bullet {
     background-image: url('/media/embed/icons/nouns/stack.svg');

media/css/reader/darkmode.css

@@ -2048,6 +2048,10 @@ border-bottom-color: #303739;
     background-color: rgb(43, 49, 51);
 }
 
+.NB-dark .NB-search-semantic-badge {
+    background-color: #6C5CE7;
+}
+
 .NB-dark .NB-story-content-wrapper .NB-story-content-expander {
     background-color: #191b1c;
 }

media/css/reader/reader.css

@@ -21349,6 +21349,12 @@ h5.NB-classifier-section-header {
     /* indigo-purple */
 }
 
+.NB-premium-tier-features-archive li.NB-15 .NB-premium-bullet-image {
+    background-image: url('/media/embed/icons/nouns/search-filled.svg');
+    filter: invert(55%) sepia(75%) saturate(1500%) hue-rotate(140deg) brightness(95%) contrast(95%);
+    /* teal */
+}
+
 /* Pro tier icons with individual vivid colors */
 .NB-premium-tier-features-pro li.NB-1 .NB-premium-bullet-image {
     background-image: url('/media/embed/icons/nouns/stack.svg');
@@ -26053,6 +26059,20 @@ h5.NB-classifier-section-header {
     background-color: transparent;
 }
 
+.NB-search-semantic-badge {
+    display: inline-block;
+    font-size: 9px;
+    font-weight: bold;
+    text-transform: uppercase;
+    color: #fff;
+    background-color: #7B68EE;
+    padding: 1px 4px;
+    border-radius: 3px;
+    margin-left: 4px;
+    vertical-align: middle;
+    letter-spacing: 0.5px;
+}
+
 .NB-search-header .NB-search-header-icon {
     background: transparent url('/media/embed/reader/search_light.png') no-repeat 0 0;
     background-size: 16px;

media/js/newsblur/reader/reader.js

@@ -7629,6 +7629,11 @@
                 var progress = total ? Math.ceil(indexed / total * 100) : 0;
                 NEWSBLUR.app.active_search.show_search_indexing_banner(progress);
             }
+
+            NEWSBLUR.reader.flags.semantic_search = !!data.semantic_search;
+            if (NEWSBLUR.app.feed_search_header) {
+                NEWSBLUR.app.feed_search_header.render();
+            }
         },
 
         add_recommended_feed: function (feed_id) {

media/js/newsblur/reader/reader_premium_upgrade.js

@@ -180,6 +180,10 @@ _.extend(NEWSBLUR.ReaderPremiumUpgrade.prototype, {
                         $.make('li', { className: 'NB-14' }, [
                             $.make('div', { className: 'NB-premium-bullet-image' }),
                             'Web Feeds from any website, even without RSS'
+                        ]),
+                        $.make('li', { className: 'NB-15' }, [
+                            $.make('div', { className: 'NB-premium-bullet-image' }),
+                            'Natural language search by near terms'
                         ])
                     ]),
                     $.make('div', { className: 'NB-premium-tier-actions' }, [
@@ -212,6 +216,11 @@ _.extend(NEWSBLUR.ReaderPremiumUpgrade.prototype, {
                         $.make('li', { className: 'NB-4' }, [
                             $.make('div', { className: 'NB-premium-bullet-image' }),
                             'Priority support'
+                        ]),
+                        $.make('li', { className: 'NB-premium-tier-upcoming-header' }, 'Coming soon:'),
+                        $.make('li', { className: 'NB-upcoming NB-5' }, [
+                            $.make('div', { className: 'NB-premium-bullet-image' }),
+                            'Natural language filters'
                         ])
                     ]),
                     $.make('div', { className: 'NB-premium-tier-actions' }, [

media/js/newsblur/views/feed_search_header.js

@@ -87,12 +87,14 @@ NEWSBLUR.Views.FeedSearchHeader = Backbone.View.extend({
             </div>');
             return $view;
         } else if (searching) {
+            var semantic_label = NEWSBLUR.reader.flags.semantic_search ? ' <span class="NB-search-semantic-badge">AI</span>' : '';
             var $view = $(_.template('<div>\
                 Searching \
-                <b><%= feed_title %></b> for "<b><%= query %></b>"\
+                <b><%= feed_title %></b> for "<b><%= query %></b>"<%= semantic_label %>\
             </div>', {
                 feed_title: feed_title,
-                query: NEWSBLUR.reader.flags.search
+                query: NEWSBLUR.reader.flags.search,
+                semantic_label: semantic_label
             }));
             return $view;
         } else if (date_filter_start || date_filter_end) {

templates/reader/welcome.xhtml

@@ -442,6 +442,7 @@
                         <li class="NB-12"><div class="NB-pricing-bullet"></div>Apply training across a folder</li>
                         <li class="NB-13"><div class="NB-pricing-bullet"></div>Apply training globally</li>
                         <li class="NB-14"><div class="NB-pricing-bullet"></div>Web Feeds from any website, even without RSS</li>
+                        <li class="NB-15"><div class="NB-pricing-bullet"></div>Natural language search by near terms</li>
                     </ul>
                 </div>
 
@@ -459,6 +460,8 @@
                         <li class="NB-2"><div class="NB-pricing-bullet"></div>All feeds fetched every 5-15 minutes</li>
                         <li class="NB-3"><div class="NB-pricing-bullet"></div>Train stories with regular expressions</li>
                         <li class="NB-4"><div class="NB-pricing-bullet"></div>Priority support</li>
+                        <li class="NB-pricing-tier-upcoming-header">Coming soon:</li>
+                        <li class="NB-upcoming NB-5"><div class="NB-pricing-bullet"></div>Natural language filters</li>
                     </ul>
                 </div>
             </div>