Use flexsearch to power search

package.json

@@ -8,7 +8,8 @@
     "@swissquote/crafty-preset-postcss": "^1.8.0",
     "@swissquote/crafty-runner-gulp": "^1.8.0",
     "@swissquote/crafty-runner-rollup": "^1.8.0",
-    "preact": "^8.5.2"
+    "flexsearch": "^0.6.30",
+    "preact": "^10.0.0-rc.3"
   },
   "scripts": {
     "build": "crafty run",

src/js/search/Pagination.js

@@ -1,4 +1,4 @@
-import preact from "preact";
+import * as preact from "preact";
 import { textLinkPrevious, textLinkNext } from "./translation";
 /** @jsx preact.h */
 

src/js/search/Result.js

@@ -1,4 +1,4 @@
-import preact from "preact";
+import * as preact from "preact";
 /** @jsx preact.h */
 
 // TODO :: restore highlight
@@ -18,17 +18,17 @@ import preact from "preact";
 }*/
 
 export default function Result({ settings, item }) {
-    let description;
+    let text;
-    if (item.desc) {
+    if (item.text) {
-        description = item.desc
+        text = item.text
             .split(" ")
             .slice(0, settings.descriptiveWords)
             .join(" ");
         if (
-            item.desc.length < description.length &&
+            item.text.length < text.length &&
-            description.charAt(description.length - 1) !== "."
+            text.charAt(text.length - 1) !== "."
         ) {
-            description += " ...";
+            text += " ...";
         }
     }
 
@@ -37,9 +37,6 @@ export default function Result({ settings, item }) {
             <div className="SearchResults__title">
                 <a href={settings.base_url + item.url}>{item.title}</a>
             </div>
-            {settings.debug && (
-                <div className="SearchResults__debug">Score: {item.score}</div>
-            )}
             {settings.showURL && (
                 <div className="SearchResults__url">
                     <a href={settings.base_url + item.url}>
@@ -47,9 +44,7 @@ export default function Result({ settings, item }) {
                     </a>
                 </div>
             )}
-            {description.desc && (
+            {text && <div className="SearchResults__text">{text}</div>}
-                <div className="SearchResults__text">{description}</div>
-            )}
         </div>
     );
 }

src/js/search/Search.js

@@ -1,9 +1,8 @@
-import preact from "preact";
+import * as preact from "preact";
 
 import Pagination from "./Pagination";
 import Result from "./Result";
 import {
-    textSearchCommonWordsIgnored,
     textSearchNoResults,
     textSearchOneCharacterOrMore,
     textSearchOneResult,
@@ -11,7 +10,6 @@ import {
     textSearchShouldBeXOrMore,
     textSearchTooShort
 } from "./translation";
-import { getResults, getSearchString } from "./utils";
 
 /** @jsx preact.h */
 
@@ -47,42 +45,28 @@ export default class Search extends preact.Component {
     };
 
     getResults() {
-        const { settings, searchIndex } = this.props;
+        const { settings } = this.props;
         const { start } = this.state;
 
-        const searchString = getSearchString(
-            this.state.search.toLowerCase().trim()
-        );
-        const searchFor = searchString.searchFor;
-
         const warnings = [];
         let counter = 0;
         let results = [];
 
-        if (searchFor.length < settings.minimumLength) {
+        if (this.state.search.length < settings.minimumLength) {
-            if (searchString.hasStopWords) {
+            warnings.push(textSearchTooShort);
-                warnings.push(
+            warnings.push(
-                    `${textSearchNoResults}. ${textSearchCommonWordsIgnored}`
+                settings.minimumLength === 1
-                );
+                    ? textSearchOneCharacterOrMore
-            } else {
+                    : textSearchShouldBeXOrMore.replace(
-                warnings.push(textSearchTooShort);
+                          "!min",
-                warnings.push(
+                          settings.minimumLength
-                    settings.minimumLength === 1
+                      )
-                        ? textSearchOneCharacterOrMore
+            );
-                        : textSearchShouldBeXOrMore.replace(
+
-                              "!min",
-                              settings.minimumLength
-                          )
-                );
-            }
             return { warnings, counter, results, start };
         }
 
-        const found = getResults(
+        const found = this.props.onSearch(this.state.search);
-            searchIndex,
-            searchString.searchFor,
-            searchString.isStandard
-        );
 
         counter = found.length;
 

src/js/search/index.js

@@ -1,13 +1,24 @@
-import preact from "preact";
+import * as preact from "preact";
+import FlexSearch from "flexsearch";
 
 import Search from "./Search";
 
-import { getURLP } from "./utils";
-
 /** @jsx preact.h */
 
 const originalTitle = document.title;
 
+function getURLP(name) {
+    const elements = new RegExp(`[?|&]${name}=([^&;]+?)(&|#|;|$)`).exec(
+        window.location.search
+    );
+
+    return (
+        decodeURIComponent(
+            ((elements && elements[1]) || "").replace(/\+/g, "%20")
+        ) || null
+    );
+}
+
 class SearchEngine {
     constructor(options) {
         this.settings = {
@@ -20,7 +31,6 @@ class SearchEngine {
             highlightTerms: true,
             highlightEveryTerm: false,
             contentLocation: "search/search_index.json",
-            debug: false,
             ...options
         };
 
@@ -36,7 +46,24 @@ class SearchEngine {
             )
                 .then(data => data.json())
                 .then(json => {
-                    this.searchIndex = json;
+                    this.searchIndex = new FlexSearch({
+                        doc: {
+                            id: "url",
+                            field: ["title", "text", "tags"]
+                        }
+                    });
+
+                    let pages = json.pages;
+
+                    // Only keep the pages related to the current language
+                    if (window.searchLanguage) {
+                        const pagePrefix = `${window.searchLanguage}/`;
+                        pages = pages.filter(
+                            item => item.url.indexOf(pagePrefix) === 0
+                        );
+                    }
+
+                    this.searchIndex.add(pages);
                 });
         }
 
@@ -77,9 +104,8 @@ class SearchEngine {
         document.removeEventListener("keyup", this.keyUpHandler);
 
         document.body.classList.remove("with-search");
-        preact.render("", this.resultContainer, this.renderedElement);
+        preact.render(null, this.resultContainer);
         this.resultContainer = null;
-        this.renderedElement = null;
     };
 
     displaySearch() {
@@ -90,9 +116,9 @@ class SearchEngine {
 
         document.addEventListener("keyup", this.keyUpHandler);
 
-        this.renderedElement = preact.render(
+        preact.render(
             <Search
-                searchIndex={this.searchIndex}
+                onSearch={term => this.searchIndex.search(term)}
                 onClose={this.handleClose}
                 onTitleChange={title => {
                     document.title = `${title} ${originalTitle}`;

src/js/search/stopwords.js

@@ -1,177 +0,0 @@
-// Stop words (list from http://www.ranks.nl/stopwords)
-export default [
-    "a",
-    "about",
-    "above",
-    "after",
-    "again",
-    "against",
-    "all",
-    "am",
-    "an",
-    "and",
-    "any",
-    "are",
-    "aren't",
-    "as",
-    "at",
-    "be",
-    "because",
-    "been",
-    "before",
-    "being",
-    "below",
-    "between",
-    "both",
-    "but",
-    "by",
-    "can't",
-    "cannot",
-    "could",
-    "couldn't",
-    "did",
-    "didn't",
-    "do",
-    "does",
-    "doesn't",
-    "doing",
-    "don't",
-    "down",
-    "during",
-    "each",
-    "few",
-    "for",
-    "from",
-    "further",
-    "had",
-    "hadn't",
-    "has",
-    "hasn't",
-    "have",
-    "haven't",
-    "having",
-    "he",
-    "he'd",
-    "he'll",
-    "he's",
-    "her",
-    "here",
-    "here's",
-    "hers",
-    "herself",
-    "him",
-    "himself",
-    "his",
-    "how",
-    "how's",
-    "i",
-    "i'd",
-    "i'll",
-    "i'm",
-    "i've",
-    "if",
-    "in",
-    "into",
-    "is",
-    "isn't",
-    "it",
-    "it's",
-    "its",
-    "itself",
-    "let's",
-    "me",
-    "more",
-    "most",
-    "mustn't",
-    "my",
-    "myself",
-    "no",
-    "nor",
-    "not",
-    "of",
-    "off",
-    "on",
-    "once",
-    "only",
-    "or",
-    "other",
-    "ought",
-    "our",
-    "ours",
-    "ourselves",
-    "out",
-    "over",
-    "own",
-    "same",
-    "shan't",
-    "she",
-    "she'd",
-    "she'll",
-    "she's",
-    "should",
-    "shouldn't",
-    "so",
-    "some",
-    "such",
-    "than",
-    "that",
-    "that's",
-    "the",
-    "their",
-    "theirs",
-    "them",
-    "themselves",
-    "then",
-    "there",
-    "there's",
-    "these",
-    "they",
-    "they'd",
-    "they'll",
-    "they're",
-    "they've",
-    "this",
-    "those",
-    "through",
-    "to",
-    "too",
-    "under",
-    "until",
-    "up",
-    "very",
-    "was",
-    "wasn't",
-    "we",
-    "we'd",
-    "we'll",
-    "we're",
-    "we've",
-    "were",
-    "weren't",
-    "what",
-    "what's",
-    "when",
-    "when's",
-    "where",
-    "where's",
-    "which",
-    "while",
-    "who",
-    "who's",
-    "whom",
-    "why",
-    "why's",
-    "with",
-    "won't",
-    "would",
-    "wouldn't",
-    "you",
-    "you'd",
-    "you'll",
-    "you're",
-    "you've",
-    "your",
-    "yours",
-    "yourself",
-    "yourselves"
-];

src/js/search/translation.js

@@ -2,7 +2,6 @@
 const {
     Link_previous,
     Link_next,
-    Search_common_words_ignored,
     Search_no_results,
     Search_one_character_or_more,
     Search_one_result,
@@ -13,7 +12,6 @@ const {
 
 const textLinkPrevious = Link_previous;
 const textLinkNext = Link_next;
-const textSearchCommonWordsIgnored = Search_common_words_ignored;
 const textSearchNoResults = Search_no_results;
 const textSearchOneCharacterOrMore = Search_one_character_or_more;
 const textSearchOneResult = Search_one_result;
@@ -25,7 +23,6 @@ const textSearchTooShort = Search_too_short;
 export {
     textLinkPrevious,
     textLinkNext,
-    textSearchCommonWordsIgnored,
     textSearchNoResults,
     textSearchOneCharacterOrMore,
     textSearchOneResult,

src/js/search/utils.js

@@ -1,120 +0,0 @@
-import stopWords from "./stopwords";
-
-export function getURLP(name) {
-    const elements = new RegExp(`[?|&]${name}=([^&;]+?)(&|#|;|$)`).exec(
-        window.location.search
-    );
-
-    return (
-        decodeURIComponent(
-            ((elements && elements[1]) || "").replace(/\+/g, "%20")
-        ) || null
-    );
-}
-
-function getScore(searchFor, page) {
-    let score = 0;
-    const pat = new RegExp(searchFor, "gi");
-
-    if (page.title.search(pat) !== -1) {
-        score += 20 * page.title.match(pat).length;
-    }
-
-    if (page.text.search(pat) !== -1) {
-        score += 20 * page.text.match(pat).length;
-    }
-
-    if (page.tags.search(pat) !== -1) {
-        score += 10 * page.tags.match(pat).length;
-    }
-
-    if (page.url.search(pat) !== -1) {
-        score += 20;
-    }
-
-    return score;
-}
-
-function getStandardScore(searchWords, page) {
-    let score = 0;
-    for (let f = 0; f < searchWords.length; f++) {
-        if (searchWords[f].match("^-")) {
-            const pat = new RegExp(searchWords[f].substring(1), "i");
-            if (
-                page.title.search(pat) !== -1 ||
-                page.text.search(pat) !== -1 ||
-                page.tags.search(pat) !== -1
-            ) {
-                score = 0;
-            }
-        } else {
-            score += getScore(searchWords[f], page);
-        }
-    }
-
-    return score;
-}
-
-export function getSearchString(search) {
-    let isStandard = true;
-    let hasStopWords = false;
-    if (
-        (search.match('^"') && search.match('"$')) ||
-        (search.match("^'") && search.match("'$"))
-    ) {
-        isStandard = false;
-    }
-
-    let searchFor;
-    if (isStandard) {
-        const searchWords = search.split(" ");
-        searchFor = searchWords
-            .filter(word => stopWords.indexOf(word) === -1)
-            .join(" ");
-        hasStopWords = search !== searchFor;
-    } else {
-        searchFor = searchFor.substring(1, searchFor.length - 1);
-    }
-
-    return {
-        hasStopWords,
-        isStandard,
-        searchFor
-    };
-}
-
-function makeResult(score, { title, url }, desc) {
-    return {
-        score,
-        title,
-        desc,
-        url
-    };
-}
-
-export function getResults(index, searchFor, standard) {
-    const found = [];
-
-    let pages = index.pages;
-
-    // If a searchLanguage is set, filter out all other pages
-    if (window.searchLanguage) {
-        pages = pages.filter(
-            item => item.url.indexOf(`${window.searchLanguage}/`) === 0
-        );
-    }
-
-    const searchWords = searchFor.split(" ");
-    for (let i = 0; i < pages.length; i++) {
-        const score = standard
-            ? getStandardScore(searchWords, pages[i])
-            : getScore(searchFor, pages[i]);
-        if (score !== 0) {
-            found.push(makeResult(score, pages[i], pages[i].text));
-        }
-    }
-
-    found.sort((a, b) => b.score - a.score);
-
-    return found;
-}

yarn.lock

@@ -3470,6 +3470,11 @@ flatten@^1.0.2:
   resolved "https://registry.yarnpkg.com/flatten/-/flatten-1.0.2.tgz#dae46a9d78fbe25292258cc1e780a41d95c03782"
   integrity sha1-2uRqnXj74lKSJYzB54CkHZXAN4I=
 
+flexsearch@^0.6.30:
+  version "0.6.30"
+  resolved "https://registry.yarnpkg.com/flexsearch/-/flexsearch-0.6.30.tgz#d3f14389c9a4e5758b12290b3bafcd383cdc53de"
+  integrity sha512-zDBhMWbM65TsJJPBYoxV+MENufDylNtMz38e6MLTShwwuHeRNBxRYGAxR0DlwSkC4u+X2S8mlcdROWXMDleNwQ==
+
 flush-write-stream@^1.0.0, flush-write-stream@^1.0.2:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"
@@ -6269,10 +6274,10 @@ postcss@^6.0.0, postcss@^6.0.1, postcss@^6.0.10, postcss@^6.0.18, postcss@^6.0.2
     source-map "^0.6.1"
     supports-color "^5.4.0"
 
-preact@^8.5.2:
+preact@^10.0.0-rc.3:
-  version "8.5.2"
+  version "10.0.0-rc.3"
-  resolved "https://registry.yarnpkg.com/preact/-/preact-8.5.2.tgz#2f532da485287c07369e08150cf4d23921a09789"
+  resolved "https://registry.yarnpkg.com/preact/-/preact-10.0.0-rc.3.tgz#258d1bbf11744e0460b8681422cd6a0c18200df7"
-  integrity sha512-37tlDJGq5IQKqGUbqPZ7qPtsTOWFyxe+ojAOFfzKo0dEPreenqrqgJuS83zGpeGAqD9h9L9Yr7QuxH2W4ZrKxg==
+  integrity sha512-IvDc2AGvHJncEtORciLDzpluDF2MsZqf9eo6xHt7HVY4E6OvxZzAePYJtv3siVdEntxmB9NciQpbToT1APqJYQ==
 
 prelude-ls@~1.1.2:
   version "1.1.2"