D3Public/daux.io
8
0
Fork 0
Code Pull Requests Releases Activity

Use flexsearch to power search

Browse Source
This commit is contained in:
Stéphane Goetz 2019-09-19 22:00:34 +02:00
parent 69d4e0cbfa
commit 94893a2c7a
11 changed files with 71 additions and 360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
package.json
View File

@ -8,7 +8,8 @@
"@swissquote/crafty-preset-postcss": "^1.8.0",
"@swissquote/crafty-runner-gulp": "^1.8.0",
"@swissquote/crafty-runner-rollup": "^1.8.0",
"preact": "^8.5.2"
"flexsearch": "^0.6.30",
"preact": "^10.0.0-rc.3"
},
"scripts": {
"build": "crafty run",

2
search/search.min.js vendored
View File

File diff suppressed because one or more lines are too long

2
search/search.min.js.map
View File

File diff suppressed because one or more lines are too long

2
src/js/search/Pagination.js
View File

@ -1,4 +1,4 @@
import preact from "preact";
import * as preact from "preact";
import { textLinkPrevious, textLinkNext } from "./translation";
/** @jsx preact.h */

21
src/js/search/Result.js
View File

@ -1,4 +1,4 @@
import preact from "preact";
import * as preact from "preact";
/** @jsx preact.h */
// TODO :: restore highlight
@ -18,17 +18,17 @@ import preact from "preact";
}*/
export default function Result({ settings, item }) {
let description;
if (item.desc) {
description = item.desc
let text;
if (item.text) {
text = item.text
.split(" ")
.slice(0, settings.descriptiveWords)
.join(" ");
if (
item.desc.length < description.length &&
description.charAt(description.length - 1) !== "."
item.text.length < text.length &&
text.charAt(text.length - 1) !== "."
) {
description += " ...";
text += " ...";
}
}
@ -37,9 +37,6 @@ export default function Result({ settings, item }) {
<div className="SearchResults__title">
<a href={settings.base_url + item.url}>{item.title}</a>
</div>
{settings.debug && (
<div className="SearchResults__debug">Score: {item.score}</div>
)}
{settings.showURL && (
<div className="SearchResults__url">
<a href={settings.base_url + item.url}>
@ -47,9 +44,7 @@ export default function Result({ settings, item }) {
</a>
</div>
)}
{description.desc && (
<div className="SearchResults__text">{description}</div>
)}
{text && <div className="SearchResults__text">{text}</div>}
</div>
);
}

44
src/js/search/Search.js
View File

@ -1,9 +1,8 @@
import preact from "preact";
import * as preact from "preact";
import Pagination from "./Pagination";
import Result from "./Result";
import {
textSearchCommonWordsIgnored,
textSearchNoResults,
textSearchOneCharacterOrMore,
textSearchOneResult,
@ -11,7 +10,6 @@ import {
textSearchShouldBeXOrMore,
textSearchTooShort
} from "./translation";
import { getResults, getSearchString } from "./utils";
/** @jsx preact.h */
@ -47,42 +45,28 @@ export default class Search extends preact.Component {
};
getResults() {
const { settings, searchIndex } = this.props;
const { settings } = this.props;
const { start } = this.state;
const searchString = getSearchString(
this.state.search.toLowerCase().trim()
);
const searchFor = searchString.searchFor;
const warnings = [];
let counter = 0;
let results = [];
if (searchFor.length < settings.minimumLength) {
if (searchString.hasStopWords) {
warnings.push(
`${textSearchNoResults}. ${textSearchCommonWordsIgnored}`
);
} else {
warnings.push(textSearchTooShort);
warnings.push(
settings.minimumLength === 1
? textSearchOneCharacterOrMore
: textSearchShouldBeXOrMore.replace(
"!min",
settings.minimumLength
)
);
}
if (this.state.search.length < settings.minimumLength) {
warnings.push(textSearchTooShort);
warnings.push(
settings.minimumLength === 1
? textSearchOneCharacterOrMore
: textSearchShouldBeXOrMore.replace(
"!min",
settings.minimumLength
)
);
return { warnings, counter, results, start };
}
const found = getResults(
searchIndex,
searchString.searchFor,
searchString.isStandard
);
const found = this.props.onSearch(this.state.search);
counter = found.length;

44
src/js/search/index.js
View File

@ -1,13 +1,24 @@
import preact from "preact";
import * as preact from "preact";
import FlexSearch from "flexsearch";
import Search from "./Search";
import { getURLP } from "./utils";
/** @jsx preact.h */
const originalTitle = document.title;
function getURLP(name) {
const elements = new RegExp(`[?|&]${name}=([^&;]+?)(&|#|;|$)`).exec(
window.location.search
);
return (
decodeURIComponent(
((elements && elements[1]) || "").replace(/\+/g, "%20")
) || null
);
}
class SearchEngine {
constructor(options) {
this.settings = {
@ -20,7 +31,6 @@ class SearchEngine {
highlightTerms: true,
highlightEveryTerm: false,
contentLocation: "search/search_index.json",
debug: false,
...options
};
@ -36,7 +46,24 @@ class SearchEngine {
)
.then(data => data.json())
.then(json => {
this.searchIndex = json;
this.searchIndex = new FlexSearch({
doc: {
id: "url",
field: ["title", "text", "tags"]
}
});
let pages = json.pages;
// Only keep the pages related to the current language
if (window.searchLanguage) {
const pagePrefix = `${window.searchLanguage}/`;
pages = pages.filter(
item => item.url.indexOf(pagePrefix) === 0
);
}
this.searchIndex.add(pages);
});
}
@ -77,9 +104,8 @@ class SearchEngine {
document.removeEventListener("keyup", this.keyUpHandler);
document.body.classList.remove("with-search");
preact.render("", this.resultContainer, this.renderedElement);
preact.render(null, this.resultContainer);
this.resultContainer = null;
this.renderedElement = null;
};
displaySearch() {
@ -90,9 +116,9 @@ class SearchEngine {
document.addEventListener("keyup", this.keyUpHandler);
this.renderedElement = preact.render(
preact.render(
<Search
searchIndex={this.searchIndex}
onSearch={term => this.searchIndex.search(term)}
onClose={this.handleClose}
onTitleChange={title => {
document.title = `${title} ${originalTitle}`;

177
src/js/search/stopwords.js
View File

@ -1,177 +0,0 @@
// Stop words (list from http://www.ranks.nl/stopwords)
export default [
"a",
"about",
"above",
"after",
"again",
"against",
"all",
"am",
"an",
"and",
"any",
"are",
"aren't",
"as",
"at",
"be",
"because",
"been",
"before",
"being",
"below",
"between",
"both",
"but",
"by",
"can't",
"cannot",
"could",
"couldn't",
"did",
"didn't",
"do",
"does",
"doesn't",
"doing",
"don't",
"down",
"during",
"each",
"few",
"for",
"from",
"further",
"had",
"hadn't",
"has",
"hasn't",
"have",
"haven't",
"having",
"he",
"he'd",
"he'll",
"he's",
"her",
"here",
"here's",
"hers",
"herself",
"him",
"himself",
"his",
"how",
"how's",
"i",
"i'd",
"i'll",
"i'm",
"i've",
"if",
"in",
"into",
"is",
"isn't",
"it",
"it's",
"its",
"itself",
"let's",
"me",
"more",
"most",
"mustn't",
"my",
"myself",
"no",
"nor",
"not",
"of",
"off",
"on",
"once",
"only",
"or",
"other",
"ought",
"our",
"ours",
"ourselves",
"out",
"over",
"own",
"same",
"shan't",
"she",
"she'd",
"she'll",
"she's",
"should",
"shouldn't",
"so",
"some",
"such",
"than",
"that",
"that's",
"the",
"their",
"theirs",
"them",
"themselves",
"then",
"there",
"there's",
"these",
"they",
"they'd",
"they'll",
"they're",
"they've",
"this",
"those",
"through",
"to",
"too",
"under",
"until",
"up",
"very",
"was",
"wasn't",
"we",
"we'd",
"we'll",
"we're",
"we've",
"were",
"weren't",
"what",
"what's",
"when",
"when's",
"where",
"where's",
"which",
"while",
"who",
"who's",
"whom",
"why",
"why's",
"with",
"won't",
"would",
"wouldn't",
"you",
"you'd",
"you'll",
"you're",
"you've",
"your",
"yours",
"yourself",
"yourselves"
];

3
src/js/search/translation.js
View File

@ -2,7 +2,6 @@
const {
Link_previous,
Link_next,
Search_common_words_ignored,
Search_no_results,
Search_one_character_or_more,
Search_one_result,
@ -13,7 +12,6 @@ const {
const textLinkPrevious = Link_previous;
const textLinkNext = Link_next;
const textSearchCommonWordsIgnored = Search_common_words_ignored;
const textSearchNoResults = Search_no_results;
const textSearchOneCharacterOrMore = Search_one_character_or_more;
const textSearchOneResult = Search_one_result;
@ -25,7 +23,6 @@ const textSearchTooShort = Search_too_short;
export {
textLinkPrevious,
textLinkNext,
textSearchCommonWordsIgnored,
textSearchNoResults,
textSearchOneCharacterOrMore,
textSearchOneResult,

120
src/js/search/utils.js
View File

@ -1,120 +0,0 @@
import stopWords from "./stopwords";
export function getURLP(name) {
const elements = new RegExp(`[?|&]${name}=([^&;]+?)(&|#|;|$)`).exec(
window.location.search
);
return (
decodeURIComponent(
((elements && elements[1]) || "").replace(/\+/g, "%20")
) || null
);
}
function getScore(searchFor, page) {
let score = 0;
const pat = new RegExp(searchFor, "gi");
if (page.title.search(pat) !== -1) {
score += 20 * page.title.match(pat).length;
}
if (page.text.search(pat) !== -1) {
score += 20 * page.text.match(pat).length;
}
if (page.tags.search(pat) !== -1) {
score += 10 * page.tags.match(pat).length;
}
if (page.url.search(pat) !== -1) {
score += 20;
}
return score;
}
function getStandardScore(searchWords, page) {
let score = 0;
for (let f = 0; f < searchWords.length; f++) {
if (searchWords[f].match("^-")) {
const pat = new RegExp(searchWords[f].substring(1), "i");
if (
page.title.search(pat) !== -1 ||
page.text.search(pat) !== -1 ||
page.tags.search(pat) !== -1
) {
score = 0;
}
} else {
score += getScore(searchWords[f], page);
}
}
return score;
}
export function getSearchString(search) {
let isStandard = true;
let hasStopWords = false;
if (
(search.match('^"') && search.match('"$')) ||
(search.match("^'") && search.match("'$"))
) {
isStandard = false;
}
let searchFor;
if (isStandard) {
const searchWords = search.split(" ");
searchFor = searchWords
.filter(word => stopWords.indexOf(word) === -1)
.join(" ");
hasStopWords = search !== searchFor;
} else {
searchFor = searchFor.substring(1, searchFor.length - 1);
}
return {
hasStopWords,
isStandard,
searchFor
};
}
function makeResult(score, { title, url }, desc) {
return {
score,
title,
desc,
url
};
}
export function getResults(index, searchFor, standard) {
const found = [];
let pages = index.pages;
// If a searchLanguage is set, filter out all other pages
if (window.searchLanguage) {
pages = pages.filter(
item => item.url.indexOf(`${window.searchLanguage}/`) === 0
);
}
const searchWords = searchFor.split(" ");
for (let i = 0; i < pages.length; i++) {
const score = standard
? getStandardScore(searchWords, pages[i])
: getScore(searchFor, pages[i]);
if (score !== 0) {
found.push(makeResult(score, pages[i], pages[i].text));
}
}
found.sort((a, b) => b.score - a.score);
return found;
}

13
yarn.lock
View File

@ -3470,6 +3470,11 @@ flatten@^1.0.2:
resolved "https://registry.yarnpkg.com/flatten/-/flatten-1.0.2.tgz#dae46a9d78fbe25292258cc1e780a41d95c03782"
integrity sha1-2uRqnXj74lKSJYzB54CkHZXAN4I=
flexsearch@^0.6.30:
version "0.6.30"
resolved "https://registry.yarnpkg.com/flexsearch/-/flexsearch-0.6.30.tgz#d3f14389c9a4e5758b12290b3bafcd383cdc53de"
integrity sha512-zDBhMWbM65TsJJPBYoxV+MENufDylNtMz38e6MLTShwwuHeRNBxRYGAxR0DlwSkC4u+X2S8mlcdROWXMDleNwQ==
flush-write-stream@^1.0.0, flush-write-stream@^1.0.2:
version "1.1.1"
resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"
@ -6269,10 +6274,10 @@ postcss@^6.0.0, postcss@^6.0.1, postcss@^6.0.10, postcss@^6.0.18, postcss@^6.0.2
source-map "^0.6.1"
supports-color "^5.4.0"
preact@^8.5.2:
version "8.5.2"
resolved "https://registry.yarnpkg.com/preact/-/preact-8.5.2.tgz#2f532da485287c07369e08150cf4d23921a09789"
integrity sha512-37tlDJGq5IQKqGUbqPZ7qPtsTOWFyxe+ojAOFfzKo0dEPreenqrqgJuS83zGpeGAqD9h9L9Yr7QuxH2W4ZrKxg==
preact@^10.0.0-rc.3:
version "10.0.0-rc.3"
resolved "https://registry.yarnpkg.com/preact/-/preact-10.0.0-rc.3.tgz#258d1bbf11744e0460b8681422cd6a0c18200df7"
integrity sha512-IvDc2AGvHJncEtORciLDzpluDF2MsZqf9eo6xHt7HVY4E6OvxZzAePYJtv3siVdEntxmB9NciQpbToT1APqJYQ==
prelude-ls@~1.1.2:
version "1.1.2"