gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
alex
2026-08-08 23:50:04 -04:00
parent 9177c8902b
commit 79894f654d
26 changed files with 748 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files
search/src/error.rs
+1 -1
View File
@@ -11,7 +11,7 @@ pub enum SearchError {
#[error(transparent)]
OpenDirectoryError(#[from] tantivy::directory::error::OpenDirectoryError),
#[error(transparent)]
Io(#[from] std::io::Error),
search/src/index.rs
+8 -14
View File
@@ -1,7 +1,7 @@
use std::collections::HashMap;
use std::fs;
use crate::error;
use crate::schema::Schema;
use std::collections::HashMap;
use std::fs;
use std::path::PathBuf;
use tantivy::collector::TopDocs;
use tantivy::directory::{ManagedDirectory, MmapDirectory};
@@ -24,9 +24,7 @@ impl SearchIndexBuilder {
pub fn build(self) -> error::Result<SearchIndex> {
let ngram_32 = NgramTokenizer::new(1, 32, false)?;
let ngram_32_lowercase = TextAnalyzer::builder(ngram_32)
.filter(LowerCaser)
.build();
let ngram_32_lowercase = TextAnalyzer::builder(ngram_32).filter(LowerCaser).build();
let tokenizer_manager = TokenizerManager::default();
tokenizer_manager.register("ngram_32", ngram_32_lowercase);
@@ -49,10 +47,7 @@ impl SearchIndexBuilder {
.reload_policy(ReloadPolicy::OnCommitWithDelay)
.try_into()?;
Ok(SearchIndex {
index,
reader,
})
Ok(SearchIndex { index, reader })
}
}
@@ -82,9 +77,7 @@ impl SearchIndex {
let parser = QueryParser::for_index(&self.index, default_fields);
let (user_query, _) = parser.parse_query_lenient(user_query);
let mut subqueries = vec![
(Occur::Must, user_query)
];
let mut subqueries = vec![(Occur::Must, user_query)];
if let Some(type_) = type_ {
let doc_type_term = Term::from_field_u64(Schema::discriminant_field(), type_);
@@ -94,11 +87,12 @@ impl SearchIndex {
let query = BooleanQuery::new(subqueries);
let searcher = self.reader.searcher();
let results: Vec<TantivyDocument> = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?
let results: Vec<TantivyDocument> = searcher
.search(&query, &TopDocs::with_limit(limit).order_by_score())?
.iter()
.map(|(_, address)| searcher.doc(*address))
.filter_map(Result::ok)
.collect();
Ok(results)
}
}
}
search/src/language.rs
+17 -8
View File
@@ -1,11 +1,13 @@
use std::sync::LazyLock;
use oxigraph::model::{Term, TermRef};
use oxilangtag::LanguageTag;
use std::sync::LazyLock;
pub const ENGLISH_PRIMARY: &str = "en";
pub static ENGLISH_OR_UNTAGGED: LazyLock<LanguageCondition> = LazyLock::new(|| {
LanguageCondition::ExactMatchOrUntagged(LanguageTag::parse(ENGLISH_PRIMARY.to_string()).unwrap())
LanguageCondition::ExactMatchOrUntagged(
LanguageTag::parse(ENGLISH_PRIMARY.to_string()).unwrap(),
)
});
pub enum LanguageCondition {
@@ -18,13 +20,16 @@ pub enum LanguageCondition {
impl LanguageCondition {
pub fn primary_matches_term<'a>(&self, term: impl Into<TermRef<'a>>) -> bool {
if let TermRef::Literal(literal) = term.into() {
let tag = literal.language()
let tag = literal
.language()
.map(LanguageTag::parse_and_normalize)
.and_then(Result::ok);
match (tag, self) {
(Some(language), LanguageCondition::ExactMatchOnly(expectation)) |
(Some(language), LanguageCondition::ExactMatchOrUntagged(expectation)) => language.primary_language() == expectation.primary_language(),
(Some(language), LanguageCondition::ExactMatchOnly(expectation))
| (Some(language), LanguageCondition::ExactMatchOrUntagged(expectation)) => {
language.primary_language() == expectation.primary_language()
}
(None, LanguageCondition::ExactMatchOrUntagged(_)) => true,
(None, LanguageCondition::UntaggedOnly) => true,
(_, LanguageCondition::AnyOrNone) => true,
@@ -45,10 +50,14 @@ impl LanguageCondition {
pub fn to_filter_expression(&self, var: &str) -> String {
match self {
LanguageCondition::ExactMatchOnly(language) => format!(r#"FILTER(langMATCHES(LANG(?{var}), "{language}"))"#),
LanguageCondition::ExactMatchOrUntagged(language) => format!(r#"FILTER(langMATCHES(LANG(?{var}), "{language}") || !hasLANG(?{var}))"#),
LanguageCondition::ExactMatchOnly(language) => {
format!(r#"FILTER(langMATCHES(LANG(?{var}), "{language}"))"#)
}
LanguageCondition::ExactMatchOrUntagged(language) => {
format!(r#"FILTER(langMATCHES(LANG(?{var}), "{language}") || !hasLANG(?{var}))"#)
}
LanguageCondition::UntaggedOnly => format!("FILTER(!hasLANG(?{var}))"),
LanguageCondition::AnyOrNone => "".to_string(),
}
}
}
}
search/src/lib.rs
+4 -4
View File
@@ -1,15 +1,15 @@
mod error;
mod index;
pub mod language;
pub mod rdf;
mod schema;
pub mod subtitles;
pub mod rdf;
pub mod language;
use oxigraph::model::NamedNodeRef;
pub use tantivy::TantivyDocument as SearchDocument;
pub use tantivy::doc;
pub use tantivy::schema::document::{Document, Value};
pub use tantivy::indexer::IndexWriter;
pub use tantivy::schema::document::{Document, Value};
pub use error::{Result, SearchError};
use gl_graph::vocab;
@@ -55,4 +55,4 @@ impl DocType {
_ => None,
}
}
}
}
search/src/rdf.rs
+98 -37
View File
@@ -1,14 +1,14 @@
use std::collections::HashMap;
use oxigraph::model::{Dataset, NamedNode, NamedNodeRef, NamedOrBlankNodeRef, Term, TermRef};
use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use oxigraph::store::Store;
use tantivy::IndexWriter;
use tantivy::schema::{Field, OwnedValue};
use gl_graph::{vocab, CurieHelper};
use crate::DocType;
use crate::language::LanguageCondition;
use crate::schema::Schema;
use gl_graph::{CurieHelper, vocab};
use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::model::{Dataset, NamedNode, NamedNodeRef, NamedOrBlankNodeRef, Term, TermRef};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use oxigraph::store::Store;
use std::collections::HashMap;
use tantivy::IndexWriter;
use tantivy::schema::{Field, OwnedValue};
fn term_to_named_node(term: &Term) -> Option<&NamedNode> {
if let Term::NamedNode(node) = term {
@@ -52,13 +52,18 @@ fn term_ref_as_str(term: TermRef<'_>) -> Option<&str> {
}
}
pub fn index_entity(dataset: &Dataset, language: &LanguageCondition, writer: &IndexWriter<HashMap<Field, OwnedValue>>) -> crate::Result<usize> {
pub fn index_entity(
dataset: &Dataset,
language: &LanguageCondition,
writer: &IndexWriter<HashMap<Field, OwnedValue>>,
) -> crate::Result<usize> {
let curie_helper = CurieHelper::new((&*gl_graph::PREFIXES).clone());
let mut subject_type_map = HashMap::new();
for quad in dataset.quads_for_pattern(None, Some(rdf::TYPE), None, None) {
if let NamedOrBlankNodeRef::NamedNode(subject) = quad.subject {
subject_type_map.entry(subject)
subject_type_map
.entry(subject)
.and_modify(|types: &mut Vec<_>| types.push(quad.object))
.or_insert_with(|| vec![quad.object]);
}
@@ -67,19 +72,28 @@ pub fn index_entity(dataset: &Dataset, language: &LanguageCondition, writer: &In
let mut counter = 0usize;
for (subject, types) in subject_type_map {
for type_ in types {
if let Some(doc_type) = term_ref_as_named_node(type_).and_then(DocType::try_from_named_node) {
if let Some(doc_type) =
term_ref_as_named_node(type_).and_then(DocType::try_from_named_node)
{
let mut doc = match doc_type {
DocType::Person => index_person(dataset, subject, language),
DocType::CorporateBody => index_corporate_body(dataset, subject, language),
_ => continue
_ => continue,
};
let subject_str = subject.as_str();
doc.insert(Schema::discriminant_field(), OwnedValue::U64(doc_type as u64));
doc.insert(Schema::iri_field(), OwnedValue::Str(subject_str.to_string()));
doc.insert(
Schema::discriminant_field(),
OwnedValue::U64(doc_type as u64),
);
doc.insert(
Schema::iri_field(),
OwnedValue::Str(subject_str.to_string()),
);
let curie = curie_helper.abbreviate(None, subject_str)
let curie = curie_helper
.abbreviate(None, subject_str)
.map(OwnedValue::Str)
.unwrap_or(OwnedValue::Null);
doc.insert(Schema::curie_field(), curie);
@@ -93,8 +107,18 @@ pub fn index_entity(dataset: &Dataset, language: &LanguageCondition, writer: &In
Ok(counter)
}
fn index_person(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &LanguageCondition) -> HashMap<Field, OwnedValue> {
let given_name = dataset.quads_for_pattern(Some(subject.into()), Some(vocab::rdaad::GIVEN_NAME), None, None)
fn index_person(
dataset: &Dataset,
subject: NamedNodeRef<'_>,
language: &LanguageCondition,
) -> HashMap<Field, OwnedValue> {
let given_name = dataset
.quads_for_pattern(
Some(subject.into()),
Some(vocab::rdaad::GIVEN_NAME),
None,
None,
)
.map(|q| q.object)
.filter(|term| language.primary_matches_term(*term))
.filter_map(term_ref_as_str)
@@ -103,7 +127,13 @@ fn index_person(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &Languag
.next()
.unwrap_or_else(|| OwnedValue::Null);
let surname = dataset.quads_for_pattern(Some(subject.into()), Some(vocab::rdaad::SURNAME), None, None)
let surname = dataset
.quads_for_pattern(
Some(subject.into()),
Some(vocab::rdaad::SURNAME),
None,
None,
)
.map(|q| q.object)
.filter(|term| language.primary_matches_term(*term))
.filter_map(term_ref_as_str)
@@ -113,13 +143,29 @@ fn index_person(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &Languag
.unwrap_or_else(|| OwnedValue::Null);
HashMap::from_iter([
(Schema::field("given_name", language.primary_language()), given_name),
(Schema::field("surname", language.primary_language()), surname),
(
Schema::field("given_name", language.primary_language()),
given_name,
),
(
Schema::field("surname", language.primary_language()),
surname,
),
])
}
fn index_corporate_body(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &LanguageCondition) -> HashMap<Field, OwnedValue> {
let name = dataset.quads_for_pattern(Some(subject.into()), Some(vocab::rdaad::NAME_OF_CORPORATE_BODY), None, None)
fn index_corporate_body(
dataset: &Dataset,
subject: NamedNodeRef<'_>,
language: &LanguageCondition,
) -> HashMap<Field, OwnedValue> {
let name = dataset
.quads_for_pattern(
Some(subject.into()),
Some(vocab::rdaad::NAME_OF_CORPORATE_BODY),
None,
None,
)
.map(|q| q.object)
.filter(|term| language.primary_matches_term(*term))
.filter_map(term_ref_as_str)
@@ -128,18 +174,24 @@ fn index_corporate_body(dataset: &Dataset, subject: NamedNodeRef<'_>, language:
.next()
.unwrap_or_else(|| OwnedValue::Null);
HashMap::from_iter([
(Schema::field("corporate_name", language.primary_language()), name),
])
HashMap::from_iter([(
Schema::field("corporate_name", language.primary_language()),
name,
)])
}
pub fn index_schema(store: Store, language: &LanguageCondition, writer: &IndexWriter<HashMap<Field, OwnedValue>>) -> crate::Result<usize> {
pub fn index_schema(
store: Store,
language: &LanguageCondition,
writer: &IndexWriter<HashMap<Field, OwnedValue>>,
) -> crate::Result<usize> {
let curie_helper = CurieHelper::new((&*gl_graph::PREFIXES).clone());
let label_filter = language.to_filter_expression("label");
let description_filter = language.to_filter_expression("description");
let query = format!(r#"SELECT ?class ?subject ?label ?description WHERE {{
let query = format!(
r#"SELECT ?class ?subject ?label ?description WHERE {{
VALUES ?class {{ rdf:Property rdfs:Class skos:Concept }}
?subject a ?class ;
rdfs:label ?label .
@@ -149,11 +201,15 @@ pub fn index_schema(store: Store, language: &LanguageCondition, writer: &IndexWr
OPTIONAL {{ ?subject skos:definition ?definition }}
BIND(COALESCE(?definition, ?comment) AS ?description)
{description_filter}
}}"#);
}}"#
);
let mut sparql = SparqlEvaluator::new()
.with_prefix("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#").unwrap()
.with_prefix("rdfs", "http://www.w3.org/2000/01/rdf-schema#").unwrap()
.with_prefix("skos", "http://www.w3.org/2004/02/skos/core#").unwrap()
.with_prefix("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#")
.unwrap()
.with_prefix("rdfs", "http://www.w3.org/2000/01/rdf-schema#")
.unwrap()
.with_prefix("skos", "http://www.w3.org/2004/02/skos/core#")
.unwrap()
.parse_query(&query)?;
sparql.dataset_mut().set_default_graph_as_union();
@@ -167,7 +223,8 @@ pub fn index_schema(store: Store, language: &LanguageCondition, writer: &IndexWr
for solution in solutions.filter_map(Result::ok) {
let mut document = HashMap::with_capacity(4);
let discriminant = solution.get("class")
let discriminant = solution
.get("class")
.and_then(term_to_named_node)
.and_then(DocType::try_from_named_node)
.map(|doc_type| doc_type as u64)
@@ -175,7 +232,8 @@ pub fn index_schema(store: Store, language: &LanguageCondition, writer: &IndexWr
.unwrap_or(OwnedValue::Null);
document.insert(Schema::discriminant_field(), discriminant);
let subject_str = solution.get("subject")
let subject_str = solution
.get("subject")
.and_then(term_to_named_node)
.map(NamedNode::as_str);
@@ -185,17 +243,20 @@ pub fn index_schema(store: Store, language: &LanguageCondition, writer: &IndexWr
.unwrap_or(OwnedValue::Null);
document.insert(Schema::curie_field(), curie);
let subject = subject_str.map(OwnedValue::from)
let subject = subject_str
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::iri_field(), subject);
let label = solution.get("label")
let label = solution
.get("label")
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(label_field, label);
let definition = solution.get("description")
let definition = solution
.get("description")
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
@@ -208,4 +269,4 @@ pub fn index_schema(store: Store, language: &LanguageCondition, writer: &IndexWr
} else {
unreachable!()
}
}
}
search/src/schema.rs
+14 -10
View File
@@ -11,17 +11,21 @@ pub struct Schema;
impl Schema {
pub fn schema() -> &'static TantivySchema {
SCHEMA.get_or_init(|| {
let stored_ngram32 = TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_index_option(IndexRecordOption::WithFreqsAndPositions)
.set_tokenizer("ngram_32"),
).set_stored();
let stored_ngram32 = TextOptions::default()
.set_indexing_options(
TextFieldIndexing::default()
.set_index_option(IndexRecordOption::WithFreqsAndPositions)
.set_tokenizer("ngram_32"),
)
.set_stored();
let stored_en_stem = TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_index_option(IndexRecordOption::WithFreqsAndPositions)
.set_tokenizer("en_stem"),
).set_stored();
let stored_en_stem = TextOptions::default()
.set_indexing_options(
TextFieldIndexing::default()
.set_index_option(IndexRecordOption::WithFreqsAndPositions)
.set_tokenizer("en_stem"),
)
.set_stored();
let mut schema_builder = TantivySchema::builder();
schema_builder.add_u64_field("discriminant", schema::FAST | schema::INDEXED);
search/src/subtitles.rs
+12 -9
View File
@@ -1,7 +1,7 @@
use crate::{Schema, SearchDocument};
use subtitler;
use subtitler::SubtitleFormat;
use tantivy::doc;
use crate::{Schema, SearchDocument};
pub fn bytes_to_documents(data: &[u8]) -> crate::Result<Vec<SearchDocument>> {
let subtitle_start = Schema::field("subtitle_start", None);
@@ -10,12 +10,15 @@ pub fn bytes_to_documents(data: &[u8]) -> crate::Result<Vec<SearchDocument>> {
let subtitle_file = subtitler::parse_bytes(data)?;
let subtitles = subtitle_file.subtitles();
let documents = subtitles.iter().map(|subtitle| {
doc!(
subtitle_start => subtitle.start,
subtitle_end => subtitle.end,
subtitle_content => subtitle.text,
)
}).collect::<Vec<_>>();
let documents = subtitles
.iter()
.map(|subtitle| {
doc!(
subtitle_start => subtitle.start,
subtitle_end => subtitle.end,
subtitle_content => subtitle.text,
)
})
.collect::<Vec<_>>();
Ok(documents)
}
}