gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
alex
2026-08-12 14:11:47 -04:00
parent 79894f654d
commit f852da4f4e
48 changed files with 4379 additions and 779 deletions
Download Patch File Download Diff File Expand all files Collapse all files
search/src/index.rs
+7 -3
View File
@@ -8,7 +8,7 @@ use tantivy::directory::{ManagedDirectory, MmapDirectory};
use tantivy::query::{BooleanQuery, Occur, QueryParser, TermQuery};
use tantivy::schema::{Field, IndexRecordOption, OwnedValue};
use tantivy::tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer, TokenizerManager};
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term};
use tantivy::{Document, Index, IndexReader, IndexWriter, ReloadPolicy, Term};
use tracing::debug_span;
#[derive(Default)]
@@ -71,7 +71,7 @@ impl SearchIndex {
user_query: &str,
default_fields: Vec<Field>,
limit: usize,
) -> error::Result<Vec<TantivyDocument>> {
) -> error::Result<Vec<HashMap<Field, OwnedValue>>> {
let _enter = debug_span!("Search Query").entered();
let parser = QueryParser::for_index(&self.index, default_fields);
@@ -87,7 +87,7 @@ impl SearchIndex {
let query = BooleanQuery::new(subqueries);
let searcher = self.reader.searcher();
let results: Vec<TantivyDocument> = searcher
let results = searcher
.search(&query, &TopDocs::with_limit(limit).order_by_score())?
.iter()
.map(|(_, address)| searcher.doc(*address))
@@ -96,3 +96,7 @@ impl SearchIndex {
Ok(results)
}
}
pub fn to_json(document: HashMap<Field, OwnedValue>) -> String {
document.to_json(Schema::schema())
}
search/src/language.rs
-63
View File
@@ -1,63 +0,0 @@
use oxigraph::model::{Term, TermRef};
use oxilangtag::LanguageTag;
use std::sync::LazyLock;
pub const ENGLISH_PRIMARY: &str = "en";
pub static ENGLISH_OR_UNTAGGED: LazyLock<LanguageCondition> = LazyLock::new(|| {
LanguageCondition::ExactMatchOrUntagged(
LanguageTag::parse(ENGLISH_PRIMARY.to_string()).unwrap(),
)
});
pub enum LanguageCondition {
ExactMatchOnly(LanguageTag<String>),
ExactMatchOrUntagged(LanguageTag<String>),
UntaggedOnly,
AnyOrNone,
}
impl LanguageCondition {
pub fn primary_matches_term<'a>(&self, term: impl Into<TermRef<'a>>) -> bool {
if let TermRef::Literal(literal) = term.into() {
let tag = literal
.language()
.map(LanguageTag::parse_and_normalize)
.and_then(Result::ok);
match (tag, self) {
(Some(language), LanguageCondition::ExactMatchOnly(expectation))
| (Some(language), LanguageCondition::ExactMatchOrUntagged(expectation)) => {
language.primary_language() == expectation.primary_language()
}
(None, LanguageCondition::ExactMatchOrUntagged(_)) => true,
(None, LanguageCondition::UntaggedOnly) => true,
(_, LanguageCondition::AnyOrNone) => true,
_ => false,
}
} else {
false
}
}
pub fn primary_language(&self) -> Option<&str> {
match self {
LanguageCondition::ExactMatchOnly(tag) => Some(tag.primary_language()),
LanguageCondition::ExactMatchOrUntagged(tag) => Some(tag.primary_language()),
_ => None,
}
}
pub fn to_filter_expression(&self, var: &str) -> String {
match self {
LanguageCondition::ExactMatchOnly(language) => {
format!(r#"FILTER(langMATCHES(LANG(?{var}), "{language}"))"#)
}
LanguageCondition::ExactMatchOrUntagged(language) => {
format!(r#"FILTER(langMATCHES(LANG(?{var}), "{language}") || !hasLANG(?{var}))"#)
}
LanguageCondition::UntaggedOnly => format!("FILTER(!hasLANG(?{var}))"),
LanguageCondition::AnyOrNone => "".to_string(),
}
}
}
search/src/lib.rs
+4 -51
View File
@@ -1,58 +1,11 @@
mod error;
mod index;
pub mod language;
pub mod rdf;
mod schema;
pub mod subtitles;
use oxigraph::model::NamedNodeRef;
pub use tantivy::TantivyDocument as SearchDocument;
pub use tantivy::doc;
pub use tantivy::indexer::IndexWriter;
pub use tantivy::schema::document::{Document, Value};
pub use tantivy::schema::Field;
pub use tantivy::schema::document::OwnedValue;
pub use error::{Result, SearchError};
use gl_graph::vocab;
pub use index::{SearchIndex, SearchIndexBuilder};
pub use schema::Schema;
pub enum DocType {
RdfProperty = 0,
RdfsClass = 1,
SkosConcept = 2,
Person = 3,
CorporateBody = 4,
Work = 5,
Expression = 6,
Manifestation = 7,
}
impl DocType {
pub fn to_named_node(&self) -> NamedNodeRef<'_> {
match self {
DocType::RdfProperty => vocab::rdf::PROPERTY,
DocType::RdfsClass => vocab::rdfs::CLASS,
DocType::SkosConcept => vocab::skos::CONCEPT,
DocType::Person => vocab::rdac::PERSON,
DocType::CorporateBody => vocab::rdac::CORPORATE_BODY,
DocType::Work => vocab::rdac::WORK,
DocType::Expression => vocab::rdac::EXPRESSION,
DocType::Manifestation => vocab::rdac::MANIFESTATION,
}
}
pub fn try_from_named_node<'a>(node: impl Into<NamedNodeRef<'a>>) -> Option<Self> {
let node = node.into();
match node {
vocab::rdf::PROPERTY => Some(DocType::RdfProperty),
vocab::rdfs::CLASS => Some(DocType::RdfsClass),
vocab::skos::CONCEPT => Some(DocType::SkosConcept),
vocab::rdac::PERSON => Some(DocType::Person),
vocab::rdac::CORPORATE_BODY => Some(DocType::CorporateBody),
vocab::rdac::WORK => Some(DocType::Work),
vocab::rdac::EXPRESSION => Some(DocType::Expression),
vocab::rdac::MANIFESTATION => Some(DocType::Manifestation),
_ => None,
}
}
}
pub use index::{SearchIndex, SearchIndexBuilder, to_json};
pub use schema::Schema;
search/src/rdf.rs
-272
View File
@@ -1,272 +0,0 @@
use crate::DocType;
use crate::language::LanguageCondition;
use crate::schema::Schema;
use gl_graph::{CurieHelper, vocab};
use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::model::{Dataset, NamedNode, NamedNodeRef, NamedOrBlankNodeRef, Term, TermRef};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use oxigraph::store::Store;
use std::collections::HashMap;
use tantivy::IndexWriter;
use tantivy::schema::{Field, OwnedValue};
fn term_to_named_node(term: &Term) -> Option<&NamedNode> {
if let Term::NamedNode(node) = term {
Some(node)
} else {
None
}
}
fn term_ref_as_named_node(term: TermRef<'_>) -> Option<NamedNodeRef<'_>> {
if let TermRef::NamedNode(node) = term {
Some(node)
} else {
None
}
}
fn term_as_str(term: &Term) -> Option<&str> {
if let Term::Literal(literal) = term {
match literal.datatype() {
xsd::STRING | xsd::NORMALIZED_STRING | rdf::LANG_STRING | rdf::DIR_LANG_STRING => {
Some(literal.value())
}
_ => None,
}
} else {
None
}
}
fn term_ref_as_str(term: TermRef<'_>) -> Option<&str> {
if let TermRef::Literal(literal) = term {
match literal.datatype() {
xsd::STRING | xsd::NORMALIZED_STRING | rdf::LANG_STRING | rdf::DIR_LANG_STRING => {
Some(literal.value())
}
_ => None,
}
} else {
None
}
}
pub fn index_entity(
dataset: &Dataset,
language: &LanguageCondition,
writer: &IndexWriter<HashMap<Field, OwnedValue>>,
) -> crate::Result<usize> {
let curie_helper = CurieHelper::new((&*gl_graph::PREFIXES).clone());
let mut subject_type_map = HashMap::new();
for quad in dataset.quads_for_pattern(None, Some(rdf::TYPE), None, None) {
if let NamedOrBlankNodeRef::NamedNode(subject) = quad.subject {
subject_type_map
.entry(subject)
.and_modify(|types: &mut Vec<_>| types.push(quad.object))
.or_insert_with(|| vec![quad.object]);
}
}
let mut counter = 0usize;
for (subject, types) in subject_type_map {
for type_ in types {
if let Some(doc_type) =
term_ref_as_named_node(type_).and_then(DocType::try_from_named_node)
{
let mut doc = match doc_type {
DocType::Person => index_person(dataset, subject, language),
DocType::CorporateBody => index_corporate_body(dataset, subject, language),
_ => continue,
};
let subject_str = subject.as_str();
doc.insert(
Schema::discriminant_field(),
OwnedValue::U64(doc_type as u64),
);
doc.insert(
Schema::iri_field(),
OwnedValue::Str(subject_str.to_string()),
);
let curie = curie_helper
.abbreviate(None, subject_str)
.map(OwnedValue::Str)
.unwrap_or(OwnedValue::Null);
doc.insert(Schema::curie_field(), curie);
writer.add_document(doc)?;
counter += 1;
}
}
}
Ok(counter)
}
fn index_person(
dataset: &Dataset,
subject: NamedNodeRef<'_>,
language: &LanguageCondition,
) -> HashMap<Field, OwnedValue> {
let given_name = dataset
.quads_for_pattern(
Some(subject.into()),
Some(vocab::rdaad::GIVEN_NAME),
None,
None,
)
.map(|q| q.object)
.filter(|term| language.primary_matches_term(*term))
.filter_map(term_ref_as_str)
.map(String::from)
.map(OwnedValue::Str)
.next()
.unwrap_or_else(|| OwnedValue::Null);
let surname = dataset
.quads_for_pattern(
Some(subject.into()),
Some(vocab::rdaad::SURNAME),
None,
None,
)
.map(|q| q.object)
.filter(|term| language.primary_matches_term(*term))
.filter_map(term_ref_as_str)
.map(String::from)
.map(OwnedValue::Str)
.next()
.unwrap_or_else(|| OwnedValue::Null);
HashMap::from_iter([
(
Schema::field("given_name", language.primary_language()),
given_name,
),
(
Schema::field("surname", language.primary_language()),
surname,
),
])
}
fn index_corporate_body(
dataset: &Dataset,
subject: NamedNodeRef<'_>,
language: &LanguageCondition,
) -> HashMap<Field, OwnedValue> {
let name = dataset
.quads_for_pattern(
Some(subject.into()),
Some(vocab::rdaad::NAME_OF_CORPORATE_BODY),
None,
None,
)
.map(|q| q.object)
.filter(|term| language.primary_matches_term(*term))
.filter_map(term_ref_as_str)
.map(String::from)
.map(OwnedValue::Str)
.next()
.unwrap_or_else(|| OwnedValue::Null);
HashMap::from_iter([(
Schema::field("corporate_name", language.primary_language()),
name,
)])
}
pub fn index_schema(
store: Store,
language: &LanguageCondition,
writer: &IndexWriter<HashMap<Field, OwnedValue>>,
) -> crate::Result<usize> {
let curie_helper = CurieHelper::new((&*gl_graph::PREFIXES).clone());
let label_filter = language.to_filter_expression("label");
let description_filter = language.to_filter_expression("description");
let query = format!(
r#"SELECT ?class ?subject ?label ?description WHERE {{
VALUES ?class {{ rdf:Property rdfs:Class skos:Concept }}
?subject a ?class ;
rdfs:label ?label .
{label_filter}
OPTIONAL {{ ?subject rdfs:comment ?comment }}
OPTIONAL {{ ?subject skos:definition ?definition }}
BIND(COALESCE(?definition, ?comment) AS ?description)
{description_filter}
}}"#
);
let mut sparql = SparqlEvaluator::new()
.with_prefix("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#")
.unwrap()
.with_prefix("rdfs", "http://www.w3.org/2000/01/rdf-schema#")
.unwrap()
.with_prefix("skos", "http://www.w3.org/2004/02/skos/core#")
.unwrap()
.parse_query(&query)?;
sparql.dataset_mut().set_default_graph_as_union();
let mut counter = 0usize;
let query_results = sparql.on_store(&store).execute()?;
if let QueryResults::Solutions(solutions) = query_results {
let primary_language = language.primary_language();
let label_field = Schema::field("label", primary_language);
let definition_field = Schema::field("definition", primary_language);
for solution in solutions.filter_map(Result::ok) {
let mut document = HashMap::with_capacity(4);
let discriminant = solution
.get("class")
.and_then(term_to_named_node)
.and_then(DocType::try_from_named_node)
.map(|doc_type| doc_type as u64)
.map(OwnedValue::U64)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::discriminant_field(), discriminant);
let subject_str = solution
.get("subject")
.and_then(term_to_named_node)
.map(NamedNode::as_str);
let curie = subject_str
.and_then(|subject| curie_helper.abbreviate(None, subject))
.map(OwnedValue::Str)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::curie_field(), curie);
let subject = subject_str
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::iri_field(), subject);
let label = solution
.get("label")
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(label_field, label);
let definition = solution
.get("description")
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(definition_field, definition);
writer.add_document(document)?;
counter += 1;
}
Ok(counter)
} else {
unreachable!()
}
}
search/src/subtitles.rs
+2 -3
View File
@@ -1,8 +1,7 @@
use crate::{Schema, SearchDocument};
use subtitler;
use subtitler::SubtitleFormat;
use tantivy::doc;
/*
pub fn bytes_to_documents(data: &[u8]) -> crate::Result<Vec<SearchDocument>> {
let subtitle_start = Schema::field("subtitle_start", None);
let subtitle_end = Schema::field("subtitle_end", None);
@@ -21,4 +20,4 @@ pub fn bytes_to_documents(data: &[u8]) -> crate::Result<Vec<SearchDocument>> {
})
.collect::<Vec<_>>();
Ok(documents)
}
}*/