.

search/src/error.rs

@@ -14,4 +14,16 @@ pub enum SearchError {
     
     #[error(transparent)]
     Io(#[from] std::io::Error),
+
+    #[error(transparent)]
+    Parse(#[from] subtitler::error::ParseError),
+
+    #[error(transparent)]
+    QueryEvaluation(#[from] oxigraph::sparql::QueryEvaluationError),
+
+    #[error(transparent)]
+    SparqlSyntax(#[from] oxigraph::sparql::SparqlSyntaxError),
+    
+    #[error("The provided DocType is not valid for this operation")]
+    InvalidDocType,
 }

search/src/index.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use std::fs;
 use crate::error;
 use crate::schema::Schema;
@@ -5,7 +6,7 @@ use std::path::PathBuf;
 use tantivy::collector::TopDocs;
 use tantivy::directory::{ManagedDirectory, MmapDirectory};
 use tantivy::query::{BooleanQuery, Occur, QueryParser, TermQuery};
-use tantivy::schema::{Field, IndexRecordOption};
+use tantivy::schema::{Field, IndexRecordOption, OwnedValue};
 use tantivy::tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer, TokenizerManager};
 use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term};
 use tracing::debug_span;
@@ -65,7 +66,7 @@ impl SearchIndex {
         SearchIndexBuilder::default()
     }
 
-    pub fn writer(&mut self) -> crate::Result<IndexWriter> {
+    pub fn writer(&mut self) -> crate::Result<IndexWriter<HashMap<Field, OwnedValue>>> {
         Ok(self.index.writer(128 * 1024 * 1024)?)
     }
 
@@ -86,7 +87,7 @@ impl SearchIndex {
         ];
 
         if let Some(type_) = type_ {
-            let doc_type_term = Term::from_field_u64(Schema::type_field(), type_);
+            let doc_type_term = Term::from_field_u64(Schema::discriminant_field(), type_);
             let doc_type_query = Box::new(TermQuery::new(doc_type_term, IndexRecordOption::Basic));
             subqueries.push((Occur::Must, doc_type_query));
         }

search/src/language.rs

@@ -0,0 +1,54 @@
+use std::sync::LazyLock;
+use oxigraph::model::{Term, TermRef};
+use oxilangtag::LanguageTag;
+
+pub const ENGLISH_PRIMARY: &str = "en";
+
+pub static ENGLISH_OR_UNTAGGED: LazyLock<LanguageCondition> = LazyLock::new(|| {
+    LanguageCondition::ExactMatchOrUntagged(LanguageTag::parse(ENGLISH_PRIMARY.to_string()).unwrap())
+});
+
+pub enum LanguageCondition {
+    ExactMatchOnly(LanguageTag<String>),
+    ExactMatchOrUntagged(LanguageTag<String>),
+    UntaggedOnly,
+    AnyOrNone,
+}
+
+impl LanguageCondition {
+    pub fn primary_matches_term<'a>(&self, term: impl Into<TermRef<'a>>) -> bool {
+        if let TermRef::Literal(literal) = term.into() {
+            let tag = literal.language()
+                .map(LanguageTag::parse_and_normalize)
+                .and_then(Result::ok);
+
+            match (tag, self) {
+                (Some(language), LanguageCondition::ExactMatchOnly(expectation)) |
+                (Some(language), LanguageCondition::ExactMatchOrUntagged(expectation)) => language.primary_language() == expectation.primary_language(),
+                (None, LanguageCondition::ExactMatchOrUntagged(_)) => true,
+                (None, LanguageCondition::UntaggedOnly) => true,
+                (_, LanguageCondition::AnyOrNone) => true,
+                _ => false,
+            }
+        } else {
+            false
+        }
+    }
+
+    pub fn primary_language(&self) -> Option<&str> {
+        match self {
+            LanguageCondition::ExactMatchOnly(tag) => Some(tag.primary_language()),
+            LanguageCondition::ExactMatchOrUntagged(tag) => Some(tag.primary_language()),
+            _ => None,
+        }
+    }
+
+    pub fn to_filter_expression(&self, var: &str) -> String {
+        match self {
+            LanguageCondition::ExactMatchOnly(language) => format!(r#"FILTER(langMATCHES(LANG(?{var}), "{language}"))"#),
+            LanguageCondition::ExactMatchOrUntagged(language) => format!(r#"FILTER(langMATCHES(LANG(?{var}), "{language}") || !hasLANG(?{var}))"#),
+            LanguageCondition::UntaggedOnly => format!("FILTER(!hasLANG(?{var}))"),
+            LanguageCondition::AnyOrNone => "".to_string(),
+        }
+    }
+}

search/src/lib.rs

@@ -1,11 +1,32 @@
 mod error;
 mod index;
 mod schema;
+pub mod subtitles;
+pub mod rdf;
+pub mod language;
 
+use oxigraph::model::NamedNodeRef;
 pub use tantivy::TantivyDocument as SearchDocument;
 pub use tantivy::doc;
-pub use tantivy::schema::document::Value;
+pub use tantivy::schema::document::{Document, Value};
+pub use tantivy::indexer::IndexWriter;
 
 pub use error::{Result, SearchError};
 pub use index::{SearchIndex, SearchIndexBuilder};
-pub use schema::Schema;
+pub use schema::Schema;
+
+pub enum DocType {
+    RdfProperty = 0,
+    RdfsClass = 1,
+    SkosConcept = 2,
+    Person = 3,
+}
+
+impl DocType {
+    pub fn from_named_node(node: NamedNodeRef) -> Option<Self> {
+        match node.as_str() {
+            "http://rdaregistry.info/Elements/c/C10004" => Some(DocType::Person),
+            _ => None,
+        }
+    }
+}

search/src/rdf.rs

@@ -0,0 +1,177 @@
+use std::collections::HashMap;
+use oxigraph::model::{Dataset, NamedNode, NamedNodeRef, NamedOrBlankNodeRef, Term, TermRef};
+use oxigraph::model::vocab::{rdf, xsd};
+use oxigraph::sparql::{QueryResults, SparqlEvaluator};
+use oxigraph::store::Store;
+use tantivy::IndexWriter;
+use tantivy::schema::{Field, OwnedValue};
+use crate::DocType;
+use crate::language::LanguageCondition;
+use crate::schema::Schema;
+use crate::SearchError::InvalidDocType;
+
+fn term_to_named_node(term: &Term) -> Option<&NamedNode> {
+    if let Term::NamedNode(node) = term {
+        Some(node)
+    } else {
+        None
+    }
+}
+
+fn term_ref_as_named_node(term: TermRef<'_>) -> Option<NamedNodeRef<'_>> {
+    if let TermRef::NamedNode(node) = term {
+        Some(node)
+    } else {
+        None
+    }
+}
+
+fn term_as_str(term: &Term) -> Option<&str> {
+    if let Term::Literal(literal) = term {
+        match literal.datatype() {
+            xsd::STRING | xsd::NORMALIZED_STRING | rdf::LANG_STRING | rdf::DIR_LANG_STRING => {
+                Some(literal.value())
+            }
+            _ => None,
+        }
+    } else {
+        None
+    }
+}
+
+fn term_ref_as_str(term: TermRef<'_>) -> Option<&str> {
+    if let TermRef::Literal(literal) = term {
+        match literal.datatype() {
+            xsd::STRING | xsd::NORMALIZED_STRING | rdf::LANG_STRING | rdf::DIR_LANG_STRING => {
+                Some(literal.value())
+            }
+            _ => None,
+        }
+    } else {
+        None
+    }
+}
+
+const GIVEN_NAME: NamedNodeRef = NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50292");
+const SURNAME: NamedNodeRef = NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50291");
+
+pub fn index_entity(dataset: &Dataset, language: &LanguageCondition, writer: &IndexWriter<HashMap<Field, OwnedValue>>) -> crate::Result<usize> {
+    let mut subject_type_map = HashMap::new();
+    for quad in dataset.quads_for_pattern(None, Some(rdf::TYPE), None, None) {
+        if let NamedOrBlankNodeRef::NamedNode(subject) = quad.subject {
+            subject_type_map.entry(subject)
+                .and_modify(|types: &mut Vec<_>| types.push(quad.object))
+                .or_insert_with(Vec::new);
+        }
+    }
+
+    let mut counter = 0usize;
+    for (subject, types) in subject_type_map {
+        for type_ in types {
+            if let Some(doc_type) = term_ref_as_named_node(type_).and_then(DocType::from_named_node) {
+                match doc_type {
+                    DocType::Person => {
+                        let mut doc = index_person(dataset, subject, language);
+                        doc.insert(Schema::discriminant_field(), OwnedValue::U64(doc_type as u64));
+                        doc.insert(Schema::iri_field(), OwnedValue::Str(subject.as_str().to_string()));
+                        writer.add_document(doc)?;
+                        counter += 1;
+                    }
+                    _ => {}
+                }
+            }
+        }
+    }
+
+    Ok(counter)
+}
+
+fn index_person(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &LanguageCondition) -> HashMap<Field, OwnedValue> {
+    let given_name = dataset.quads_for_pattern(Some(subject.into()), Some(GIVEN_NAME), None, None)
+        .map(|q| q.object)
+        .filter(|term| language.primary_matches_term(*term))
+        .filter_map(term_ref_as_str)
+        .map(String::from)
+        .map(OwnedValue::Str)
+        .next()
+        .unwrap_or_else(|| OwnedValue::Null);
+
+    let surname = dataset.quads_for_pattern(Some(subject.into()), Some(SURNAME), None, None)
+        .map(|q| q.object)
+        .filter(|term| language.primary_matches_term(*term))
+        .filter_map(term_ref_as_str)
+        .map(String::from)
+        .map(OwnedValue::Str)
+        .next()
+        .unwrap_or_else(|| OwnedValue::Null);
+
+    HashMap::from_iter([
+        (Schema::field("given_name", language.primary_language()), given_name),
+        (Schema::field("surname", language.primary_language()), surname),
+    ])
+}
+
+pub fn index_triples(doc_type: DocType, store: Store, language: &LanguageCondition, writer: &IndexWriter<HashMap<Field, OwnedValue>>) -> crate::Result<usize> {
+    let label_filter = language.to_filter_expression("label");
+    let description_filter = language.to_filter_expression("description");
+    let rdf_type = match doc_type {
+        DocType::RdfsClass => Ok("rdfs:Class"),
+        DocType::RdfProperty => Ok("rdf:Property"),
+        DocType::SkosConcept => Ok("skos:Concept"),
+        _ => Err(InvalidDocType),
+    }?;
+
+    let query = format!(r#"SELECT ?subject ?label ?description WHERE {{
+    ?subject a {rdf_type} ;
+    	rdfs:label ?label .
+    {label_filter}
+
+    OPTIONAL {{ ?subject rdfs:comment ?comment }}
+    OPTIONAL {{ ?subject skos:definition ?definition }}
+    BIND(COALESCE(?definition, ?comment) AS ?description)
+    {description_filter}
+}}"#);
+    let mut sparql = SparqlEvaluator::new()
+        .with_prefix("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#").unwrap()
+        .with_prefix("rdfs", "http://www.w3.org/2000/01/rdf-schema#").unwrap()
+        .with_prefix("skos", "http://www.w3.org/2004/02/skos/core#").unwrap()
+        .parse_query(&query)?;
+    sparql.dataset_mut().set_default_graph_as_union();
+
+    let mut counter = 0usize;
+    let query_results = sparql.on_store(&store).execute()?;
+    if let QueryResults::Solutions(solutions) = query_results {
+        let discriminant = OwnedValue::U64(doc_type as u64);
+        let primary_language = language.primary_language();
+
+        for solution in solutions.filter_map(Result::ok) {
+            let mut document = HashMap::with_capacity(4);
+            document.insert(Schema::discriminant_field(), discriminant.clone());
+
+            let subject = solution.get("subject")
+                .and_then(term_to_named_node)
+                .map(NamedNode::as_str)
+                .map(OwnedValue::from)
+                .unwrap_or(OwnedValue::Null);
+            document.insert(Schema::iri_field(), subject);
+
+            let label = solution.get("label")
+                .and_then(term_as_str)
+                .map(OwnedValue::from)
+                .unwrap_or(OwnedValue::Null);
+            document.insert(Schema::field("label", primary_language), label);
+
+            let definition = solution.get("description")
+                .and_then(term_as_str)
+                .map(OwnedValue::from)
+                .unwrap_or(OwnedValue::Null);
+            document.insert(Schema::field("definition", primary_language), definition);
+
+            writer.add_document(document)?;
+            counter += 1;
+        }
+        Ok(counter)
+    } else {
+        unreachable!()
+    }
+}

search/src/schema.rs

@@ -24,7 +24,7 @@ impl Schema {
             ).set_stored();
 
             let mut schema_builder = TantivySchema::builder();
-            schema_builder.add_u64_field("type", schema::FAST | schema::INDEXED);
+            schema_builder.add_u64_field("discriminant", schema::FAST | schema::INDEXED);
             schema_builder.add_text_field("iri", schema::STORED | schema::STRING);
             schema_builder.add_text_field("curie", stored_ngram32.clone());
 
@@ -34,12 +34,16 @@ impl Schema {
             schema_builder.add_text_field("surname:en", stored_ngram32.clone());
             schema_builder.add_text_field("given_name:en", stored_ngram32.clone());
 
+            schema_builder.add_u64_field("subtitle_start", schema::STORED);
+            schema_builder.add_u64_field("subtitle_end", schema::STORED);
+            schema_builder.add_text_field("subtitle:en", stored_en_stem.clone());
+
             schema_builder.build()
         })
     }
 
-    pub fn type_field() -> Field {
-        Self::schema().get_field("type").unwrap()
+    pub fn discriminant_field() -> Field {
+        Self::schema().get_field("discriminant").unwrap()
     }
 
     pub fn iri_field() -> Field {
@@ -50,9 +54,15 @@ impl Schema {
         Self::schema().get_field("curie").unwrap()
     }
 
-    pub fn field(name: &str, language: &str) -> Field {
+    pub fn field(name: &str, language: Option<&str>) -> Field {
+        let field_name = if let Some(language) = language {
+            format!("{name}:{language}")
+        } else {
+            name.to_string()
+        };
+
         Schema::schema()
-            .get_field(&format!("{name}:{language}"))
+            .get_field(&field_name)
             .expect("Field not found in schema")
     }
 

search/src/subtitles.rs

@@ -0,0 +1,21 @@
+use subtitler;
+use subtitler::SubtitleFormat;
+use tantivy::doc;
+use crate::{Schema, SearchDocument};
+
+pub fn bytes_to_documents(data: &[u8]) -> crate::Result<Vec<SearchDocument>> {
+    let subtitle_start = Schema::field("subtitle_start", None);
+    let subtitle_end = Schema::field("subtitle_end", None);
+    let subtitle_content = Schema::field("subtitle", Some("en"));
+
+    let subtitle_file = subtitler::parse_bytes(data)?;
+    let subtitles = subtitle_file.subtitles();
+    let documents = subtitles.iter().map(|subtitle| {
+        doc!(
+            subtitle_start => subtitle.start,
+            subtitle_end => subtitle.end,
+            subtitle_content => subtitle.text,
+        )
+    }).collect::<Vec<_>>();
+    Ok(documents)
+}