diff --git a/Cargo.lock b/Cargo.lock index 2105336..5db1a7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1580,6 +1580,14 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "gl-graph" +version = "0.1.0" +dependencies = [ + "oxigraph", + "tracing", +] + [[package]] name = "gl-publisher" version = "0.1.0" @@ -1587,6 +1595,7 @@ dependencies = [ "clap", "color-eyre", "csv", + "gl-graph", "gl-search", "http", "iced", @@ -1608,6 +1617,7 @@ name = "gl-search" version = "0.1.0" dependencies = [ "futures", + "gl-graph", "oxigraph", "oxilangtag", "subtitler", diff --git a/Cargo.toml b/Cargo.toml index 1a02a1e..18230ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,13 @@ [workspace] resolver = "2" members = [ + "graph", "publish", "search", ] [workspace.dependencies] +gl-graph = { path = "graph" } gl-search = { path = "search" } ldp = { path = "../../ldp/ldp", features = ["keyed"] } diff --git a/graph/Cargo.toml b/graph/Cargo.toml new file mode 100644 index 0000000..f9d542f --- /dev/null +++ b/graph/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "gl-graph" +version = "0.1.0" +edition = "2024" + +[dependencies] +oxigraph.workspace = true +tracing.workspace = true \ No newline at end of file diff --git a/graph/src/curie.rs b/graph/src/curie.rs new file mode 100644 index 0000000..89b2d6b --- /dev/null +++ b/graph/src/curie.rs @@ -0,0 +1,87 @@ +use std::collections::BTreeMap; +use std::sync::LazyLock; +use oxigraph::model::{GraphNameRef, NamedNodeRef}; +use tracing::debug; + +const ONTOLOGY_PREFIX: &str = "https://graphofliberty.org/2026/04/ont/"; +const ONTOLOGY_GRAPH_NAME: GraphNameRef = GraphNameRef::NamedNode(NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont")); + +pub static PREFIXES: LazyLock> = LazyLock::new(|| { + BTreeMap::from_iter([ + ("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"), + ("rdfs", "http://www.w3.org/2000/01/rdf-schema#"), + ("owl", "http://www.w3.org/2002/07/owl#"), + ("xsd", "http://www.w3.org/2001/XMLSchema#"), + ("ldp", "http://www.w3.org/ns/ldp#"), + ("dc", "http://purl.org/dc/elements/1.1/"), + ("posix", "http://www.w3.org/ns/posix/stat#"), + ( + "ebucore", + "http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#", + ), + ("prov", "http://www.w3.org/ns/prov#"), + ("locid", "http://id.loc.gov/vocabulary/identifiers/"), + ("loclang", "http://id.loc.gov/vocabulary/languages/"), + ("premis", "http://www.loc.gov/premis/rdf/v1#"), + ("premis3", "http://www.loc.gov/premis/rdf/v3/"), + ("dcterms", "http://purl.org/dc/terms/"), + ("fedora", "http://fedora.info/definitions/v4/repository#"), + ("rdaa", "http://rdaregistry.info/Elements/a/"), + ("rdac", "http://rdaregistry.info/Elements/c/"), + ("rdae", "http://rdaregistry.info/Elements/e/"), + ("rdai", "http://rdaregistry.info/Elements/i/"), + ("rdam", "http://rdaregistry.info/Elements/m/"), + ("rdan", "http://rdaregistry.info/Elements/n/"), + ("rdap", "http://rdaregistry.info/Elements/p/"), + ("rdaf", "http://rdaregistry.info/Elements/rof/"), + ("rdat", "http://rdaregistry.info/Elements/t/"), + ("rdau", "http://rdaregistry.info/Elements/u/"), + ("rdaw", "http://rdaregistry.info/Elements/w/"), + ("rdax", "http://rdaregistry.info/Elements/x/"), + ("rdaco", "http://rdaregistry.info/termList/RDAContentType/"), + ("rdact", "http://rdaregistry.info/termList/RDACarrierType/"), + ("rdamt", "http://rdaregistry.info/termList/RDAMediaType/"), + ("rdaft", "http://rdaregistry.info/termList/fileType/"), + ("schema", "https://schema.org/"), + ("gl", "http://fedora.quill.lan/rest/"), + ("glo", ONTOLOGY_PREFIX), + ].map(|(k, v)| (k.to_string(), v.to_string()))) +}); + +#[derive(Clone)] +pub struct CurieHelper { + prefixes: BTreeMap, +} + +impl CurieHelper { + pub fn new(prefixes: BTreeMap) -> Self { + Self { + prefixes, + } + } + + pub fn abbreviate(&self, base: Option<&str>, iri: &str) -> Option { + if let Some(base) = base && let Some(local_name) = iri.strip_prefix(base) { + return Some(format!(":{local_name}")); + } + + for (name, base) in &self.prefixes { + if let Some(local_name) = iri.strip_prefix(base) { + return Some(format!("{name}:{local_name}")); + } + } + None + } + + pub fn expand(&self, base: Option<&str>, abbreviated_iri: &str) -> Option { + let (prefix, name) = abbreviated_iri.split_once(':')?; + + if prefix == "" && let Some(base) = base { + Some(format!("{base}{name}")) + } else { + self.prefixes + .get(prefix) + .map(|base| format!("{base}{name}")) + } + } +} \ No newline at end of file diff --git a/graph/src/lib.rs b/graph/src/lib.rs new file mode 100644 index 0000000..1206aba --- /dev/null +++ b/graph/src/lib.rs @@ -0,0 +1,4 @@ +mod curie; +pub mod vocab; + +pub use curie::{PREFIXES, CurieHelper}; \ No newline at end of file diff --git a/graph/src/vocab.rs b/graph/src/vocab.rs new file mode 100644 index 0000000..743ee48 --- /dev/null +++ b/graph/src/vocab.rs @@ -0,0 +1,73 @@ +pub use oxigraph::model::vocab::rdf; +pub use oxigraph::model::vocab::rdfs; + +pub mod gl { + use oxigraph::model::NamedNodeRef; + + pub const TEMPLATE: NamedNodeRef = + NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/template"); + + pub const INDEXED_BY_FIELD: NamedNodeRef = + NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/indexedByField"); + + pub const CATEGORY_ID: NamedNodeRef = + NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/categoryId"); + + pub const SEARCHABLE_CLASS: NamedNodeRef = + NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/SearchableClass"); + + pub const ENTITY: NamedNodeRef = + NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/Entity"); + + pub const ASSOCIATED_PROPERTY: NamedNodeRef = + NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/associatedProperty"); + + pub const READ_ONLY: NamedNodeRef = + NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/readOnly"); +} + +pub mod skos { + use oxigraph::model::NamedNodeRef; + + pub const CONCEPT: NamedNodeRef = + NamedNodeRef::new_unchecked("http://www.w3.org/2004/02/skos/core#Concept"); +} + +pub mod owl { + use oxigraph::model::NamedNodeRef; + + pub const SAME_AS: NamedNodeRef = + NamedNodeRef::new_unchecked("http://www.w3.org/2002/07/owl#sameAs"); +} + +pub mod rdac { + use oxigraph::model::NamedNodeRef; + + pub const PERSON: NamedNodeRef = + NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10004"); + + pub const CORPORATE_BODY: NamedNodeRef = + NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10005"); + + pub const WORK: NamedNodeRef = + NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10001"); + + pub const EXPRESSION: NamedNodeRef = + NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10006"); + + pub const MANIFESTATION: NamedNodeRef = + NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10007"); +} + +pub mod rdaad { + use oxigraph::model::NamedNodeRef; + + pub const GIVEN_NAME: NamedNodeRef = + NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50292"); + + pub const SURNAME: NamedNodeRef = + NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50291"); + + pub const NAME_OF_CORPORATE_BODY: NamedNodeRef = + NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50032"); +} \ No newline at end of file diff --git a/publish/Cargo.toml b/publish/Cargo.toml index 9d72540..b91b381 100644 --- a/publish/Cargo.toml +++ b/publish/Cargo.toml @@ -4,15 +4,15 @@ version = "0.1.0" edition = "2024" [dependencies] -#gl-ldp.workspace = true +gl-graph.workspace = true gl-search.workspace = true -ldp.workspace = true clap.workspace = true color-eyre.workspace = true csv = "1.4" http.workspace = true iced.workspace = true +ldp.workspace = true rfd.workspace = true oxigraph.workspace = true oxilangtag.workspace = true diff --git a/publish/src/app.rs b/publish/src/app.rs index 9315e25..13cd20e 100644 --- a/publish/src/app.rs +++ b/publish/src/app.rs @@ -1,12 +1,11 @@ use crate::navigator::Navigator; -use crate::rdf::curie::CurieHelper; use crate::rdf::ontology::{LabeledIri, Ontology}; use crate::rdf::term_helper::{TermHelper, TermHelperMut}; -use crate::rdf::vocab::rda; use crate::widget::iri_input::iri_input; use crate::widget::navigation_area::navigation_area; +use gl_graph::vocab::rdac; use gl_search::language; -use gl_search::{IndexWriter, Schema, SearchDocument, SearchIndex, Value}; +use gl_search::{Schema, SearchDocument, SearchIndex, Value}; use http::StatusCode; use iced::alignment::Horizontal; use iced::keyboard::{Event, key}; @@ -25,6 +24,7 @@ use oxigraph::io::RdfFormat; use oxigraph::model::vocab::{rdf, rdfs}; use oxigraph::model::{BaseDirection, Dataset, NamedNode, NamedOrBlankNode, Quad, Term}; use tracing::{debug_span, error, trace}; +use gl_graph::{vocab, CurieHelper}; #[derive(Clone, Debug)] pub(crate) enum Message { @@ -816,7 +816,7 @@ impl Publisher { .collect(); let body: Element = if self.show_new_document_buttons { - let subclasses = self.ontology.subclasses_of(&rda::ENTITY.into_owned()); + let subclasses = Vec::new(); // TODO self.ontology.subclasses_of(vocab::rda::ENTITY.into_owned()); let labeled_subclasses = subclasses.iter().map(|iri| self.ontology.info(iri, &*language::ENGLISH_OR_UNTAGGED)); column![self.view_new_entity_buttons(labeled_subclasses)].into() } else { diff --git a/publish/src/main.rs b/publish/src/main.rs index a55b98f..29470d1 100644 --- a/publish/src/main.rs +++ b/publish/src/main.rs @@ -7,26 +7,20 @@ mod navigator; mod theme; mod args; -use std::sync::{Arc, RwLock}; -use std::thread; use clap::Parser; -use color_eyre::eyre::eyre; use iced::futures::StreamExt; -use iced::Task; use ldp::middleware::BasicAuthMiddleware; use ldp::reqwest::{Client, Url}; use ldp::reqwest_middleware::ClientBuilder; use ldp::traverse::Traverse; -use oxigraph::model::Dataset; use tracing::{debug, debug_span, error, field}; -use crate::app::{Message, Publisher}; +use crate::app::Publisher; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, fmt}; use tracing_subscriber::fmt::format::FmtSpan; -use gl_search::{DocType, Document, Schema, SearchIndex}; +use gl_search::{Document, Schema, SearchIndex}; use crate::args::{AppArgs, Command}; -use crate::rdf::curie::CurieHelper; use crate::rdf::ontology::Ontology; fn main() -> color_eyre::Result<()> { @@ -50,52 +44,25 @@ fn main() -> color_eyre::Result<()> { let args = AppArgs::parse(); match args.command { Some(Command::Search(args)) => { - for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 5)? { + for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? { println!("{}", doc.to_json(Schema::schema())); } } Some(Command::Reindex) => { - let curie_helper = CurieHelper::new(Ontology::prefixes().clone()); - - let writer = Arc::new(RwLock::new(index.writer()?)); + let mut writer = index.writer()?; debug_span!("Clear Index").in_scope(|| { - let mut writer = writer.write()?; writer.delete_all_documents()?; writer.commit() })?; - let store = ontology.store(); - let writer_clone = writer.clone(); - let index_classes_thread = thread::spawn(move || { - let span = debug_span!("Index Classes", documents = field::Empty).entered(); - let writer = writer_clone.read().unwrap(); - let count = gl_search::rdf::index_triples(DocType::RdfsClass, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &*writer)?; + { + let span = debug_span!("Index Schema", documents = field::Empty).entered(); + let store = ontology.store(); + let count = gl_search::rdf::index_schema(store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?; span.record("documents", count); - Ok::<(), gl_search::SearchError>(()) - }); - - let store = ontology.store(); - let writer_clone = writer.clone(); - let index_properties_thread = thread::spawn(move || { - let span = debug_span!("Index Properties", documents = field::Empty).entered(); - let writer = writer_clone.read().unwrap(); - let count = gl_search::rdf::index_triples(DocType::RdfProperty, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?; - span.record("documents", count); - Ok::<(), gl_search::SearchError>(()) - }); - - let store = ontology.store(); - let writer_clone = writer.clone(); - let index_concepts_thread = thread::spawn(move || { - let span = debug_span!("Index Concepts", documents = field::Empty).entered(); - let writer = writer_clone.read().unwrap(); - let count = gl_search::rdf::index_triples(DocType::SkosConcept, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?; - span.record("documents", count); - Ok::<(), gl_search::SearchError>(()) - }); + } let starting_url = Url::parse("http://fedora.quill.lan/rest/")?; - let writer_clone = writer.clone(); let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .name("repository-traversal") @@ -114,23 +81,17 @@ fn main() -> color_eyre::Result<()> { while let Some(result) = traversal.next().await { match result { Ok(rdf_source) => { - let writer = writer_clone.read().unwrap(); documents += gl_search::rdf::index_entity(rdf_source.dataset(), &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?; } Err(err) => error!(?err), } } debug!(documents, "Repository Traversal"); - Ok::<(), gl_search::SearchError>(()) + Ok::<_, gl_search::SearchError>(writer) }); - index_classes_thread.join().unwrap()?; - index_properties_thread.join().unwrap()?; - index_concepts_thread.join().unwrap()?; - runtime.block_on(traversal_task)??; - + let mut writer = runtime.block_on(traversal_task)??; debug_span!("Commit").in_scope(|| { - let mut writer = writer.write()?; writer.commit() })?; } diff --git a/publish/src/rdf/curie.rs b/publish/src/rdf/curie.rs deleted file mode 100644 index fe1cd30..0000000 --- a/publish/src/rdf/curie.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::collections::BTreeMap; - -#[derive(Clone)] -pub struct CurieHelper { - prefixes: BTreeMap, -} - -impl CurieHelper { - pub fn new(prefixes: BTreeMap) -> Self { - Self { - prefixes, - } - } - - pub fn abbreviate(&self, base: Option<&str>, iri: &str) -> Option { - if let Some(base) = base && let Some(local_name) = iri.strip_prefix(base) { - return Some(format!(":{local_name}")); - } - - for (name, base) in &self.prefixes { - if let Some(local_name) = iri.strip_prefix(base) { - return Some(format!("{name}:{local_name}")); - } - } - None - } - - pub fn expand(&self, base: Option<&str>, abbreviated_iri: &str) -> Option { - let (prefix, name) = abbreviated_iri.split_once(':')?; - - if prefix == "" && let Some(base) = base { - Some(format!("{base}{name}")) - } else { - self.prefixes - .get(prefix) - .map(|base| format!("{base}{name}")) - } - } -} \ No newline at end of file diff --git a/publish/src/rdf/materialize.rs b/publish/src/rdf/materialize.rs index a0eac69..4ffef2e 100644 --- a/publish/src/rdf/materialize.rs +++ b/publish/src/rdf/materialize.rs @@ -1,9 +1,9 @@ +use crate::error; +use gl_graph::vocab::owl; use oxigraph::model::{Dataset, GraphName, NamedNodeRef, NamedOrBlankNode, NamedOrBlankNodeRef, Term}; use oxigraph::sparql::SparqlEvaluator; use oxigraph::store::Store; use tracing::{debug, debug_span}; -use crate::error; -use crate::rdf::vocab::owl; const INFERENCE_GRAPH: NamedNodeRef = NamedNodeRef::new_unchecked("https://graphofliberty.org/inference"); const RDF_SCHEMA: NamedNodeRef = NamedNodeRef::new_unchecked("http://www.w3.org/2000/01/rdf-schema#"); diff --git a/publish/src/rdf/mod.rs b/publish/src/rdf/mod.rs index ee9b7f6..6adaefb 100644 --- a/publish/src/rdf/mod.rs +++ b/publish/src/rdf/mod.rs @@ -1,6 +1,4 @@ pub(crate) mod ontology; pub(crate) mod term_helper; -pub mod vocab; pub(crate) mod materialize; -pub(crate) mod conversion; -pub(crate) mod curie; \ No newline at end of file +pub(crate) mod conversion; \ No newline at end of file diff --git a/publish/src/rdf/ontology.rs b/publish/src/rdf/ontology.rs index 3ddb5ea..bac93fa 100644 --- a/publish/src/rdf/ontology.rs +++ b/publish/src/rdf/ontology.rs @@ -1,6 +1,6 @@ use crate::error; -use crate::rdf::vocab::gl; use crate::rdf::{conversion, materialize}; +use gl_graph::vocab::gl; use gl_search::language::LanguageCondition; use iced::futures::TryFutureExt; use oxigraph::model::vocab::{rdf, rdfs, xsd}; diff --git a/publish/src/rdf/vocab.rs b/publish/src/rdf/vocab.rs deleted file mode 100644 index 99a19a3..0000000 --- a/publish/src/rdf/vocab.rs +++ /dev/null @@ -1,38 +0,0 @@ -pub mod gl { - use oxigraph::model::NamedNodeRef; - - pub const TEMPLATE: NamedNodeRef = - NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/template"); - - pub const INDEXED_BY_FIELD: NamedNodeRef = - NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/indexedByField"); - - pub const CATEGORY_ID: NamedNodeRef = - NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/categoryId"); - - pub const SEARCHABLE_CLASS: NamedNodeRef = - NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/SearchableClass"); - - pub const ENTITY: NamedNodeRef = - NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/Entity"); - - pub const ASSOCIATED_PROPERTY: NamedNodeRef = - NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/associatedProperty"); - - pub const READ_ONLY: NamedNodeRef = - NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/readOnly"); -} - -pub mod owl { - use oxigraph::model::NamedNodeRef; - - pub const SAME_AS: NamedNodeRef = - NamedNodeRef::new_unchecked("http://www.w3.org/2002/07/owl#sameAs"); -} - -pub mod rda { - use oxigraph::model::NamedNodeRef; - - pub const ENTITY: NamedNodeRef = - NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10013"); -} diff --git a/publish/src/widget/iri_input.rs b/publish/src/widget/iri_input.rs index e036fa6..beb2d6d 100644 --- a/publish/src/widget/iri_input.rs +++ b/publish/src/widget/iri_input.rs @@ -1,4 +1,4 @@ -use iced::{alignment, keyboard, widget, Element, Event, Length, Rectangle, Size}; +use iced::{keyboard, widget, Element, Event, Length, Rectangle, Size}; use iced::advanced::{mouse, text, Shell, renderer}; use iced::advanced::layout::{Limits, Node}; use iced::advanced::{Layout, Widget}; @@ -6,7 +6,7 @@ use iced::advanced::widget::{tree, Tree, Operation}; use iced::clipboard::Content; use iced::mouse::{Cursor, Interaction}; use iced::widget::text_input::{Catalog, Status, Style, StyleFn}; -use crate::rdf::curie::CurieHelper; +use gl_graph::CurieHelper; pub struct State { control: bool, diff --git a/search/Cargo.toml b/search/Cargo.toml index a419f9e..3fbd463 100644 --- a/search/Cargo.toml +++ b/search/Cargo.toml @@ -5,10 +5,11 @@ edition = "2024" publish = false [dependencies] +gl-graph.workspace = true + futures.workspace = true tantivy.workspace = true thiserror.workspace = true - #poppler-rs = "0.26.0-alpha.0" subtitler.workspace = true tracing.workspace = true diff --git a/search/src/error.rs b/search/src/error.rs index 651e746..783fff0 100644 --- a/search/src/error.rs +++ b/search/src/error.rs @@ -23,7 +23,4 @@ pub enum SearchError { #[error(transparent)] SparqlSyntax(#[from] oxigraph::sparql::SparqlSyntaxError), - - #[error("The provided DocType is not valid for this operation")] - InvalidDocType, } diff --git a/search/src/lib.rs b/search/src/lib.rs index 1490a10..82e1327 100644 --- a/search/src/lib.rs +++ b/search/src/lib.rs @@ -12,6 +12,7 @@ pub use tantivy::schema::document::{Document, Value}; pub use tantivy::indexer::IndexWriter; pub use error::{Result, SearchError}; +use gl_graph::vocab; pub use index::{SearchIndex, SearchIndexBuilder}; pub use schema::Schema; @@ -20,12 +21,37 @@ pub enum DocType { RdfsClass = 1, SkosConcept = 2, Person = 3, + CorporateBody = 4, + Work = 5, + Expression = 6, + Manifestation = 7, } impl DocType { - pub fn from_named_node(node: NamedNodeRef) -> Option { - match node.as_str() { - "http://rdaregistry.info/Elements/c/C10004" => Some(DocType::Person), + pub fn to_named_node(&self) -> NamedNodeRef<'_> { + match self { + DocType::RdfProperty => vocab::rdf::PROPERTY, + DocType::RdfsClass => vocab::rdfs::CLASS, + DocType::SkosConcept => vocab::skos::CONCEPT, + DocType::Person => vocab::rdac::PERSON, + DocType::CorporateBody => vocab::rdac::CORPORATE_BODY, + DocType::Work => vocab::rdac::WORK, + DocType::Expression => vocab::rdac::EXPRESSION, + DocType::Manifestation => vocab::rdac::MANIFESTATION, + } + } + + pub fn from_named_node<'a>(node: impl Into>) -> Option { + let node = node.into(); + match node { + vocab::rdf::PROPERTY => Some(DocType::RdfProperty), + vocab::rdfs::CLASS => Some(DocType::RdfsClass), + vocab::skos::CONCEPT => Some(DocType::SkosConcept), + vocab::rdac::PERSON => Some(DocType::Person), + vocab::rdac::CORPORATE_BODY => Some(DocType::CorporateBody), + vocab::rdac::WORK => Some(DocType::Work), + vocab::rdac::EXPRESSION => Some(DocType::Expression), + vocab::rdac::MANIFESTATION => Some(DocType::Manifestation), _ => None, } } diff --git a/search/src/rdf.rs b/search/src/rdf.rs index ee12100..557c5ab 100644 --- a/search/src/rdf.rs +++ b/search/src/rdf.rs @@ -5,10 +5,11 @@ use oxigraph::sparql::{QueryResults, SparqlEvaluator}; use oxigraph::store::Store; use tantivy::IndexWriter; use tantivy::schema::{Field, OwnedValue}; +use tracing::debug; +use gl_graph::{vocab, CurieHelper}; use crate::DocType; use crate::language::LanguageCondition; use crate::schema::Schema; -use crate::SearchError::InvalidDocType; fn term_to_named_node(term: &Term) -> Option<&NamedNode> { if let Term::NamedNode(node) = term { @@ -52,16 +53,15 @@ fn term_ref_as_str(term: TermRef<'_>) -> Option<&str> { } } -const GIVEN_NAME: NamedNodeRef = NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50292"); -const SURNAME: NamedNodeRef = NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50291"); - pub fn index_entity(dataset: &Dataset, language: &LanguageCondition, writer: &IndexWriter>) -> crate::Result { + let curie_helper = CurieHelper::new((&*gl_graph::PREFIXES).clone()); + let mut subject_type_map = HashMap::new(); for quad in dataset.quads_for_pattern(None, Some(rdf::TYPE), None, None) { if let NamedOrBlankNodeRef::NamedNode(subject) = quad.subject { subject_type_map.entry(subject) .and_modify(|types: &mut Vec<_>| types.push(quad.object)) - .or_insert_with(Vec::new); + .or_insert_with(|| vec![quad.object]); } } @@ -69,16 +69,24 @@ pub fn index_entity(dataset: &Dataset, language: &LanguageCondition, writer: &In for (subject, types) in subject_type_map { for type_ in types { if let Some(doc_type) = term_ref_as_named_node(type_).and_then(DocType::from_named_node) { - match doc_type { - DocType::Person => { - let mut doc = index_person(dataset, subject, language); - doc.insert(Schema::discriminant_field(), OwnedValue::U64(doc_type as u64)); - doc.insert(Schema::iri_field(), OwnedValue::Str(subject.as_str().to_string())); - writer.add_document(doc)?; - counter += 1; - } - _ => {} - } + let mut doc = match doc_type { + DocType::Person => index_person(dataset, subject, language), + DocType::CorporateBody => index_corporate_body(dataset, subject, language), + _ => continue + }; + + let subject_str = subject.as_str(); + + doc.insert(Schema::discriminant_field(), OwnedValue::U64(doc_type as u64)); + doc.insert(Schema::iri_field(), OwnedValue::Str(subject_str.to_string())); + + let curie = curie_helper.abbreviate(None, subject_str) + .map(OwnedValue::Str) + .unwrap_or(OwnedValue::Null); + doc.insert(Schema::curie_field(), curie); + + writer.add_document(doc)?; + counter += 1; } } } @@ -87,7 +95,7 @@ pub fn index_entity(dataset: &Dataset, language: &LanguageCondition, writer: &In } fn index_person(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &LanguageCondition) -> HashMap { - let given_name = dataset.quads_for_pattern(Some(subject.into()), Some(GIVEN_NAME), None, None) + let given_name = dataset.quads_for_pattern(Some(subject.into()), Some(vocab::rdaad::GIVEN_NAME), None, None) .map(|q| q.object) .filter(|term| language.primary_matches_term(*term)) .filter_map(term_ref_as_str) @@ -96,7 +104,7 @@ fn index_person(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &Languag .next() .unwrap_or_else(|| OwnedValue::Null); - let surname = dataset.quads_for_pattern(Some(subject.into()), Some(SURNAME), None, None) + let surname = dataset.quads_for_pattern(Some(subject.into()), Some(vocab::rdaad::SURNAME), None, None) .map(|q| q.object) .filter(|term| language.primary_matches_term(*term)) .filter_map(term_ref_as_str) @@ -111,18 +119,30 @@ fn index_person(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &Languag ]) } -pub fn index_triples(doc_type: DocType, store: Store, language: &LanguageCondition, writer: &IndexWriter>) -> crate::Result { +fn index_corporate_body(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &LanguageCondition) -> HashMap { + let name = dataset.quads_for_pattern(Some(subject.into()), Some(vocab::rdaad::NAME_OF_CORPORATE_BODY), None, None) + .map(|q| q.object) + .filter(|term| language.primary_matches_term(*term)) + .filter_map(term_ref_as_str) + .map(String::from) + .map(OwnedValue::Str) + .next() + .unwrap_or_else(|| OwnedValue::Null); + + HashMap::from_iter([ + (Schema::field("corporate_name", language.primary_language()), name), + ]) +} + +pub fn index_schema(store: Store, language: &LanguageCondition, writer: &IndexWriter>) -> crate::Result { + let curie_helper = CurieHelper::new((&*gl_graph::PREFIXES).clone()); + let label_filter = language.to_filter_expression("label"); let description_filter = language.to_filter_expression("description"); - let rdf_type = match doc_type { - DocType::RdfsClass => Ok("rdfs:Class"), - DocType::RdfProperty => Ok("rdf:Property"), - DocType::SkosConcept => Ok("skos:Concept"), - _ => Err(InvalidDocType), - }?; - let query = format!(r#"SELECT ?subject ?label ?description WHERE {{ - ?subject a {rdf_type} ; + let query = format!(r#"SELECT ?class ?subject ?label ?description WHERE {{ + VALUES ?class {{ rdf:Property rdfs:Class skos:Concept }} + ?subject a ?class ; rdfs:label ?label . {label_filter} @@ -141,17 +161,32 @@ pub fn index_triples(doc_type: DocType, store: Store, language: &LanguageConditi let mut counter = 0usize; let query_results = sparql.on_store(&store).execute()?; if let QueryResults::Solutions(solutions) = query_results { - let discriminant = OwnedValue::U64(doc_type as u64); let primary_language = language.primary_language(); + let label_field = Schema::field("label", primary_language); + let definition_field = Schema::field("definition", primary_language); for solution in solutions.filter_map(Result::ok) { let mut document = HashMap::with_capacity(4); - document.insert(Schema::discriminant_field(), discriminant.clone()); - let subject = solution.get("subject") + let discriminant = solution.get("class") .and_then(term_to_named_node) - .map(NamedNode::as_str) - .map(OwnedValue::from) + .and_then(DocType::from_named_node) + .map(|doc_type| doc_type as u64) + .map(OwnedValue::U64) + .unwrap_or(OwnedValue::Null); + document.insert(Schema::discriminant_field(), discriminant); + + let subject_str = solution.get("subject") + .and_then(term_to_named_node) + .map(NamedNode::as_str); + + let curie = subject_str + .and_then(|subject| curie_helper.abbreviate(None, subject)) + .map(OwnedValue::Str) + .unwrap_or(OwnedValue::Null); + document.insert(Schema::curie_field(), curie); + + let subject = subject_str.map(OwnedValue::from) .unwrap_or(OwnedValue::Null); document.insert(Schema::iri_field(), subject); @@ -159,13 +194,13 @@ pub fn index_triples(doc_type: DocType, store: Store, language: &LanguageConditi .and_then(term_as_str) .map(OwnedValue::from) .unwrap_or(OwnedValue::Null); - document.insert(Schema::field("label", primary_language), label); + document.insert(label_field, label); let definition = solution.get("description") .and_then(term_as_str) .map(OwnedValue::from) .unwrap_or(OwnedValue::Null); - document.insert(Schema::field("definition", primary_language), definition); + document.insert(definition_field, definition); writer.add_document(document)?; counter += 1; diff --git a/search/src/schema.rs b/search/src/schema.rs index 11d8da3..a78c544 100644 --- a/search/src/schema.rs +++ b/search/src/schema.rs @@ -33,6 +33,7 @@ impl Schema { schema_builder.add_text_field("surname:en", stored_ngram32.clone()); schema_builder.add_text_field("given_name:en", stored_ngram32.clone()); + schema_builder.add_text_field("corporate_name:en", stored_ngram32.clone()); schema_builder.add_u64_field("subtitle_start", schema::STORED); schema_builder.add_u64_field("subtitle_end", schema::STORED);