From 5cea8e83078a889d24d580b3829785e3a191cd8536f5eef639db008fe0c0a0ed Mon Sep 17 00:00:00 2001 From: Alex Wied <543423+centromere@users.noreply.github.com> Date: Mon, 29 Jun 2026 23:10:22 -0400 Subject: [PATCH] . --- Cargo.toml | 2 +- publish/src/app.rs | 131 ++++++++++++------------ publish/src/rdf/ontologies/ontology.ttl | 18 +++- publish/src/rdf/ontology.rs | 63 +++++++----- publish/src/widget/iri_input.rs | 2 - search/src/index.rs | 27 +++-- search/src/lib.rs | 1 + 7 files changed, 129 insertions(+), 115 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 20f31da..b02f6a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,4 +27,4 @@ thiserror = "2.0" tokio = { version = "1.52", features = ["rt", "rt-multi-thread", "macros", "fs"] } tracing = "0.1" tracing-appender = "0.2" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = { version = "0.3", features = ["env-filter"] } \ No newline at end of file diff --git a/publish/src/app.rs b/publish/src/app.rs index 5dd9a3c..3dbc7ca 100644 --- a/publish/src/app.rs +++ b/publish/src/app.rs @@ -1,7 +1,7 @@ use crate::rdf::ontology::{LabeledIri, Ontology}; use crate::rdf::term_helper::{TermHelper, TermHelperMut}; use crate::rdf::vocab::{gl, rda}; -use gl_search::{Schema, SearchDocument, SearchIndex, NamedFieldDocument, OwnedValue, doc}; +use gl_search::{Schema, SearchDocument, SearchIndex, NamedFieldDocument, OwnedValue, doc, IndexWriter}; use http::StatusCode; use iced::alignment::Horizontal; use iced::widget::button::Style; @@ -43,6 +43,7 @@ pub(crate) enum Message { HoverRow(QuadKey), UnhoverRow(QuadKey), QueryUpdated(String), + SetSearchResults(Vec), QueryTypeUpdated(NamedNode), SearchResultClicked(NamedNode), DatatypeUpdated(QuadKey, Option), @@ -88,6 +89,7 @@ pub(crate) struct Publisher { hovered_row: Option, search_state: Option, index: SearchIndex, + index_writer: Option, show_overwrite_confirmation: bool, modified: bool, show_new_document_buttons: bool, @@ -108,26 +110,39 @@ impl Publisher { .build() .expect("Failed to build search index"); - if let Some(id) = ontology.catalog_id(&rdf::PROPERTY.into_owned()) { index.remove_all_of_type(id); } - if let Some(id) = ontology.catalog_id(&rdfs::CLASS.into_owned()) { index.remove_all_of_type(id); } + let writer = index.writer().expect("Failed to create search index writer"); - for (iri, entity) in ontology.entities() { - let mut document = doc!( - Schema::type_field() => entity.catalog_id, - Schema::iri_field() => iri.as_str(), - ); + let classes_to_index = [ + rdf::PROPERTY, + rdfs::CLASS, + ]; - if let Some(field) = ontology.field_for_property(&rdfs::LABEL.into_owned()) { - document.add_text(Schema::field(&field.name), &entity.label); + for class in &classes_to_index { + let class = class.into_owned(); + if let Some(id) = ontology.catalog_id(&class) { + index.remove_all_of_type(id).expect("Failed to remove documents from search index"); + for individual in ontology.individuals(&class) { + let info = ontology.info(&individual); + if info.label.is_some() || info.comment.is_some() { + let mut document = doc!( + Schema::type_field() => id, + Schema::iri_field() => individual.as_str(), + ); + + if let Some(field) = ontology.field_for_property(&rdfs::LABEL.into_owned()) { + document.add_text(Schema::field(&field.name), info.label.unwrap_or_default()); + } + + if let Some(field) = ontology.field_for_property(&rdfs::COMMENT.into_owned()) { + document.add_text(Schema::field(&field.name), info.comment.unwrap_or_default()); + } + + writer.add_document(document).expect("Failed to add document to search index"); + } + } } - - if let Some(field) = ontology.field_for_property(&rdfs::COMMENT.into_owned()) { - document.add_text(Schema::field(&field.name), entity.comment.clone().unwrap_or(String::from(""))); - } - - index.add(document).expect("Unable to add annotated IRI to search index"); } - index.commit().expect("Unable to commit ontology to index"); + index }); @@ -161,6 +176,7 @@ impl Publisher { hovered_row: None, search_state: None, index, + index_writer: None, show_overwrite_confirmation: false, modified: false, show_new_document_buttons: false, @@ -178,6 +194,9 @@ impl Publisher { let client = self.http_client.clone(); let root = Url::parse(&self.url_input).expect("Invalid URL"); let stream = Traverse::new(client, root, None); + + self.index_writer = Some(self.index.writer().expect("Unable to create search index writer")); + task = Task::run(stream, |result| match result { Ok(rdf_source) => Message::IndexRdfSource(rdf_source), Err(err) => Message::ShowError(format!("Unable to fetch RDF Source: {err}")), @@ -203,14 +222,14 @@ impl Publisher { } } - self.index - .add(document) - .expect("Unable to add document to search index"); + if let Some(writer) = &self.index_writer { + writer.add_document(document).expect("Unable to add document to search index"); + } } } Message::CommitIndex => { - if let Err(err) = self.index.commit() { - task = Task::done(Message::ShowError(format!("Unable to commit index: {err}"))); + if let Some(writer) = &mut self.index_writer { + writer.commit().expect("Unable to commit updates to search index"); } } Message::WindowClosed(id) => { @@ -349,13 +368,22 @@ impl Publisher { SearchResultClickAction::URLInput => self.ontology.catalog_id(&search_state.type_.iri), }; - search_state.results = self - .index - .query(catalog_id, new_query.as_str(), Schema::all_fields()) - .expect("Error encountered while querying index"); - search_state.query = new_query; + search_state.query = new_query.clone(); + let index = self.index.clone(); + task = Task::future(async move { + tokio::task::spawn_blocking(move || { + let result = index.query(catalog_id, new_query.as_str(), Schema::all_fields()) + .expect("Error encountered while querying index"); + Message::SetSearchResults(result) + }).await.unwrap() + }); }; } + Message::SetSearchResults(results) => { + if let Some(search_state) = &mut self.search_state { + search_state.results = results; + } + } Message::SearchResultClicked(node) => { if let Some(search_state) = &self.search_state { match search_state.action { @@ -542,12 +570,12 @@ impl Publisher { let term = TermHelper::new(&triple.object); - let value_label = "bar"; /*term.value_as_named_node().and_then(|node| { + let value_label = term.value_as_named_node().and_then(|node| { self.ontology - .info(node) - .and_then(|info| info.label.clone()) + .info(&node.into_owned()) + .label .map(|label| container(text(label))) - });*/ + }); let value = term .value_as_named_node() @@ -563,43 +591,10 @@ impl Publisher { }) .unwrap_or(Horizontal::Left); - /*let value_input_base = text_input("Value", value.as_str()) - .align_x(value_alignment);*/ - - /*let value_input = if !state.read_only { - let is_named_node = term.is_named_node(); - value_input_base.on_input(move |input| { - let expanded_input = if is_named_node { - self.ontology - .expand(input.as_str()) - .map(|node| node.as_str().to_string()) - .unwrap_or(input) - } else { - input - }; - - Message::ValueUpdated(key, expanded_input) - }) - } else { - value_input_base - };*/ - let foo = iri_input(self.ontology.prefixes(), "Object", value.as_str()) .on_input(move |value| Message::ValueUpdated(key, value)) .on_control_click(Message::OpenQueryWindow(SearchResultClickAction::Object(key))); - /*let search_launcher = if term.is_named_node() && !state.read_only { - Some(container( - button(text("\u{1f50e}")) - .on_press(Message::OpenQueryWindow(SearchResultClickAction::Object( - key, - ))) - .style(button::text), - )) - } else { - None - };*/ - let selected_datatype = term.datatype().map(|node| self.ontology.abbreviate(node)); let datatype_selector: Element = if state.read_only { selected_datatype.map(text).into() @@ -671,11 +666,11 @@ impl Publisher { entities: impl Iterator, ) -> Element<'_, Message> { let buttons = entities.map(|entity| { - let label = "baz"; /*self + let label = self .ontology - .info(entity.as_ref()) - .and_then(|info| info.label.clone()) - .unwrap_or(entity.as_str().to_string());*/ + .info(&entity) + .label + .unwrap_or(entity.as_str().to_string()); button(text(label)) .on_press(Message::NewDocument(entity)) .into() diff --git a/publish/src/rdf/ontologies/ontology.ttl b/publish/src/rdf/ontologies/ontology.ttl index faf93da..a1a14ef 100644 --- a/publish/src/rdf/ontologies/ontology.ttl +++ b/publish/src/rdf/ontologies/ontology.ttl @@ -214,18 +214,28 @@ fedora:lastModifiedBy rdf:type owl:NamedIndividual ; ### http://www.w3.org/1999/02/22-rdf-syntax-ns#Property rdf:Property rdf:type owl:NamedIndividual ; - :associatedProperty :comment , - :label ; + :associatedProperty rdfs:comment , + rdfs:label ; :catalogId "0"^^xsd:nonNegativeInteger . ### http://www.w3.org/2000/01/rdf-schema#Class rdfs:Class rdf:type owl:NamedIndividual ; - :associatedProperty :comment , - :label ; + :associatedProperty rdfs:comment , + rdfs:label ; :catalogId "1"^^xsd:nonNegativeInteger . +### http://www.w3.org/2000/01/rdf-schema#comment +rdfs:comment rdf:type owl:NamedIndividual ; + :indexedByField :comment . + + +### http://www.w3.org/2000/01/rdf-schema#label +rdfs:label rdf:type owl:NamedIndividual ; + :indexedByField :label . + + ### http://www.w3.org/ns/ldp#BasicContainer ldp:BasicContainer rdf:type owl:NamedIndividual ; :readOnly "true"^^xsd:boolean . diff --git a/publish/src/rdf/ontology.rs b/publish/src/rdf/ontology.rs index c1aeaa6..febdc58 100644 --- a/publish/src/rdf/ontology.rs +++ b/publish/src/rdf/ontology.rs @@ -135,10 +135,12 @@ impl OntologyBuilder { PREFIX gl: SELECT DISTINCT ?subject ?name ?label { - ?subject a gl:IndexField ; - gl:fieldName ?name ; - gl:fieldLabel ?label . - FILTER (langMATCHES(LANG(?label), "en") || !hasLANG(?label)) + GRAPH ?graph { + ?subject a gl:IndexDocumentField ; + gl:fieldName ?name ; + gl:fieldLabel ?label . + FILTER (langMATCHES(LANG(?label), "en") || !hasLANG(?label)) + } }"#, ) .expect("Unable to parse field query"); @@ -195,7 +197,7 @@ SELECT ?class {{ #[derive(Clone, Debug)] pub struct LabeledIri { pub iri: NamedNode, - pub label: String, + pub label: Option, pub comment: Option, } @@ -207,18 +209,10 @@ impl PartialEq for LabeledIri { impl Display for LabeledIri { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.label.clone()) + write!(f, "{}", self.label.clone().unwrap_or_default()) } } -#[derive(Clone, Debug)] -pub struct IriInformation { - pub type_: NamedNode, - pub label: Option, - pub comment: Option, - pub read_only: bool, -} - #[derive(Clone, Debug)] pub struct Entity { pub label: String, @@ -302,11 +296,39 @@ impl Ontology { .map(|entity| entity.catalog_id) } + pub fn individuals(&self, class: &NamedNode) -> impl Iterator { + self.store + .quads_for_pattern(None, Some(rdf::TYPE), Some(class.into()), None) + .filter_map(Result::ok) + .filter_map(|quad| + if let NamedOrBlankNode::NamedNode(subject) = quad.subject { + Some(subject) + } else { None }) + } + + pub fn info(&self, iri: &NamedNode) -> LabeledIri { + let label = self.store.quads_for_pattern(Some(iri.as_ref().into()), Some(rdfs::LABEL), None, None) + .filter_map(Result::ok) + .filter_map(|quad| conversion::term_to_string(&quad.object).map(String::from)) + .next(); + + let comment = self.store.quads_for_pattern(Some(iri.as_ref().into()), Some(rdfs::COMMENT), None, None) + .filter_map(Result::ok) + .filter_map(|quad| conversion::term_to_string(&quad.object).map(String::from)) + .next(); + + LabeledIri { + iri: iri.clone(), + label, + comment, + } + } + pub fn labeled_entity(&self, class: &NamedNode) -> Option { self.entities.get(class) .map(|entity| LabeledIri { iri: class.to_owned(), - label: entity.label.to_owned(), + label: Some(entity.label.to_owned()), comment: entity.comment.to_owned(), }) } @@ -315,15 +337,11 @@ impl Ontology { self.entities.iter() .map(|(iri, entity)| LabeledIri { iri: iri.to_owned(), - label: entity.label.to_owned(), + label: Some(entity.label.to_owned()), comment: entity.comment.to_owned(), }) } - pub fn entities(&self) -> impl Iterator { - self.entities.iter() - } - pub fn datatypes(&self) -> impl Iterator { self.store .quads_for_pattern( @@ -354,11 +372,6 @@ impl Ontology { .count() >= 1 } - /*pub fn for_each_annotated_(&self) -> impl Iterator> { - self.store - .quads_for_pattern() - }*/ - pub fn template_triples<'a>( &'a self, class: NamedNodeRef<'a>, diff --git a/publish/src/widget/iri_input.rs b/publish/src/widget/iri_input.rs index 8157f2d..34d7200 100644 --- a/publish/src/widget/iri_input.rs +++ b/publish/src/widget/iri_input.rs @@ -19,7 +19,6 @@ where { prefixes: &'a BTreeMap, on_control_click: Option, - on_input: Option Message + 'a>>, text_input: widget::TextInput<'a, Message, Theme, Renderer>, } @@ -51,7 +50,6 @@ where Self { prefixes, on_control_click: None, - on_input: None, text_input, } } diff --git a/search/src/index.rs b/search/src/index.rs index 17f8d31..f8e871b 100644 --- a/search/src/index.rs +++ b/search/src/index.rs @@ -1,5 +1,4 @@ use crate::{error, SearchDocument}; -use crate::error::SearchError; use crate::schema::Schema; use std::path::PathBuf; use tantivy::collector::TopDocs; @@ -8,6 +7,8 @@ use tantivy::query::{BooleanQuery, Occur, QueryParser, TermQuery}; use tantivy::schema::{Field, IndexRecordOption, NamedFieldDocument, Value}; use tantivy::tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer, TokenizerManager}; use tantivy::{Document, Index, IndexReader, IndexWriter, ReloadPolicy, Score, TantivyDocument, Term}; +use tantivy::indexer::UserOperation; +use tracing::{info, instrument, span, Level}; #[derive(Default)] pub struct SearchIndexBuilder { @@ -46,20 +47,17 @@ impl SearchIndexBuilder { .reload_policy(ReloadPolicy::OnCommitWithDelay) .try_into()?; - let writer = index.writer(50_000_000)?; - Ok(SearchIndex { index, reader, - writer, }) } } +#[derive(Clone)] pub struct SearchIndex { index: Index, reader: IndexReader, - writer: IndexWriter, } impl SearchIndex { @@ -67,18 +65,14 @@ impl SearchIndex { SearchIndexBuilder::default() } - pub fn add<'a>(&self, document: TantivyDocument) -> crate::Result<()> { - self.writer.add_document(document)?; - Ok(()) + pub fn writer(&self) -> crate::Result { + Ok(self.index.writer(128_000_000)?) } - pub fn remove_all_of_type(&mut self, type_: u64) { - self.writer - .delete_term(Term::from_field_u64(Schema::type_field(), type_)); - } - - pub fn commit(&mut self) -> crate::Result<()> { - self.writer.commit()?; + pub fn remove_all_of_type(&mut self, type_: u64) -> crate::Result<()> { + let mut writer: IndexWriter = self.index.writer(64_000_000)?; + writer.delete_term(Term::from_field_u64(Schema::type_field(), type_)); + writer.commit()?; Ok(()) } @@ -88,6 +82,9 @@ impl SearchIndex { user_query: &str, default_fields: Vec, ) -> error::Result> { + let span = span!(Level::INFO, "Search Query"); + let _enter = span.enter(); + let parser = QueryParser::for_index(&self.index, default_fields); let (user_query, _) = parser.parse_query_lenient(user_query); diff --git a/search/src/lib.rs b/search/src/lib.rs index f94c46b..6bb9d8f 100644 --- a/search/src/lib.rs +++ b/search/src/lib.rs @@ -5,6 +5,7 @@ mod schema; pub use tantivy::TantivyDocument as SearchDocument; pub use tantivy::doc; pub use tantivy::schema::{NamedFieldDocument, OwnedValue}; +pub use tantivy::IndexWriter; pub use error::{Result, SearchError}; pub use index::{SearchIndex, SearchIndexBuilder};