.

publish/src/app.rs

@@ -1,4 +1,5 @@
-use crate::rdf::ontology::{LabeledIri, Ontology};
+use std::collections::HashMap;
+use crate::rdf::ontology::{IndexEntry, LabeledIri, Ontology};
 use crate::rdf::term_helper::{TermHelper, TermHelperMut};
 use gl_search::{Schema, SearchDocument, SearchIndex, doc, IndexWriter, Value};
 use http::StatusCode;
@@ -17,9 +18,10 @@ use ldp::traverse::Traverse;
 use ldp::{RdfSource, RdfSourceUpdateResponse, ResourceRequestBuilder, SerializationOptions};
 use oxigraph::io::RdfFormat;
 use oxigraph::model::vocab::{rdf, rdfs};
-use oxigraph::model::{BaseDirection, BlankNode, Dataset, NamedNode, NamedNodeRef, NamedOrBlankNode, Quad, Term, Triple};
-use tracing::{debug, debug_span, error, info, trace};
-use crate::app::Message::URLInputSubmitted;
+use oxigraph::model::{BaseDirection, Dataset, NamedNode, Quad, Term};
+use tracing::{debug, debug_span, error, trace};
+use tracing::span::EnteredSpan;
+use crate::rdf::curie::CurieHelper;
 use crate::rdf::language;
 use crate::rdf::vocab::rda;
 use crate::widget::iri_input::iri_input;
@@ -27,9 +29,11 @@ use crate::widget::iri_input::iri_input;
 #[derive(Clone, Debug)]
 pub(crate) enum Message {
     None,
+    RebuildIndex,
     Traverse,
-    IndexRdfSource(RdfSource<Dataset>),
-    CommitIndex,
+    AddRdfSource(RdfSource<Dataset>),
+    ConcludeTraversal,
+    IndexQueryResults(HashMap<NamedNode, IndexEntry>),
     WindowClosed(window::Id),
     URLInputChanged(String),
     URLInputSubmitted,
@@ -86,6 +90,7 @@ struct SearchState {
 
 pub(crate) struct Publisher {
     http_client: ClientWithMiddleware,
+    curie_helper: CurieHelper,
     ontology: Ontology,
     abbreviated_datatypes: Vec<String>,
     window_id: window::Id,
@@ -94,7 +99,7 @@ pub(crate) struct Publisher {
     hovered_row: Option<QuadKey>,
     search_state: Option<SearchState>,
     index: SearchIndex,
-    index_writer: Option<IndexWriter>,
+    traversal: Option<(Dataset, EnteredSpan)>,
     show_overwrite_confirmation: bool,
     modified: bool,
     show_new_document_buttons: bool,
@@ -102,6 +107,8 @@ pub(crate) struct Publisher {
 
 impl Publisher {
     pub(crate) fn new() -> (Self, Task<Message>) {
+        let curie_helper = CurieHelper::new(Ontology::prefixes().clone());
+
         let ontology = debug_span!("Ontology Creation").in_scope(|| {
             Ontology::builder()
                 .with_path("/home/alex/.local/share/org.graphofliberty.desktop/ontology")
@@ -114,31 +121,12 @@ impl Publisher {
             .build()
             .expect("Failed to build search index");
 
-        /*debug_span!("Ontology Indexing").in_scope(|| {
-            let mut counter = 0;
-            let mut writer = index.writer().expect("Failed to build index writer");
-            let index = ontology.index(&*language::ENGLISH_OR_UNTAGGED).expect("Unable to generate index of entities");
-            for (individual, entry) in index {
-                let mut document = doc!(
-                    Schema::type_field() => entry.catalog_id,
-                    Schema::iri_field() => individual.as_str(),
-                    Schema::curie_field() => ontology.abbreviate(individual.as_ref()),
-                );
-                for (key, value) in entry.fields {
-                    document.add_text(Schema::field(&key, language::ENGLISH_PRIMARY), value.as_str());
-                }
-                writer.add(document).expect("Failed to add document to search index");
-                counter += 1;
-            }
-            writer.commit().expect("Failed to commit changes to search index");
-            debug!("Added {counter} documents to search index");
-        });*/
-
         let mut abbreviated_datatypes = ontology
             .datatypes()
             .into_iter()
-            .map(|node| ontology.abbreviate(node.as_ref()))
-            .collect::<Vec<_>>();
+            .map(|node| curie_helper.abbreviate(node.as_str())
+                .unwrap_or(node.as_str().to_string())
+            ).collect::<Vec<_>>();
         abbreviated_datatypes.sort();
 
         let (id, task) = window::open(Settings::default());
@@ -157,6 +145,7 @@ impl Publisher {
         (
             Self {
                 http_client,
+                curie_helper,
                 ontology,
                 abbreviated_datatypes,
                 window_id: id,
@@ -165,7 +154,7 @@ impl Publisher {
                 hovered_row: None,
                 search_state: None,
                 index,
-                index_writer: None,
+                traversal: None,
                 show_overwrite_confirmation: false,
                 modified: false,
                 show_new_document_buttons: false,
@@ -179,48 +168,80 @@ impl Publisher {
         trace!(?message);
 
         match message {
+            Message::RebuildIndex => {
+                self.index
+                    .writer()
+                    .expect("Unable to create writer")
+                    .remove_all()
+                    .expect("Unable to clear index");
+
+                let query = Ontology::index_query(&*language::ENGLISH_OR_UNTAGGED);
+                let query_results = self.ontology.execute_query(query).expect("Unable to generate index of entities");
+                let results = Ontology::transform_index_results(query_results);
+                task = Task::done(Message::IndexQueryResults(results));
+            }
             Message::Traverse => {
+                self.traversal = Some((self.ontology.to_dataset(), debug_span!("Repository Traversal").entered()));
+
                 let client = self.http_client.clone();
                 let root = Url::parse(&self.url_input).expect("Invalid URL");
                 let stream = Traverse::new(client, root, None);
 
-                self.index_writer = Some(self.index.writer().expect("Unable to create search index writer"));
-
                 task = Task::run(stream, |result| match result {
-                    Ok(rdf_source) => Message::IndexRdfSource(rdf_source),
+                    Ok(rdf_source) => Message::AddRdfSource(rdf_source),
                     Err(err) => Message::ShowError(format!("Unable to fetch RDF Source: {err}")),
-                })
-                .chain(Task::done(Message::CommitIndex));
+                }).chain(Task::done(Message::ConcludeTraversal));
             }
-            Message::IndexRdfSource(rdf_source) => {
-                for catalog_id in rdf_source.classes()
-                    .filter_map(|class| self.ontology.catalog_id(&class.into_owned())) {
-                    let mut document = SearchDocument::new();
-                    document.add_u64(Schema::type_field(), catalog_id);
-                    document.add_text(Schema::iri_field(), rdf_source.origin());
+            Message::AddRdfSource(rdf_source) => {
+                if let Some((traversal, _)) = &mut self.traversal {
+                    traversal.extend(rdf_source.dataset());
+                }
+            }
+            Message::ConcludeTraversal => {
+                if let Some((traversal, _)) = &self.traversal {
+                    let query = Ontology::index_query(&*language::ENGLISH_OR_UNTAGGED);
+                    let query_results = query.on_queryable_dataset(traversal)
+                        .execute()
+                        .expect("Unable to generate index of entities");
+                    let results = Ontology::transform_index_results(query_results);
+                    task = Task::done(Message::IndexQueryResults(results));
+                }
+                self.traversal = None;
+            }
+            Message::IndexQueryResults(results) => {
+                let mut writer = self.index.writer().expect("Failed to build index writer");
+                let curie_helper = self.curie_helper.clone();
+                let index_task = tokio::task::spawn_blocking(move || {
+                    debug_span!("Indexing").in_scope(|| {
+                        let mut counter = 0;
+                        for (individual, entry) in results {
+                            debug!(%individual, ?entry);
+                            let mut document = doc!(
+                                Schema::type_field() => entry.catalog_id,
+                                Schema::iri_field() => individual.as_str(),
+                            );
 
-                    /*for quad in rdf_source.dataset() {
-                        if let Some(field) =
-                            self.ontology.field_for_property(&quad.predicate.into_owned())
-                            && let TermRef::Literal(literal) = quad.object
-                        {
-                            let field = Schema::schema()
-                                .get_field(field.name.as_str())
-                                .expect("Field not found in schema");
-                            document.add_text(field, literal.value());
+                            if let Some(curie) = curie_helper.abbreviate(individual.as_str()) {
+                                document.add_text(Schema::curie_field(), curie);
+                            }
+
+                            for (key, value) in entry.fields {
+                                document.add_text(Schema::field(&key, language::ENGLISH_PRIMARY), value.as_str());
+                            }
+                            writer.add(document).expect("Failed to add document to search index");
+                            counter += 1;
                         }
-                    }*/
+                        writer.commit().expect("Failed to commit changes to search index");
+                        debug!("Added {counter} documents to search index");
+                    });
+                });
 
-                    if let Some(writer) = &self.index_writer {
-                        writer.add(document).expect("Unable to add document to search index");
+                task = Task::future(async {
+                    match index_task.await {
+                        Ok(()) => Message::None,
+                        Err(err) => Message::ShowError(err.to_string()),
                     }
-                }
-            }
-            Message::CommitIndex => {
-                if let Some(writer) = &mut self.index_writer {
-                    writer.commit().expect("Unable to commit updates to search index");
-                    self.index_writer = None;
-                }
+                });
             }
             Message::WindowClosed(id) => {
                 if self.window_id == id {
@@ -297,7 +318,9 @@ impl Publisher {
                 let term = TermHelper::new(&quad.object);
                 let datatype = term
                     .datatype()
-                    .map(|datatype| self.ontology.abbreviate(datatype));
+                    .map(|datatype| self.curie_helper.abbreviate(datatype.as_str())
+                        .unwrap_or(datatype.as_str().to_string()));
+
                 let datatype_state = combo_box::State::with_selection(
                     self.abbreviated_datatypes.clone(),
                     datatype.as_ref(),
@@ -417,12 +440,13 @@ impl Publisher {
             }
             Message::DatatypeUpdated(key, Some(maybe_prefixed_iri)) => {
                 let node = self
-                    .ontology
+                    .curie_helper
                     .expand(&maybe_prefixed_iri)
-                    .unwrap_or(NamedNode::new_unchecked(&maybe_prefixed_iri));
+                    .unwrap_or(maybe_prefixed_iri);
+
                 if let Some(quad) = self.document.dataset_mut().quads.get_mut(key) {
                     let mut term = TermHelperMut::new(&mut quad.object);
-                    term.set_datatype(Some(node));
+                    term.set_datatype(Some(NamedNode::new_unchecked(node)));
                     self.modified = true;
                 }
             }
@@ -535,14 +559,14 @@ impl Publisher {
             Message::NavigateToPredicate(key) => {
                 if let Some(quad) = self.document.dataset_mut().quads.get_mut(key) {
                     self.url_input = quad.predicate.as_str().to_string();
-                    task = Task::done(URLInputSubmitted);
+                    task = Task::done(Message::URLInputSubmitted);
                 }
             }
             Message::NavigateToObject(key) => {
                 if let Some(quad) = self.document.dataset_mut().quads.get_mut(key) {
                     if let Term::NamedNode(node) = &quad.object {
                         self.url_input = node.as_str().to_string();
-                        task = Task::done(URLInputSubmitted);
+                        task = Task::done(Message::URLInputSubmitted);
                     }
                 }
             }
@@ -578,7 +602,7 @@ impl Publisher {
             container(space()).width(BUTTON_WIDTH)
         };
 
-        let predicate_input_base = iri_input(self.ontology.prefixes(), "Predicate", triple.predicate.as_str());
+        let predicate_input_base = iri_input(&self.curie_helper, "Predicate", triple.predicate.as_str());
         let predicate_input = if self.ontology.is_read_only(triple.as_ref()) {
             predicate_input_base
         } else {
@@ -587,6 +611,9 @@ impl Publisher {
                 .on_shift_click(Message::NavigateToPredicate(key))
         };
 
+        let predicate_info = self.ontology.info(&triple.predicate, &*language::ENGLISH_OR_UNTAGGED);
+        let predicate_label = container(text(predicate_info.label));
+
         let term = TermHelper::new(&triple.object);
 
         let value_label = term.value_as_named_node().and_then(|node| {
@@ -607,7 +634,7 @@ impl Publisher {
             })
             .unwrap_or(Horizontal::Left);
 
-        let object_input_base = iri_input(self.ontology.prefixes(), "Object", value.as_str());
+        let object_input_base = iri_input(&self.curie_helper, "Object", value.as_str());
         let object_input = if self.ontology.is_read_only(triple.as_ref()) {
             object_input_base
         } else {
@@ -617,7 +644,9 @@ impl Publisher {
                 .on_shift_click(Message::NavigateToObject(key))
         };
 
-        let selected_datatype = term.datatype().map(|node| self.ontology.abbreviate(node));
+        let selected_datatype = term.datatype()
+            .and_then(|node| self.curie_helper.abbreviate(node.as_str()));
+
         let datatype_selector: Element<Message> = if state.read_only {
             selected_datatype.map(text).into()
         } else {
@@ -667,8 +696,9 @@ impl Publisher {
         let row = row![
             button_area,
             predicate_input,
-            value_label,
+            predicate_label,
             object_input,
+            value_label,
             datatype_selector,
             language_input,
             direction_slider,
@@ -685,7 +715,10 @@ impl Publisher {
         entities: impl IntoIterator<Item = LabeledIri>,
     ) -> Element<'_, Message> {
         let buttons = entities.into_iter().map(|entity| {
-            let label = format!("{} ({})", entity.label, self.ontology.abbreviate(entity.iri.as_ref()));
+            let abbreviation = self.curie_helper.abbreviate(entity.iri.as_str())
+                .unwrap_or_else(|| entity.iri.as_str().to_string());
+
+            let label = format!("{} ({})", entity.label, abbreviation);
             button(text(label))
                 .on_press(Message::NewDocument(entity.iri))
                 .into()
@@ -717,7 +750,9 @@ impl Publisher {
                         .and_then(|value| value.as_str())
                         .unwrap_or_default();
 
-                    let abbreviated_iri = self.ontology.abbreviate(NamedNodeRef::new_unchecked(iri));
+                    let abbreviated_iri = self.curie_helper.abbreviate(iri)
+                        .unwrap_or_else(|| iri.to_string());
+
                     button(text(abbreviated_iri).wrapping(Wrapping::Word))
                         .on_press(Message::SearchResultClicked(NamedNode::new_unchecked(iri)))
                         .style(button::text)
@@ -764,9 +799,11 @@ impl Publisher {
 
         let add_row_button = button("Add row").on_press(Message::AddRow(None));
 
-        let index_button = button("Index").on_press(Message::Traverse);
+        let reindex_ontology_button = button("Rebuild index").on_press(Message::RebuildIndex);
 
-        let address_input = iri_input(self.ontology.prefixes(), "URL", &self.url_input)
+        let traverse_button = button("Traverse").on_press(Message::Traverse);
+
+        let address_input = iri_input(&self.curie_helper, "URL", &self.url_input)
             .on_input(Message::URLInputChanged)
             .on_submit(Message::URLInputSubmitted)
             .on_control_click(Message::OpenQueryWindow(SearchResultClickAction::URLInput));
@@ -797,7 +834,8 @@ impl Publisher {
         let content = column![
             row![
                 add_row_button,
-                index_button,
+                reindex_ontology_button,
+                traverse_button,
                 address_input,
                 save_button,
             ],