gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
alex
2026-07-22 20:34:15 -04:00
parent 21016af858
commit 0ddf2caf38
6 changed files with 61 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
publish/src/app.rs
+46 -47
View File
@@ -19,8 +19,7 @@ use ldp::{RdfSource, RdfSourceUpdateResponse, ResourceRequestBuilder, Serializat
use oxigraph::io::RdfFormat;
use oxigraph::model::vocab::{rdf, rdfs};
use oxigraph::model::{BaseDirection, Dataset, NamedNode, Quad, Term};
use tracing::{debug, debug_span, error, trace};
use tracing::span::Span;
use tracing::{debug_span, error, field, trace};
use tracing_futures::Instrument;
use crate::navigator::Navigator;
use crate::rdf::curie::CurieHelper;
@@ -37,7 +36,6 @@ pub(crate) enum Message {
AddRdfSource(RdfSource<Dataset>),
ConcludeTraversal,
IndexQueryResults(HashMap<NamedNode, IndexEntry>),
CommitIndex,
WindowClosed(window::Id),
URLInputChanged(String),
URLInputSubmitted,
@@ -179,22 +177,26 @@ impl Publisher {
match message {
Message::RebuildIndex => {
if let Ok(writer) = self.index.writer() {
writer.remove_all().expect("Unable to clear index");
let mut writer = self.index.writer().expect("Unable to obtain index writer");
let clear_index_task = Task::perform(tokio::task::spawn_blocking(move || {
let _span = debug_span!("Clear Index").entered();
writer.delete_all_documents().expect("Unable to clear index");
writer.commit().expect("Unable to commit index operation");
}), |_| Message::None);
let index_future = self.ontology
.index(&*language::ENGLISH_OR_UNTAGGED, None);
let index_task = Task::perform(index_future, |task_result| {
match task_result {
Ok(results) => Message::IndexQueryResults(results),
Err(err) => Message::ShowError(err.to_string()),
}
});
task = Task::batch([
index_task,
Task::done(Message::Traverse),
]);
}
let index_future = self.ontology
.query_for_indexable_triples(&*language::ENGLISH_OR_UNTAGGED, None);
let index_task = Task::perform(index_future, |task_result| {
match task_result {
Ok(results) => Message::IndexQueryResults(results),
Err(err) => Message::ShowError(err.to_string()),
}
});
task = clear_index_task.chain(Task::batch([
index_task,
Task::done(Message::Traverse),
]));
}
Message::Traverse => {
let client = self.http_client.clone();
@@ -216,7 +218,7 @@ impl Publisher {
Message::ConcludeTraversal => {
if let Some(traversal) = self.traversal.take() {
let index_task = self.ontology
.index(&*language::ENGLISH_OR_UNTAGGED, Some(traversal));
.query_for_indexable_triples(&*language::ENGLISH_OR_UNTAGGED, Some(traversal));
task = Task::perform(index_task, |task_result| {
match task_result {
Ok(results) => Message::IndexQueryResults(results),
@@ -226,41 +228,38 @@ impl Publisher {
}
}
Message::IndexQueryResults(results) => {
let mut index = self.index.clone();
let mut writer = self.index.writer().expect("Unable to obtain index writer");
let curie_helper = self.curie_helper.clone();
task = Task::perform(tokio::task::spawn_blocking(move || {
let _span = debug_span!("Indexing").entered();
if let Ok(writer) = index.writer() {
let mut counter = 0;
for (individual, entry) in results {
let mut document = doc!(
Schema::type_field() => entry.category_id,
Schema::iri_field() => individual.as_str(),
);
let span = debug_span!("Index Query Results", documents = field::Empty).entered();
let mut counter = 0usize;
for (individual, entry) in results {
let mut document = doc!(
Schema::type_field() => entry.category_id,
Schema::iri_field() => individual.as_str(),
);
if let Some(curie) = curie_helper.abbreviate(individual.as_str()) {
document.add_text(Schema::curie_field(), curie);
}
for (key, value) in entry.fields {
document.add_text(Schema::field(&key, language::ENGLISH_PRIMARY), value.as_str());
}
writer.add(document).expect("Failed to add document to search index");
counter += 1;
if let Some(curie) = curie_helper.abbreviate(individual.as_str()) {
document.add_text(Schema::curie_field(), curie);
}
debug!("Added {counter} documents to search index");
for (key, value) in entry.fields {
document.add_text(Schema::field(&key, language::ENGLISH_PRIMARY), value.as_str());
}
writer.add_document(document).expect("Failed to add document to search index");
counter += 1;
}
span.record("documents", counter);
writer.commit()
}), |result| {
if let Err(err) = result {
Message::ShowError(err.to_string())
} else {
Message::None
}
}), |result| if let Err(err) = result {
Message::ShowError(err.to_string())
} else {
Message::CommitIndex
});
}
Message::CommitIndex => {
if let Err(err) = self.index.commit() {
task = Task::done(Message::ShowError(err.to_string()));
}
}
Message::WindowClosed(id) => {
if self.window_id == id {
task = iced::exit();
publish/src/rdf/materialize.rs
+3 -6
View File
@@ -9,8 +9,7 @@ const INFERENCE_GRAPH: NamedNodeRef = NamedNodeRef::new_unchecked("https://graph
const RDF_SCHEMA: NamedNodeRef = NamedNodeRef::new_unchecked("http://www.w3.org/2000/01/rdf-schema#");
pub fn same_as(store: &mut Store) -> error::Result<()> {
let span = debug_span!("Materialize owl:sameAs");
let _enter = span.enter();
let _span = debug_span!("Materialize owl:sameAs").entered();
let additional_quads = store
.quads_for_pattern(None, Some(owl::SAME_AS), None, None)
@@ -58,8 +57,7 @@ pub fn same_as(store: &mut Store) -> error::Result<()> {
}
pub fn super_properties(store: &mut Store) -> error::Result<()> {
let span = debug_span!("Materialize rdfs:subPropertyOf");
let _enter = span.enter();
let _span = debug_span!("Materialize rdfs:subPropertyOf").entered();
let update = SparqlEvaluator::new()
.with_prefix("rdfs", RDF_SCHEMA.as_str())?
@@ -88,8 +86,7 @@ pub fn super_properties(store: &mut Store) -> error::Result<()> {
}
pub fn super_classes(store: &mut Store) -> error::Result<()> {
let span = debug_span!("Materialize rdfs:subClassOf");
let _enter = span.enter();
let _span = debug_span!("Materialize rdfs:subClassOf").entered();
let update = SparqlEvaluator::new()
.with_prefix("rdfs", RDF_SCHEMA.as_str())?
publish/src/rdf/ontology.rs
+6 -3
View File
@@ -9,7 +9,7 @@ use std::path::{Path, PathBuf};
use std::sync::LazyLock;
use iced::futures::TryFutureExt;
use oxigraph::store::Store;
use tracing::debug_span;
use tracing::{debug_span, field};
use crate::rdf::{conversion, materialize};
use crate::rdf::language::LanguageCondition;
@@ -173,7 +173,7 @@ impl Ontology {
.collect::<Dataset>()
}
pub fn index(&self, language: &LanguageCondition, source: Option<Dataset>) -> impl Future<Output = error::Result<HashMap<NamedNode, IndexEntry>>> + 'static {
pub fn query_for_indexable_triples(&self, language: &LanguageCondition, source: Option<Dataset>) -> impl Future<Output = error::Result<HashMap<NamedNode, IndexEntry>>> + 'static {
let language_filter = language.to_filter_expression("fieldValue");
let query = format!(r#"SELECT ?individual ?categoryId ?fieldName ?fieldValue
WHERE {{
@@ -196,7 +196,7 @@ WHERE {{
let store = self.store.clone();
tokio::task::spawn_blocking(move || {
let _span = debug_span!("Index Query").entered();
let span = debug_span!("Indexable Triples Query", solutions = field::Empty).entered();
let query_results = if let Some(source) = &source {
sparql.on_queryable_dataset(source).execute()
} else {
@@ -205,6 +205,7 @@ WHERE {{
let mut results = HashMap::new();
if let QueryResults::Solutions(solutions) = query_results {
let mut counter = 0usize;
for solution in solutions.filter_map(Result::ok) {
let individual = solution.get("individual").and_then(conversion::term_to_named_node);
let catalog_id = solution.get("categoryId").and_then(conversion::term_to_u64);
@@ -219,7 +220,9 @@ WHERE {{
fields: HashMap::from_iter([(field_name.to_owned(), field_value.to_owned())]),
});
}
counter += 1;
}
span.record("solutions", counter);
} else {
unreachable!()
}