gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
Alex Wied
2026-07-20 23:53:33 -04:00
parent 57af8d592f
commit 21016af858
12 changed files with 456 additions and 372 deletions
Download Patch File Download Diff File Expand all files Collapse all files
publish/Cargo.toml
+1
View File
@@ -20,4 +20,5 @@ thiserror.workspace = true
tokio.workspace = true
tracing.workspace = true
tracing-appender.workspace = true
tracing-futures.workspace = true
tracing-subscriber.workspace = true
publish/src/app.rs
+59 -47
View File
@@ -20,7 +20,8 @@ use oxigraph::io::RdfFormat;
use oxigraph::model::vocab::{rdf, rdfs};
use oxigraph::model::{BaseDirection, Dataset, NamedNode, Quad, Term};
use tracing::{debug, debug_span, error, trace};
use tracing::span::EnteredSpan;
use tracing::span::Span;
use tracing_futures::Instrument;
use crate::navigator::Navigator;
use crate::rdf::curie::CurieHelper;
use crate::rdf::language;
@@ -36,6 +37,7 @@ pub(crate) enum Message {
AddRdfSource(RdfSource<Dataset>),
ConcludeTraversal,
IndexQueryResults(HashMap<NamedNode, IndexEntry>),
CommitIndex,
WindowClosed(window::Id),
URLInputChanged(String),
URLInputSubmitted,
@@ -105,7 +107,7 @@ pub(crate) struct Publisher {
hovered_row: Option<QuadKey>,
search_state: Option<SearchState>,
index: SearchIndex,
traversal: Option<(Dataset, EnteredSpan)>,
traversal: Option<Dataset>,
show_overwrite_confirmation: bool,
modified: bool,
show_new_document_buttons: bool,
@@ -177,20 +179,24 @@ impl Publisher {
match message {
Message::RebuildIndex => {
self.index
.writer()
.expect("Unable to create writer")
.remove_all()
.expect("Unable to clear index");
if let Ok(writer) = self.index.writer() {
writer.remove_all().expect("Unable to clear index");
let query = Ontology::index_query(&*language::ENGLISH_OR_UNTAGGED);
let query_results = self.ontology.execute_query(query).expect("Unable to generate index of entities");
let results = Ontology::transform_index_results(query_results);
task = Task::done(Message::IndexQueryResults(results));
let index_future = self.ontology
.index(&*language::ENGLISH_OR_UNTAGGED, None);
let index_task = Task::perform(index_future, |task_result| {
match task_result {
Ok(results) => Message::IndexQueryResults(results),
Err(err) => Message::ShowError(err.to_string()),
}
});
task = Task::batch([
index_task,
Task::done(Message::Traverse),
]);
}
}
Message::Traverse => {
self.traversal = Some((self.ontology.to_dataset(), debug_span!("Repository Traversal").entered()));
let client = self.http_client.clone();
let root = Url::parse(&self.url_input).expect("Invalid URL");
let stream = Traverse::new(client, root, None);
@@ -199,33 +205,36 @@ impl Publisher {
Ok(rdf_source) => Message::AddRdfSource(rdf_source),
Err(err) => Message::ShowError(format!("Unable to fetch RDF Source: {err}")),
}).chain(Task::done(Message::ConcludeTraversal));
self.traversal = Some(self.ontology.to_dataset());
}
Message::AddRdfSource(rdf_source) => {
if let Some((traversal, _)) = &mut self.traversal {
if let Some(traversal) = &mut self.traversal {
traversal.extend(rdf_source.dataset());
}
}
Message::ConcludeTraversal => {
if let Some((traversal, _)) = &self.traversal {
let query = Ontology::index_query(&*language::ENGLISH_OR_UNTAGGED);
let query_results = query.on_queryable_dataset(traversal)
.execute()
.expect("Unable to generate index of entities");
let results = Ontology::transform_index_results(query_results);
task = Task::done(Message::IndexQueryResults(results));
if let Some(traversal) = self.traversal.take() {
let index_task = self.ontology
.index(&*language::ENGLISH_OR_UNTAGGED, Some(traversal));
task = Task::perform(index_task, |task_result| {
match task_result {
Ok(results) => Message::IndexQueryResults(results),
Err(err) => Message::ShowError(err.to_string()),
}
});
}
self.traversal = None;
}
Message::IndexQueryResults(results) => {
let mut writer = self.index.writer().expect("Failed to build index writer");
let mut index = self.index.clone();
let curie_helper = self.curie_helper.clone();
let index_task = tokio::task::spawn_blocking(move || {
debug_span!("Indexing").in_scope(|| {
task = Task::perform(tokio::task::spawn_blocking(move || {
let _span = debug_span!("Indexing").entered();
if let Ok(writer) = index.writer() {
let mut counter = 0;
for (individual, entry) in results {
debug!(%individual, ?entry);
let mut document = doc!(
Schema::type_field() => entry.catalog_id,
Schema::type_field() => entry.category_id,
Schema::iri_field() => individual.as_str(),
);
@@ -239,18 +248,19 @@ impl Publisher {
writer.add(document).expect("Failed to add document to search index");
counter += 1;
}
writer.commit().expect("Failed to commit changes to search index");
debug!("Added {counter} documents to search index");
});
});
task = Task::future(async {
match index_task.await {
Ok(()) => Message::None,
Err(err) => Message::ShowError(err.to_string()),
}
}), |result| if let Err(err) = result {
Message::ShowError(err.to_string())
} else {
Message::CommitIndex
});
}
Message::CommitIndex => {
if let Err(err) = self.index.commit() {
task = Task::done(Message::ShowError(err.to_string()));
}
}
Message::WindowClosed(id) => {
if self.window_id == id {
task = iced::exit();
@@ -263,7 +273,9 @@ impl Publisher {
}
Message::URLInputSubmitted => {
let url = Url::parse(&self.url_input).expect("Invalid URL");
self.navigator.goto(url.clone());
if &url != self.navigator.current() {
self.navigator.goto(url.clone());
}
task = Task::done(Message::NavigateTo(url));
}
Message::NavigateTo(url) => {
@@ -397,24 +409,27 @@ impl Publisher {
}
Message::QueryUpdated(new_query) => {
if let Some(search_state) = &mut self.search_state {
let catalog_id = search_state.entity_selection
let category_id = search_state.entity_selection
.clone()
.and_then(|info| self.ontology.catalog_id(&info.iri));
.and_then(|info| self.ontology.category_id(&info.iri));
let index = self.index.clone();
let query = new_query.clone();
let search_task = tokio::task::spawn_blocking(move || {
index.query(catalog_id, query.as_str(), Schema::all_fields(), 25)
});
task = Task::future(async {
/*let search_task = tokio::task::spawn_blocking(move || {
index.query(category_id, query.as_str(), Schema::all_fields(), 25)
});*/
/*task = Task::future(async {
match search_task.await {
Ok(Ok(results)) => Message::SetSearchResults(results),
Ok(Err(err)) => Message::ShowError(err.to_string()),
Err(err) => Message::ShowError(err.to_string()),
}
});
});*/
search_state.query = new_query;
task = Task::done(Message::SetSearchResults(
self.index.query(category_id, query.as_str(), Schema::all_fields(), 25)
.expect("Unable to complete search")
));
};
}
Message::SetSearchResults(results) => {
@@ -829,8 +844,6 @@ impl Publisher {
let reindex_ontology_button = button("Rebuild index").on_press(Message::RebuildIndex);
let traverse_button = button("Traverse").on_press(Message::Traverse);
let address_input = iri_input(&self.curie_helper, "URL", &self.url_input)
.on_input(Message::URLInputChanged)
.on_submit(Message::URLInputSubmitted)
@@ -865,7 +878,6 @@ impl Publisher {
forward_button,
add_row_button,
reindex_ontology_button,
traverse_button,
address_input,
save_button,
],
publish/src/error.rs
+3
View File
@@ -30,4 +30,7 @@ pub(crate) enum Error {
#[error(transparent)]
SparqlSyntax(#[from] oxigraph::sparql::SparqlSyntaxError),
#[error(transparent)]
Join(#[from] tokio::task::JoinError),
}
publish/src/rdf/ontology.rs
+36 -30
View File
@@ -2,11 +2,12 @@ use crate::error;
use crate::rdf::vocab::gl;
use oxigraph::model::vocab::{rdf, rdfs, xsd};
use oxigraph::model::{Dataset, Graph, GraphNameRef, LiteralRef, NamedNode, NamedNodeRef, NamedOrBlankNode, NamedOrBlankNodeRef, Term, TermRef, Triple, TripleRef};
use oxigraph::sparql::{PreparedSparqlQuery, QueryResults, SparqlEvaluator};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fmt::Display;
use std::path::{Path, PathBuf};
use std::sync::LazyLock;
use iced::futures::TryFutureExt;
use oxigraph::store::Store;
use tracing::debug_span;
use crate::rdf::{conversion, materialize};
@@ -46,7 +47,7 @@ static PREFIXES: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
("rdax", "http://rdaregistry.info/Elements/x/"),
("schema", "https://schema.org/"),
("quill", "http://fedora.quill.lan/rest/"),
("gl", "ONTOLOGY_PREFIX"),
("gl", ONTOLOGY_PREFIX),
].map(|(k, v)| (k.to_string(), v.to_string())))
});
@@ -108,7 +109,7 @@ impl OntologyBuilder {
#[derive(Clone, Debug)]
pub struct IndexEntry {
pub catalog_id: u64,
pub category_id: u64,
pub fields: HashMap<String, String>,
}
@@ -172,12 +173,12 @@ impl Ontology {
.collect::<Dataset>()
}
pub fn index_query(language: &LanguageCondition) -> PreparedSparqlQuery {
pub fn index(&self, language: &LanguageCondition, source: Option<Dataset>) -> impl Future<Output = error::Result<HashMap<NamedNode, IndexEntry>>> + 'static {
let language_filter = language.to_filter_expression("fieldValue");
let query = format!(r#"SELECT ?individual ?catalogId ?fieldName ?fieldValue
let query = format!(r#"SELECT ?individual ?categoryId ?fieldName ?fieldValue
WHERE {{
?class a gl:SearchableClass ;
gl:catalogId ?catalogId ;
gl:categoryId ?categoryId ;
gl:associatedProperty ?property .
?property gl:indexedByField/gl:fieldName ?fieldName .
@@ -190,39 +191,44 @@ WHERE {{
let mut sparql = SparqlEvaluator::new()
.with_prefix("gl", ONTOLOGY_PREFIX).unwrap()
.parse_query(&query)
.unwrap();
.expect("Unable to parse query");
sparql.dataset_mut().set_default_graph_as_union();
sparql
}
pub fn execute_query(&self, query: PreparedSparqlQuery) -> error::Result<QueryResults<'_>> {
Ok(query.on_store(&self.store).execute()?)
}
let store = self.store.clone();
tokio::task::spawn_blocking(move || {
let _span = debug_span!("Index Query").entered();
let query_results = if let Some(source) = &source {
sparql.on_queryable_dataset(source).execute()
} else {
sparql.on_store(&store).execute()
}.expect("Unable to execute indexing query");
pub fn transform_index_results(query_results: QueryResults) -> HashMap<NamedNode, IndexEntry> {
let mut results = HashMap::new();
if let QueryResults::Solutions(solutions) = query_results {
for solution in solutions.filter_map(Result::ok) {
let individual = solution.get("individual").and_then(conversion::term_to_named_node);
let catalog_id = solution.get("catalogId").and_then(conversion::term_to_u64);
let field_name = solution.get("fieldName").and_then(conversion::term_as_str);
let field_value = solution.get("fieldValue").and_then(conversion::term_as_str);
if let (Some(individual), Some(catalog_id), Some(field_name), Some(field_value)) = (individual, catalog_id, field_name, field_value) {
results.entry(individual.to_owned())
.and_modify(|entry: &mut IndexEntry| {
entry.fields.insert(field_name.to_owned(), field_value.to_owned());
}).or_insert(IndexEntry {
catalog_id,
let mut results = HashMap::new();
if let QueryResults::Solutions(solutions) = query_results {
for solution in solutions.filter_map(Result::ok) {
let individual = solution.get("individual").and_then(conversion::term_to_named_node);
let catalog_id = solution.get("categoryId").and_then(conversion::term_to_u64);
let field_name = solution.get("fieldName").and_then(conversion::term_as_str);
let field_value = solution.get("fieldValue").and_then(conversion::term_as_str);
if let (Some(individual), Some(catalog_id), Some(field_name), Some(field_value)) = (individual, catalog_id, field_name, field_value) {
results.entry(individual.to_owned())
.and_modify(|entry: &mut IndexEntry| {
entry.fields.insert(field_name.to_owned(), field_value.to_owned());
}).or_insert(IndexEntry {
category_id: catalog_id,
fields: HashMap::from_iter([(field_name.to_owned(), field_value.to_owned())]),
});
}
}
} else {
unreachable!()
}
}
results
results
}).map_err(error::Error::from)
}
pub fn catalog_id(&self, class: &NamedNode) -> Option<u64> {
self.store.quads_for_pattern(Some(class.into()), Some(gl::CATALOG_ID), None, None)
pub fn category_id(&self, class: &NamedNode) -> Option<u64> {
self.store.quads_for_pattern(Some(class.into()), Some(gl::CATEGORY_ID), None, None)
.filter_map(Result::ok)
.map(conversion::quad_into_term)
.filter_map(conversion::term_into_u64)
publish/src/rdf/vocab.rs
+2 -2
View File
@@ -7,8 +7,8 @@ pub mod gl {
pub const INDEXED_BY_FIELD: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/indexedByField");
pub const CATALOG_ID: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/catalogId");
pub const CATEGORY_ID: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/categoryId");
pub const SEARCHABLE_CLASS: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/SearchableClass");
publish/src/widget/iri_input.rs
-1
View File
@@ -6,7 +6,6 @@ use iced::advanced::{Layout, Widget};
use iced::advanced::widget::{tree, Tree};
use iced::mouse::{Cursor, Interaction};
use iced::widget::text_input::Catalog;
use tracing::debug;
use crate::rdf::curie::CurieHelper;
pub struct State {