gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
Alex Wied
2026-07-20 23:53:33 -04:00
parent 57af8d592f
commit 21016af858
12 changed files with 456 additions and 372 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+280 -247
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+3 -1
View File
@@ -19,12 +19,14 @@ iced = { git = "https://github.com/iced-rs/iced.git", branch = "master", feature
oxigraph = { version = "0.5", features = ["rdf-12"] }
oxilangtag = "0.1"
parse_link_header = "0.4"
pin-project-lite = "0.2"
rfd = "0.17"
slotmap = "1.1"
tantivy = "0.26"
tar = "0.4"
thiserror = "2.0"
tokio = { version = "1.52", features = ["rt", "rt-multi-thread", "macros", "fs"] }
tokio = { version = "1.53", features = ["rt", "rt-multi-thread", "macros", "fs"] }
tracing = "0.1"
tracing-appender = "0.2"
tracing-futures = "0.2"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
ontology/graphofliberty.ttl
+47 -36
View File
@@ -17,6 +17,9 @@
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
@prefix fedora: <http://fedora.info/definitions/v4/repository#> .
@prefix premis: <http://www.loc.gov/premis/rdf/v1#> .
@prefix ebucore: <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#> .
@prefix premis3: <http://www.loc.gov/premis/rdf/v3/> .
@base <https://graphofliberty.org/2026/04/ont/> .
<https://graphofliberty.org/2026/04/ont> rdf:type owl:Ontology .
@@ -84,10 +87,6 @@ xsd:unsignedShort rdf:type rdfs:Datatype ;
# Object Properties
#################################################################
### http://fedora.info/definitions/v4/repository#hasParent
fedora:hasParent rdf:type owl:ObjectProperty .
### https://graphofliberty.org/2026/04/ont/associatedProperty
:associatedProperty rdf:type owl:ObjectProperty ;
rdfs:label "associated property"@en .
@@ -110,35 +109,12 @@ fedora:hasParent rdf:type owl:ObjectProperty .
# Data properties
#################################################################
### http://fedora.info/definitions/v4/repository#created
fedora:created rdf:type owl:DatatypeProperty ;
rdfs:subPropertyOf owl:topDataProperty .
### http://fedora.info/definitions/v4/repository#createdBy
fedora:createdBy rdf:type owl:DatatypeProperty ;
rdfs:subPropertyOf owl:topDataProperty .
### http://fedora.info/definitions/v4/repository#lastModified
fedora:lastModified rdf:type owl:DatatypeProperty .
### http://fedora.info/definitions/v4/repository#lastModifiedBy
fedora:lastModifiedBy rdf:type owl:DatatypeProperty .
### http://www.w3.org/ns/ldp#contains
ldp:contains rdf:type owl:DatatypeProperty ;
rdfs:subPropertyOf owl:topDataProperty .
### https://graphofliberty.org/2026/04/ont/catalogId
:catalogId rdf:type owl:DatatypeProperty ;
rdfs:domain :SearchableClass ;
rdfs:range xsd:nonNegativeInteger ;
rdfs:comment "An integer associated with the class for fast lookup in a database."@en ;
rdfs:label "catalog id" .
### https://graphofliberty.org/2026/04/ont/categoryId
:categoryId rdf:type owl:DatatypeProperty ;
rdfs:domain :SearchableClass ;
rdfs:range xsd:nonNegativeInteger ;
rdfs:comment "An integer associated with the class for fast lookup in a database."@en ;
rdfs:label "category id" .
### https://graphofliberty.org/2026/04/ont/fieldLabel
@@ -191,6 +167,11 @@ ldp:contains rdf:type owl:DatatypeProperty ;
# Individuals
#################################################################
### http://fedora.info/definitions/v4/repository#Binary
fedora:Binary rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
### http://fedora.info/definitions/v4/repository#Container
fedora:Container rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
@@ -211,6 +192,11 @@ fedora:createdBy rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
### http://fedora.info/definitions/v4/repository#hasFixityService
fedora:hasFixityService rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
### http://fedora.info/definitions/v4/repository#hasParent
fedora:hasParent rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
@@ -241,7 +227,27 @@ rdac:C10004 rdf:type owl:NamedIndividual ,
:SearchableClass ;
:associatedProperty <http://rdaregistry.info/Elements/a/datatype/P50291> ,
<http://rdaregistry.info/Elements/a/datatype/P50292> ;
:catalogId "2"^^xsd:nonNegativeInteger .
:categoryId "2"^^xsd:nonNegativeInteger .
### http://www.loc.gov/premis/rdf/v1#hasMessageDigest
premis:hasMessageDigest rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
### http://www.loc.gov/premis/rdf/v1#hasSize
premis:hasSize rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
### http://www.loc.gov/premis/rdf/v3/File
premis3:File rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
### http://www.loc.gov/premis/rdf/v3/size
premis3:size rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
### http://www.w3.org/1999/02/22-rdf-syntax-ns#Property
@@ -250,7 +256,7 @@ rdf:Property rdf:type owl:NamedIndividual ,
:associatedProperty rdfs:comment ,
rdfs:label ,
skos:definition ;
:catalogId "0"^^xsd:nonNegativeInteger .
:categoryId "0"^^xsd:nonNegativeInteger .
### http://www.w3.org/2000/01/rdf-schema#Class
@@ -259,7 +265,7 @@ rdfs:Class rdf:type owl:NamedIndividual ,
:associatedProperty rdfs:comment ,
rdfs:label ,
skos:definition ;
:catalogId "1"^^xsd:nonNegativeInteger .
:categoryId "1"^^xsd:nonNegativeInteger .
### http://www.w3.org/2000/01/rdf-schema#comment
@@ -287,6 +293,11 @@ ldp:Container rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
### http://www.w3.org/ns/ldp#NonRDFSource
ldp:NonRDFSource rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
### http://www.w3.org/ns/ldp#RDFSource
ldp:RDFSource rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
publish/Cargo.toml
+1
View File
@@ -20,4 +20,5 @@ thiserror.workspace = true
tokio.workspace = true
tracing.workspace = true
tracing-appender.workspace = true
tracing-futures.workspace = true
tracing-subscriber.workspace = true
publish/src/app.rs
+59 -47
View File
@@ -20,7 +20,8 @@ use oxigraph::io::RdfFormat;
use oxigraph::model::vocab::{rdf, rdfs};
use oxigraph::model::{BaseDirection, Dataset, NamedNode, Quad, Term};
use tracing::{debug, debug_span, error, trace};
use tracing::span::EnteredSpan;
use tracing::span::Span;
use tracing_futures::Instrument;
use crate::navigator::Navigator;
use crate::rdf::curie::CurieHelper;
use crate::rdf::language;
@@ -36,6 +37,7 @@ pub(crate) enum Message {
AddRdfSource(RdfSource<Dataset>),
ConcludeTraversal,
IndexQueryResults(HashMap<NamedNode, IndexEntry>),
CommitIndex,
WindowClosed(window::Id),
URLInputChanged(String),
URLInputSubmitted,
@@ -105,7 +107,7 @@ pub(crate) struct Publisher {
hovered_row: Option<QuadKey>,
search_state: Option<SearchState>,
index: SearchIndex,
traversal: Option<(Dataset, EnteredSpan)>,
traversal: Option<Dataset>,
show_overwrite_confirmation: bool,
modified: bool,
show_new_document_buttons: bool,
@@ -177,20 +179,24 @@ impl Publisher {
match message {
Message::RebuildIndex => {
self.index
.writer()
.expect("Unable to create writer")
.remove_all()
.expect("Unable to clear index");
if let Ok(writer) = self.index.writer() {
writer.remove_all().expect("Unable to clear index");
let query = Ontology::index_query(&*language::ENGLISH_OR_UNTAGGED);
let query_results = self.ontology.execute_query(query).expect("Unable to generate index of entities");
let results = Ontology::transform_index_results(query_results);
task = Task::done(Message::IndexQueryResults(results));
let index_future = self.ontology
.index(&*language::ENGLISH_OR_UNTAGGED, None);
let index_task = Task::perform(index_future, |task_result| {
match task_result {
Ok(results) => Message::IndexQueryResults(results),
Err(err) => Message::ShowError(err.to_string()),
}
});
task = Task::batch([
index_task,
Task::done(Message::Traverse),
]);
}
}
Message::Traverse => {
self.traversal = Some((self.ontology.to_dataset(), debug_span!("Repository Traversal").entered()));
let client = self.http_client.clone();
let root = Url::parse(&self.url_input).expect("Invalid URL");
let stream = Traverse::new(client, root, None);
@@ -199,33 +205,36 @@ impl Publisher {
Ok(rdf_source) => Message::AddRdfSource(rdf_source),
Err(err) => Message::ShowError(format!("Unable to fetch RDF Source: {err}")),
}).chain(Task::done(Message::ConcludeTraversal));
self.traversal = Some(self.ontology.to_dataset());
}
Message::AddRdfSource(rdf_source) => {
if let Some((traversal, _)) = &mut self.traversal {
if let Some(traversal) = &mut self.traversal {
traversal.extend(rdf_source.dataset());
}
}
Message::ConcludeTraversal => {
if let Some((traversal, _)) = &self.traversal {
let query = Ontology::index_query(&*language::ENGLISH_OR_UNTAGGED);
let query_results = query.on_queryable_dataset(traversal)
.execute()
.expect("Unable to generate index of entities");
let results = Ontology::transform_index_results(query_results);
task = Task::done(Message::IndexQueryResults(results));
if let Some(traversal) = self.traversal.take() {
let index_task = self.ontology
.index(&*language::ENGLISH_OR_UNTAGGED, Some(traversal));
task = Task::perform(index_task, |task_result| {
match task_result {
Ok(results) => Message::IndexQueryResults(results),
Err(err) => Message::ShowError(err.to_string()),
}
});
}
self.traversal = None;
}
Message::IndexQueryResults(results) => {
let mut writer = self.index.writer().expect("Failed to build index writer");
let mut index = self.index.clone();
let curie_helper = self.curie_helper.clone();
let index_task = tokio::task::spawn_blocking(move || {
debug_span!("Indexing").in_scope(|| {
task = Task::perform(tokio::task::spawn_blocking(move || {
let _span = debug_span!("Indexing").entered();
if let Ok(writer) = index.writer() {
let mut counter = 0;
for (individual, entry) in results {
debug!(%individual, ?entry);
let mut document = doc!(
Schema::type_field() => entry.catalog_id,
Schema::type_field() => entry.category_id,
Schema::iri_field() => individual.as_str(),
);
@@ -239,18 +248,19 @@ impl Publisher {
writer.add(document).expect("Failed to add document to search index");
counter += 1;
}
writer.commit().expect("Failed to commit changes to search index");
debug!("Added {counter} documents to search index");
});
});
task = Task::future(async {
match index_task.await {
Ok(()) => Message::None,
Err(err) => Message::ShowError(err.to_string()),
}
}), |result| if let Err(err) = result {
Message::ShowError(err.to_string())
} else {
Message::CommitIndex
});
}
Message::CommitIndex => {
if let Err(err) = self.index.commit() {
task = Task::done(Message::ShowError(err.to_string()));
}
}
Message::WindowClosed(id) => {
if self.window_id == id {
task = iced::exit();
@@ -263,7 +273,9 @@ impl Publisher {
}
Message::URLInputSubmitted => {
let url = Url::parse(&self.url_input).expect("Invalid URL");
self.navigator.goto(url.clone());
if &url != self.navigator.current() {
self.navigator.goto(url.clone());
}
task = Task::done(Message::NavigateTo(url));
}
Message::NavigateTo(url) => {
@@ -397,24 +409,27 @@ impl Publisher {
}
Message::QueryUpdated(new_query) => {
if let Some(search_state) = &mut self.search_state {
let catalog_id = search_state.entity_selection
let category_id = search_state.entity_selection
.clone()
.and_then(|info| self.ontology.catalog_id(&info.iri));
.and_then(|info| self.ontology.category_id(&info.iri));
let index = self.index.clone();
let query = new_query.clone();
let search_task = tokio::task::spawn_blocking(move || {
index.query(catalog_id, query.as_str(), Schema::all_fields(), 25)
});
task = Task::future(async {
/*let search_task = tokio::task::spawn_blocking(move || {
index.query(category_id, query.as_str(), Schema::all_fields(), 25)
});*/
/*task = Task::future(async {
match search_task.await {
Ok(Ok(results)) => Message::SetSearchResults(results),
Ok(Err(err)) => Message::ShowError(err.to_string()),
Err(err) => Message::ShowError(err.to_string()),
}
});
});*/
search_state.query = new_query;
task = Task::done(Message::SetSearchResults(
self.index.query(category_id, query.as_str(), Schema::all_fields(), 25)
.expect("Unable to complete search")
));
};
}
Message::SetSearchResults(results) => {
@@ -829,8 +844,6 @@ impl Publisher {
let reindex_ontology_button = button("Rebuild index").on_press(Message::RebuildIndex);
let traverse_button = button("Traverse").on_press(Message::Traverse);
let address_input = iri_input(&self.curie_helper, "URL", &self.url_input)
.on_input(Message::URLInputChanged)
.on_submit(Message::URLInputSubmitted)
@@ -865,7 +878,6 @@ impl Publisher {
forward_button,
add_row_button,
reindex_ontology_button,
traverse_button,
address_input,
save_button,
],
publish/src/error.rs
+3
View File
@@ -30,4 +30,7 @@ pub(crate) enum Error {
#[error(transparent)]
SparqlSyntax(#[from] oxigraph::sparql::SparqlSyntaxError),
#[error(transparent)]
Join(#[from] tokio::task::JoinError),
}
publish/src/rdf/ontology.rs
+36 -30
View File
@@ -2,11 +2,12 @@ use crate::error;
use crate::rdf::vocab::gl;
use oxigraph::model::vocab::{rdf, rdfs, xsd};
use oxigraph::model::{Dataset, Graph, GraphNameRef, LiteralRef, NamedNode, NamedNodeRef, NamedOrBlankNode, NamedOrBlankNodeRef, Term, TermRef, Triple, TripleRef};
use oxigraph::sparql::{PreparedSparqlQuery, QueryResults, SparqlEvaluator};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fmt::Display;
use std::path::{Path, PathBuf};
use std::sync::LazyLock;
use iced::futures::TryFutureExt;
use oxigraph::store::Store;
use tracing::debug_span;
use crate::rdf::{conversion, materialize};
@@ -46,7 +47,7 @@ static PREFIXES: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
("rdax", "http://rdaregistry.info/Elements/x/"),
("schema", "https://schema.org/"),
("quill", "http://fedora.quill.lan/rest/"),
("gl", "ONTOLOGY_PREFIX"),
("gl", ONTOLOGY_PREFIX),
].map(|(k, v)| (k.to_string(), v.to_string())))
});
@@ -108,7 +109,7 @@ impl OntologyBuilder {
#[derive(Clone, Debug)]
pub struct IndexEntry {
pub catalog_id: u64,
pub category_id: u64,
pub fields: HashMap<String, String>,
}
@@ -172,12 +173,12 @@ impl Ontology {
.collect::<Dataset>()
}
pub fn index_query(language: &LanguageCondition) -> PreparedSparqlQuery {
pub fn index(&self, language: &LanguageCondition, source: Option<Dataset>) -> impl Future<Output = error::Result<HashMap<NamedNode, IndexEntry>>> + 'static {
let language_filter = language.to_filter_expression("fieldValue");
let query = format!(r#"SELECT ?individual ?catalogId ?fieldName ?fieldValue
let query = format!(r#"SELECT ?individual ?categoryId ?fieldName ?fieldValue
WHERE {{
?class a gl:SearchableClass ;
gl:catalogId ?catalogId ;
gl:categoryId ?categoryId ;
gl:associatedProperty ?property .
?property gl:indexedByField/gl:fieldName ?fieldName .
@@ -190,39 +191,44 @@ WHERE {{
let mut sparql = SparqlEvaluator::new()
.with_prefix("gl", ONTOLOGY_PREFIX).unwrap()
.parse_query(&query)
.unwrap();
.expect("Unable to parse query");
sparql.dataset_mut().set_default_graph_as_union();
sparql
}
pub fn execute_query(&self, query: PreparedSparqlQuery) -> error::Result<QueryResults<'_>> {
Ok(query.on_store(&self.store).execute()?)
}
let store = self.store.clone();
tokio::task::spawn_blocking(move || {
let _span = debug_span!("Index Query").entered();
let query_results = if let Some(source) = &source {
sparql.on_queryable_dataset(source).execute()
} else {
sparql.on_store(&store).execute()
}.expect("Unable to execute indexing query");
pub fn transform_index_results(query_results: QueryResults) -> HashMap<NamedNode, IndexEntry> {
let mut results = HashMap::new();
if let QueryResults::Solutions(solutions) = query_results {
for solution in solutions.filter_map(Result::ok) {
let individual = solution.get("individual").and_then(conversion::term_to_named_node);
let catalog_id = solution.get("catalogId").and_then(conversion::term_to_u64);
let field_name = solution.get("fieldName").and_then(conversion::term_as_str);
let field_value = solution.get("fieldValue").and_then(conversion::term_as_str);
if let (Some(individual), Some(catalog_id), Some(field_name), Some(field_value)) = (individual, catalog_id, field_name, field_value) {
results.entry(individual.to_owned())
.and_modify(|entry: &mut IndexEntry| {
entry.fields.insert(field_name.to_owned(), field_value.to_owned());
}).or_insert(IndexEntry {
catalog_id,
let mut results = HashMap::new();
if let QueryResults::Solutions(solutions) = query_results {
for solution in solutions.filter_map(Result::ok) {
let individual = solution.get("individual").and_then(conversion::term_to_named_node);
let catalog_id = solution.get("categoryId").and_then(conversion::term_to_u64);
let field_name = solution.get("fieldName").and_then(conversion::term_as_str);
let field_value = solution.get("fieldValue").and_then(conversion::term_as_str);
if let (Some(individual), Some(catalog_id), Some(field_name), Some(field_value)) = (individual, catalog_id, field_name, field_value) {
results.entry(individual.to_owned())
.and_modify(|entry: &mut IndexEntry| {
entry.fields.insert(field_name.to_owned(), field_value.to_owned());
}).or_insert(IndexEntry {
category_id: catalog_id,
fields: HashMap::from_iter([(field_name.to_owned(), field_value.to_owned())]),
});
}
}
} else {
unreachable!()
}
}
results
results
}).map_err(error::Error::from)
}
pub fn catalog_id(&self, class: &NamedNode) -> Option<u64> {
self.store.quads_for_pattern(Some(class.into()), Some(gl::CATALOG_ID), None, None)
pub fn category_id(&self, class: &NamedNode) -> Option<u64> {
self.store.quads_for_pattern(Some(class.into()), Some(gl::CATEGORY_ID), None, None)
.filter_map(Result::ok)
.map(conversion::quad_into_term)
.filter_map(conversion::term_into_u64)
publish/src/rdf/vocab.rs
+2 -2
View File
@@ -7,8 +7,8 @@ pub mod gl {
pub const INDEXED_BY_FIELD: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/indexedByField");
pub const CATALOG_ID: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/catalogId");
pub const CATEGORY_ID: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/categoryId");
pub const SEARCHABLE_CLASS: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/SearchableClass");
publish/src/widget/iri_input.rs
-1
View File
@@ -6,7 +6,6 @@ use iced::advanced::{Layout, Widget};
use iced::advanced::widget::{tree, Tree};
use iced::mouse::{Cursor, Interaction};
use iced::widget::text_input::Catalog;
use tracing::debug;
use crate::rdf::curie::CurieHelper;
pub struct State {
search/Cargo.toml
+1
View File
@@ -5,6 +5,7 @@ edition = "2024"
publish = false
[dependencies]
futures.workspace = true
tantivy.workspace = true
thiserror.workspace = true
search/src/index.rs
+22 -5
View File
@@ -1,7 +1,8 @@
use std::fs;
use crate::{error, update};
use crate::{error, IndexWriter};
use crate::schema::Schema;
use std::path::PathBuf;
use std::sync::Arc;
use tantivy::collector::TopDocs;
use tantivy::directory::{ManagedDirectory, MmapDirectory};
use tantivy::query::{BooleanQuery, Occur, QueryParser, TermQuery};
@@ -51,6 +52,7 @@ impl SearchIndexBuilder {
Ok(SearchIndex {
index,
reader,
writer: None,
})
}
}
@@ -59,6 +61,7 @@ impl SearchIndexBuilder {
pub struct SearchIndex {
index: Index,
reader: IndexReader,
writer: Option<Arc<IndexWriter>>,
}
impl SearchIndex {
@@ -66,9 +69,23 @@ impl SearchIndex {
SearchIndexBuilder::default()
}
pub fn writer(&self) -> crate::Result<update::IndexWriter> {
let inner = self.index.writer(128 * 1024usize.pow(2))?;
Ok(crate::IndexWriter { inner })
pub fn writer(&mut self) -> crate::Result<&IndexWriter> {
if self.writer.is_none() {
let inner = self.index.writer(128 * 1024 * 1024)?;
self.writer = Some(Arc::new(IndexWriter { inner }));
}
Ok(self.writer.as_ref().unwrap())
}
pub fn commit(&mut self) -> crate::Result<()> {
if let Some(writer) = self.writer.take() {
if let Some(mut writer) = Arc::into_inner(writer) {
writer.commit()?;
}
}
Ok(())
}
pub fn query(
@@ -78,7 +95,7 @@ impl SearchIndex {
default_fields: Vec<Field>,
limit: usize,
) -> error::Result<Vec<TantivyDocument>> {
let _ = debug_span!("Search Query").entered();
let _span = debug_span!("Search Query").entered();
let parser = QueryParser::for_index(&self.index, default_fields);
let (user_query, _) = parser.parse_query_lenient(user_query);
search/src/update.rs
+2 -3
View File
@@ -11,13 +11,12 @@ impl IndexWriter {
Ok(())
}
pub fn remove_all_of_type(&mut self, type_: u64) -> crate::Result<()> {
pub fn remove_all_of_type(&self, type_: u64) -> crate::Result<()> {
self.inner.delete_term(Term::from_field_u64(Schema::type_field(), type_));
self.inner.commit()?;
Ok(())
}
pub fn remove_all(&mut self) -> crate::Result<()> {
pub fn remove_all(&self) -> crate::Result<()> {
self.inner.delete_all_documents()?;
Ok(())
}