gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
alex
2026-08-05 11:21:25 -04:00
parent 921d56f2b3
commit 7ce5ed118d
18 changed files with 1144 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+617 -66
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+3 -1
View File
@@ -10,8 +10,9 @@ gl-search = { path = "search" }
ldp = { path = "../../ldp/ldp", features = ["keyed"] }
async-trait = "0.1"
base64 = "0.22"
base64 = "0.23"
bytes = "1.12"
clap = { version = "4.6", features = ["derive"] }
color-eyre = "0.6"
futures = "0.3"
http = "1.4"
@@ -22,6 +23,7 @@ parse_link_header = "0.4"
pin-project-lite = "0.2"
rfd = "0.17"
slotmap = "1.1"
subtitler = "2.6"
tantivy = "0.26"
tar = "0.4"
thiserror = "2.0"
ontology/graphofliberty.ttl
+48 -4
View File
@@ -92,10 +92,18 @@ xsd:unsignedShort rdf:type rdfs:Datatype ;
rdfs:label "associated property"@en .
### https://graphofliberty.org/2026/04/ont/fullText
:fullText rdf:type owl:ObjectProperty ;
rdfs:comment "A URL to a file which is intended to be passed to a full-text search database."@en ;
rdfs:label "has full text"@en .
### https://graphofliberty.org/2026/04/ont/derivedWith
:derivedWith rdf:type owl:ObjectProperty ;
rdfs:range :DerivationMethodology ;
rdfs:comment "Relates a File to a Derivation Methodology."@en ;
rdfs:label "derived with"@en .
### https://graphofliberty.org/2026/04/ont/indexDocument
:indexDocument rdf:type owl:ObjectProperty ;
rdfs:range :IndexDocument ;
rdfs:comment "Relates an entity to an Index Document."@en ;
rdfs:label "has index document"@en .
### https://graphofliberty.org/2026/04/ont/indexedByField
@@ -105,6 +113,12 @@ xsd:unsignedShort rdf:type rdfs:Datatype ;
rdfs:label "indexed by field"@en .
### https://graphofliberty.org/2026/04/ont/subtitles
:subtitles rdf:type owl:ObjectProperty ;
rdfs:comment "Relates an entity to a File which contains subtitles for the entity."@en ;
rdfs:label "has subtitles"@en .
### https://graphofliberty.org/2026/04/ont/template
:template rdf:type owl:ObjectProperty ;
rdfs:comment "A default set of triples used during the creation of new entities of the associated class."@en ;
@@ -151,6 +165,11 @@ xsd:unsignedShort rdf:type rdfs:Datatype ;
# Classes
#################################################################
### https://graphofliberty.org/2026/04/ont/AudioBook
:AudioBook rdf:type owl:Class ;
rdfs:subClassOf :Entity .
### https://graphofliberty.org/2026/04/ont/DerivationMethodology
:DerivationMethodology rdf:type owl:Class ;
rdfs:comment "The methodology used to derive content. Individuals of this class should provide enough information to replicate the process by which the content was generated. For example, if Whisper was used derive captions for an audio book, the precise model used (e.g. large v3), a link to the source code (e.g. HuggingFace) and/or a paper should be provided."@en ;
@@ -163,6 +182,12 @@ xsd:unsignedShort rdf:type rdfs:Datatype ;
rdfs:label "Graph of Liberty Entity"@en .
### https://graphofliberty.org/2026/04/ont/IndexDocument
:IndexDocument rdf:type owl:Class ;
rdfs:comment "A document which is intended to be passed verbatim to a full-text search engine for indexing."@en ;
rdfs:label "Index Document"@en .
### https://graphofliberty.org/2026/04/ont/IndexDocumentField
:IndexDocumentField rdf:type owl:Class ;
rdfs:comment "Describes a single field present within a document that is indexed in a full-text search database."@en ;
@@ -175,6 +200,12 @@ xsd:unsignedShort rdf:type rdfs:Datatype ;
rdfs:label "Searchable Class"@en .
### https://graphofliberty.org/2026/04/ont/Transcription
:Transcription rdf:type owl:Class ;
rdfs:comment "A structured or unstructured textual file which contains a transcription of an audio file."@en ;
rdfs:label "Transcription"@en .
#################################################################
# Individuals
#################################################################
@@ -338,6 +369,12 @@ ldp:contains rdf:type owl:NamedIndividual ;
:readOnly "true"^^xsd:boolean .
### https://graphofliberty.org/2026/04/ont/AudioBook
:AudioBook rdf:type owl:NamedIndividual ,
:SearchableClass ;
:categoryId "4"^^xsd:nonNegativeInteger .
### https://graphofliberty.org/2026/04/ont/Definition
:Definition rdf:type owl:NamedIndividual ,
:IndexDocumentField ;
@@ -370,4 +407,11 @@ ldp:contains rdf:type owl:NamedIndividual ;
rdfs:label "Surname Field"@en .
#################################################################
# Annotations
#################################################################
:AudioBook rdfs:label "Audio Book"@en .
### Generated by the OWL API (version 4.5.29.2024-05-13T12:11:03Z) https://github.com/owlcs/owlapi
publish/Cargo.toml
+1
View File
@@ -8,6 +8,7 @@ edition = "2024"
gl-search.workspace = true
ldp.workspace = true
clap.workspace = true
color-eyre.workspace = true
csv = "1.4"
http.workspace = true
publish/src/app.rs
+31 -111
View File
@@ -1,40 +1,34 @@
use std::collections::HashMap;
use crate::rdf::ontology::{IndexEntry, LabeledIri, Ontology};
use crate::navigator::Navigator;
use crate::rdf::curie::CurieHelper;
use crate::rdf::ontology::{LabeledIri, Ontology};
use crate::rdf::term_helper::{TermHelper, TermHelperMut};
use gl_search::{Schema, SearchDocument, SearchIndex, doc, Value};
use crate::rdf::vocab::rda;
use crate::widget::iri_input::iri_input;
use crate::widget::navigation_area::navigation_area;
use gl_search::language;
use gl_search::{IndexWriter, Schema, SearchDocument, SearchIndex, Value};
use http::StatusCode;
use iced::alignment::Horizontal;
use iced::keyboard::{Event, key};
use iced::widget::button::Style;
use iced::widget::grid::Sizing;
use iced::widget::text::Wrapping;
use iced::widget::{button, center, column, combo_box, container, grid, mouse_area, opaque, operation, pick_list, row, scrollable, space, stack, table, text, text_input, toggler};
use iced::window::Settings;
use iced::{Background, Color, Element, Length, Subscription, Task, color, window};
use iced::widget::text::Wrapping;
use iced::{Background, Color, Element, Length, Subscription, Task, color, keyboard, window};
use ldp::middleware::BasicAuthMiddleware;
use ldp::model::{KeyedDataset, QuadKey};
use ldp::reqwest::{Client, Url};
use ldp::reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
use ldp::traverse::Traverse;
use ldp::{RdfSource, RdfSourceUpdateResponse, ResourceRequestBuilder, SerializationOptions};
use oxigraph::io::RdfFormat;
use oxigraph::model::vocab::{rdf, rdfs};
use oxigraph::model::{BaseDirection, Dataset, NamedNode, NamedOrBlankNode, Quad, Term};
use tracing::{debug_span, error, field, trace};
use crate::navigator::Navigator;
use crate::rdf::curie::CurieHelper;
use crate::rdf::language;
use crate::rdf::vocab::rda;
use crate::widget::iri_input::iri_input;
use crate::widget::navigation_area::navigation_area;
use tracing::{debug_span, error, trace};
#[derive(Clone, Debug)]
pub(crate) enum Message {
None,
RebuildIndex,
Traverse,
AddRdfSource(RdfSource<Dataset>),
ConcludeTraversal,
IndexQueryResults(HashMap<NamedNode, IndexEntry>),
WindowClosed(window::Id),
URLInputChanged(String),
URLInputSubmitted,
@@ -71,6 +65,7 @@ pub(crate) enum Message {
NavigateForward,
ResetState,
SetReadOnly(bool),
Event(Event),
}
#[derive(Default, Debug, Clone)]
@@ -181,90 +176,6 @@ impl Publisher {
trace!(?message);
match message {
Message::RebuildIndex => {
let mut writer = self.index.writer().expect("Unable to obtain index writer");
let clear_index_task = Task::perform(tokio::task::spawn_blocking(move || {
let _span = debug_span!("Clear Index").entered();
writer.delete_all_documents().expect("Unable to clear index");
writer.commit().expect("Unable to commit index operation");
}), |_| Message::None);
let index_future = self.ontology
.query_for_indexable_triples(&*language::ENGLISH_OR_UNTAGGED, None);
let index_task = Task::perform(index_future, |task_result| {
match task_result {
Ok(results) => Message::IndexQueryResults(results),
Err(err) => Message::ShowError(err.to_string()),
}
});
task = clear_index_task.chain(Task::batch([
index_task,
Task::done(Message::Traverse),
]));
}
Message::Traverse => {
let client = self.http_client.clone();
let root = Url::parse(&self.url_input).expect("Invalid URL");
let stream = Traverse::new(client, root, None);
task = Task::run(stream, |result| match result {
Ok(rdf_source) => Message::AddRdfSource(rdf_source),
Err(err) => Message::ShowError(format!("Unable to fetch RDF Source: {err}")),
}).chain(Task::done(Message::ConcludeTraversal));
self.traversal = Some(self.ontology.to_dataset());
}
Message::AddRdfSource(rdf_source) => {
if let Some(traversal) = &mut self.traversal {
traversal.extend(rdf_source.dataset());
}
}
Message::ConcludeTraversal => {
if let Some(traversal) = self.traversal.take() {
let index_task = self.ontology
.query_for_indexable_triples(&*language::ENGLISH_OR_UNTAGGED, Some(traversal));
task = Task::perform(index_task, |task_result| {
match task_result {
Ok(results) => Message::IndexQueryResults(results),
Err(err) => Message::ShowError(err.to_string()),
}
});
}
}
Message::IndexQueryResults(results) => {
let mut writer = self.index.writer().expect("Unable to obtain index writer");
let curie_helper = self.curie_helper.clone();
task = Task::perform(tokio::task::spawn_blocking(move || {
let span = debug_span!("Index Query Results", documents = field::Empty).entered();
let mut counter = 0usize;
for (individual, entry) in results {
let mut document = doc!(
Schema::type_field() => entry.category_id,
Schema::iri_field() => individual.as_str(),
);
if let Some(curie) = curie_helper.abbreviate(None, individual.as_str()) {
document.add_text(Schema::curie_field(), curie);
}
for (key, value) in entry.fields {
document.add_text(Schema::field(&key, language::ENGLISH_PRIMARY), value.as_str());
}
writer.add_document(document).expect("Failed to add document to search index");
counter += 1;
}
span.record("documents", counter);
writer.commit()
}), |result| {
if let Err(err) = result {
Message::ShowError(err.to_string())
} else {
Message::None
}
});
}
Message::WindowClosed(id) => {
if self.window_id == id {
task = iced::exit();
@@ -355,9 +266,8 @@ impl Publisher {
.map(|datatype| self.curie_helper.abbreviate(None, datatype.as_str())
.unwrap_or(datatype.as_str().to_string()));
let datatype_state = combo_box::State::with_selection(
let datatype_state = combo_box::State::new(
self.abbreviated_datatypes.clone(),
datatype.as_ref(),
);
let state = RowState {
read_only,
@@ -425,7 +335,7 @@ impl Publisher {
let query = new_query.clone();
search_state.query = new_query;
let results = self.index.query(category_id, query.as_str(), Schema::all_fields(), 10)
let results = self.index.query(category_id, query.as_str(), Schema::all_fields(), 25)
.expect("Unable to complete search");
task = Task::done(Message::SetSearchResults(results));
};
@@ -616,6 +526,17 @@ impl Publisher {
Message::SetReadOnly(value) => {
self.show_read_only = value;
}
Message::Event(Event::KeyPressed {
key: keyboard::Key::Named(key::Named::Tab),
modifiers,
..
}) => {
if modifiers.shift() {
task = operation::focus_previous();
} else {
task = operation::focus_next();
}
}
_ => {}
}
task
@@ -628,6 +549,7 @@ impl Publisher {
pub(crate) fn subscription(&self) -> Subscription<Message> {
Subscription::batch([
window::close_events().map(Message::WindowClosed),
keyboard::listen().map(Message::Event),
])
}
@@ -697,7 +619,7 @@ impl Publisher {
// If the datatype is something, then it's not a named node, and we should use a plain
// text_input as opposed to an iri_input.
let object_input: Element<Message> = if term.datatype().is_some() {
let base = text_input("Object", value.as_str());
let base = text_input("Object", value.clone());
if self.ontology.is_read_only(triple.as_ref()) {
base.into()
} else {
@@ -734,9 +656,10 @@ impl Publisher {
}).into()
};
let language = term.language().unwrap_or("en").to_owned();
let language_input = match term.datatype() {
Some(rdf::LANG_STRING) | Some(rdf::DIR_LANG_STRING) => Some(container(
text_input("Language", term.language().unwrap_or("en")).on_input(move |input| {
text_input("Language", language).on_input(move |input| {
let value = if input.is_empty() { None } else { Some(input) };
Message::LanguageUpdated(key, value)
}),
@@ -848,7 +771,7 @@ impl Publisher {
.and_then(|value| value.as_str())
.unwrap_or_default();
let value = document.get_first(Schema::field(&field_name, language::ENGLISH_PRIMARY))
let value = document.get_first(Schema::field(&field_name, Some(language::ENGLISH_PRIMARY)))
.and_then(|value| value.as_str())
.unwrap_or_default();
@@ -877,8 +800,6 @@ impl Publisher {
let add_row_button = button("Add row").on_press(Message::AddRow(None));
let reindex_ontology_button = button("Rebuild index").on_press(Message::RebuildIndex);
let address_input = iri_input(&self.curie_helper, "URL", None, &self.url_input)
.on_input(Message::URLInputChanged)
.on_submit(Message::URLInputSubmitted)
@@ -920,7 +841,6 @@ impl Publisher {
let content = navigation_area(column![
row![
reindex_ontology_button,
add_row_button,
back_button,
forward_button,
publish/src/args.rs
+23
View File
@@ -0,0 +1,23 @@
use clap::{Args, Parser, Subcommand};
#[derive(Args)]
pub(crate) struct SearchArgs {
#[arg(short, long, value_name = "DOC TYPE")]
pub(crate) discriminant: Option<u64>,
#[arg(value_name = "QUERY")]
pub(crate) query: String,
}
#[derive(Subcommand)]
pub(crate) enum Command {
Search(SearchArgs),
Reindex,
}
#[derive(Parser)]
#[clap(version, long_about = None)]
pub(crate) struct AppArgs {
#[command(subcommand)]
pub(crate) command: Option<Command>,
}
publish/src/main.rs
+122 -5
View File
@@ -5,12 +5,29 @@ mod windows;
mod widget;
mod navigator;
mod theme;
mod args;
use crate::app::Publisher;
use std::sync::{Arc, RwLock};
use std::thread;
use clap::Parser;
use color_eyre::eyre::eyre;
use iced::futures::StreamExt;
use iced::Task;
use ldp::middleware::BasicAuthMiddleware;
use ldp::reqwest::{Client, Url};
use ldp::reqwest_middleware::ClientBuilder;
use ldp::traverse::Traverse;
use oxigraph::model::Dataset;
use tracing::{debug, debug_span, error, field};
use crate::app::{Message, Publisher};
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, fmt};
use tracing_subscriber::fmt::format::FmtSpan;
use gl_search::{DocType, Document, Schema, SearchIndex};
use crate::args::{AppArgs, Command};
use crate::rdf::curie::CurieHelper;
use crate::rdf::ontology::Ontology;
fn main() -> color_eyre::Result<()> {
let appender = tracing_appender::rolling::never("/tmp", "publisher-log");
@@ -20,10 +37,110 @@ fn main() -> color_eyre::Result<()> {
.init();
color_eyre::install()?;
iced::daemon(Publisher::new, Publisher::update, Publisher::view)
.title(Publisher::title)
.subscription(Publisher::subscription)
.run()?;
let ontology = Ontology::builder()
.with_path("/home/alex/.local/share/org.graphofliberty.desktop/ontology")
.build()
.expect("Failed to build ontology");
let mut index = SearchIndex::builder()
.with_path("/home/alex/.local/share/org.graphofliberty.desktop/index")
.build()
.expect("Failed to build search index");
let args = AppArgs::parse();
match args.command {
Some(Command::Search(args)) => {
for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 5)? {
println!("{}", doc.to_json(Schema::schema()));
}
}
Some(Command::Reindex) => {
let curie_helper = CurieHelper::new(Ontology::prefixes().clone());
let writer = Arc::new(RwLock::new(index.writer()?));
debug_span!("Clear Index").in_scope(|| {
let mut writer = writer.write()?;
writer.delete_all_documents()?;
writer.commit()
})?;
let store = ontology.store();
let writer_clone = writer.clone();
let index_classes_thread = thread::spawn(move || {
let span = debug_span!("Index Classes", documents = field::Empty).entered();
let writer = writer_clone.read().unwrap();
let count = gl_search::rdf::index_triples(DocType::RdfsClass, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &*writer)?;
span.record("documents", count);
Ok::<(), gl_search::SearchError>(())
});
let store = ontology.store();
let writer_clone = writer.clone();
let index_properties_thread = thread::spawn(move || {
let span = debug_span!("Index Properties", documents = field::Empty).entered();
let writer = writer_clone.read().unwrap();
let count = gl_search::rdf::index_triples(DocType::RdfProperty, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
span.record("documents", count);
Ok::<(), gl_search::SearchError>(())
});
let store = ontology.store();
let writer_clone = writer.clone();
let index_concepts_thread = thread::spawn(move || {
let span = debug_span!("Index Concepts", documents = field::Empty).entered();
let writer = writer_clone.read().unwrap();
let count = gl_search::rdf::index_triples(DocType::SkosConcept, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
span.record("documents", count);
Ok::<(), gl_search::SearchError>(())
});
let starting_url = Url::parse("http://fedora.quill.lan/rest/")?;
let writer_clone = writer.clone();
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.name("repository-traversal")
.build()?;
let traversal_task = runtime.spawn(async move {
let http_client = ClientBuilder::new(Client::new())
.with(BasicAuthMiddleware::new(
"fedoraAdmin".to_string(),
Some("fedoraAdmin".to_string()),
))
.build();
let mut documents = 0usize;
let mut traversal = Traverse::new(http_client, starting_url, None);
while let Some(result) = traversal.next().await {
match result {
Ok(rdf_source) => {
let writer = writer_clone.read().unwrap();
documents += gl_search::rdf::index_entity(rdf_source.dataset(), &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
}
Err(err) => error!(?err),
}
}
debug!(documents, "Repository Traversal");
Ok::<(), gl_search::SearchError>(())
});
index_classes_thread.join().unwrap()?;
index_properties_thread.join().unwrap()?;
index_concepts_thread.join().unwrap()?;
runtime.block_on(traversal_task)??;
debug_span!("Commit").in_scope(|| {
let mut writer = writer.write()?;
writer.commit()
})?;
}
None => {
iced::daemon(Publisher::new, Publisher::update, Publisher::view)
.title(Publisher::title)
.subscription(Publisher::subscription)
.run()?;
}
}
Ok(())
}
publish/src/rdf/mod.rs
-1
View File
@@ -3,5 +3,4 @@ pub(crate) mod term_helper;
pub mod vocab;
pub(crate) mod materialize;
pub(crate) mod conversion;
pub(crate) mod language;
pub(crate) mod curie;
publish/src/rdf/ontology.rs
+9 -4
View File
@@ -1,17 +1,17 @@
use crate::error;
use crate::rdf::vocab::gl;
use crate::rdf::{conversion, materialize};
use gl_search::language::LanguageCondition;
use iced::futures::TryFutureExt;
use oxigraph::model::vocab::{rdf, rdfs, xsd};
use oxigraph::model::{Dataset, Graph, GraphNameRef, LiteralRef, NamedNode, NamedNodeRef, NamedOrBlankNode, NamedOrBlankNodeRef, Term, TermRef, Triple, TripleRef};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use oxigraph::store::Store;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fmt::Display;
use std::path::{Path, PathBuf};
use std::sync::LazyLock;
use iced::futures::TryFutureExt;
use oxigraph::store::Store;
use tracing::{debug_span, field};
use crate::rdf::{conversion, materialize};
use crate::rdf::language::LanguageCondition;
const ONTOLOGY_PREFIX: &str = "https://graphofliberty.org/2026/04/ont/";
const ONTOLOGY_GRAPH_NAME: GraphNameRef = GraphNameRef::NamedNode(NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont"));
@@ -50,6 +50,7 @@ static PREFIXES: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
("rdax", "http://rdaregistry.info/Elements/x/"),
("rdaco", "http://rdaregistry.info/termList/RDAContentType/"),
("rdact", "http://rdaregistry.info/termList/RDACarrierType/"),
("rdamt", "http://rdaregistry.info/termList/RDAMediaType/"),
("rdaft", "http://rdaregistry.info/termList/fileType/"),
("schema", "https://schema.org/"),
("gl", "http://fedora.quill.lan/rest/"),
@@ -179,6 +180,10 @@ impl Ontology {
.collect::<Dataset>()
}
pub fn store(&self) -> Store {
self.store.clone()
}
pub fn query_for_indexable_triples(&self, language: &LanguageCondition, source: Option<Dataset>) -> impl Future<Output = error::Result<HashMap<NamedNode, IndexEntry>>> + 'static {
let language_filter = language.to_filter_expression("fieldValue");
let query = format!(r#"SELECT ?individual ?categoryId ?fieldName ?fieldValue
publish/src/widget/iri_input.rs
+24 -8
View File
@@ -1,8 +1,8 @@
use iced::{alignment, keyboard, widget, Element, Event, Length, Rectangle, Size, Theme, Background, Border};
use iced::{alignment, keyboard, widget, Element, Event, Length, Rectangle, Size};
use iced::advanced::{mouse, text, Shell, renderer};
use iced::advanced::layout::{Limits, Node};
use iced::advanced::{Layout, Widget};
use iced::advanced::widget::{tree, Tree};
use iced::advanced::widget::{tree, Tree, Operation};
use iced::clipboard::Content;
use iced::mouse::{Cursor, Interaction};
use iced::widget::text_input::{Catalog, Status, Style, StyleFn};
@@ -31,9 +31,9 @@ where
Theme: Catalog,
Renderer: text::Renderer,
{
pub fn new(curie_helper: &'a CurieHelper, placeholder: &str, base: Option<&str>, iri: &str) -> Self {
pub fn new(curie_helper: &'a CurieHelper, placeholder: &'a str, base: Option<&str>, iri: &str) -> Self {
let display_value = curie_helper.abbreviate(base, iri).unwrap_or_else(|| iri.to_string());
let text_input = widget::TextInput::new(placeholder, &display_value);
let text_input = widget::TextInput::new(placeholder, display_value);
Self {
curie_helper,
base: base.map(String::from),
@@ -44,7 +44,7 @@ where
}
#[must_use]
pub fn align_x(mut self, alignment: impl Into<alignment::Horizontal>) -> Self {
pub fn align_x(mut self, alignment: impl Into<text::Alignment>) -> Self {
self.text_input = self.text_input.align_x(alignment);
self
}
@@ -87,6 +87,12 @@ where
self.text_input = self.text_input.style(style);
self
}
#[must_use]
pub fn id(mut self, id: impl Into<widget::Id>) -> Self {
self.text_input = self.text_input.id(id);
self
}
}
impl<'a, Message, Theme, Renderer> From<IriInput<'a, Message, Theme, Renderer>>
@@ -94,14 +100,14 @@ for Element<'a, Message, Theme, Renderer>
where
Message: Clone + 'a,
Theme: Catalog + 'a,
Renderer: text::Renderer + 'a,
Renderer: text::Renderer + 'static,
{
fn from(value: IriInput<'a, Message, Theme, Renderer>) -> Self {
Element::new(value)
}
}
pub fn iri_input<'a, Message, Theme, Renderer>(curie_helper: &'a CurieHelper, placeholder: &str, base: Option<&str>, iri: &str) -> IriInput<'a, Message, Theme, Renderer>
pub fn iri_input<'a, Message, Theme, Renderer>(curie_helper: &'a CurieHelper, placeholder: &'a str, base: Option<&str>, iri: &str) -> IriInput<'a, Message, Theme, Renderer>
where
Message: Clone,
Theme: Catalog,
@@ -114,7 +120,7 @@ impl <Message, Theme, Renderer> Widget<Message, Theme, Renderer> for IriInput<'_
where
Message: Clone,
Theme: Catalog,
Renderer: text::Renderer,
Renderer: text::Renderer + 'static,
{
fn size(&self) -> Size<Length> {
Widget::size(&self.text_input)
@@ -143,6 +149,16 @@ where
tree.diff_children(&mut [&mut self.text_input as &mut dyn Widget<_, _, _>]);
}
fn operate(
&mut self,
tree: &mut Tree,
layout: Layout<'_>,
renderer: &Renderer,
operation: &mut dyn Operation,
) {
self.text_input.operate(&mut tree.children[0], layout, renderer, operation);
}
fn update(&mut self, tree: &mut Tree, event: &Event, layout: Layout<'_>, cursor: Cursor, renderer: &Renderer, shell: &mut Shell<'_, Message>, viewport: &Rectangle) {
let state = tree.state.downcast_mut::<State>();
search/Cargo.toml
+3
View File
@@ -10,6 +10,9 @@ tantivy.workspace = true
thiserror.workspace = true
#poppler-rs = "0.26.0-alpha.0"
subtitler.workspace = true
tracing.workspace = true
#oxidize-pdf = { version = "1.6", features = ["ocr-tesseract"] }
oxigraph.workspace = true
oxilangtag.workspace = true
#xml = "1.2"
search/src/error.rs
+12
View File
@@ -14,4 +14,16 @@ pub enum SearchError {
#[error(transparent)]
Io(#[from] std::io::Error),
#[error(transparent)]
Parse(#[from] subtitler::error::ParseError),
#[error(transparent)]
QueryEvaluation(#[from] oxigraph::sparql::QueryEvaluationError),
#[error(transparent)]
SparqlSyntax(#[from] oxigraph::sparql::SparqlSyntaxError),
#[error("The provided DocType is not valid for this operation")]
InvalidDocType,
}
search/src/index.rs
+4 -3
View File
@@ -1,3 +1,4 @@
use std::collections::HashMap;
use std::fs;
use crate::error;
use crate::schema::Schema;
@@ -5,7 +6,7 @@ use std::path::PathBuf;
use tantivy::collector::TopDocs;
use tantivy::directory::{ManagedDirectory, MmapDirectory};
use tantivy::query::{BooleanQuery, Occur, QueryParser, TermQuery};
use tantivy::schema::{Field, IndexRecordOption};
use tantivy::schema::{Field, IndexRecordOption, OwnedValue};
use tantivy::tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer, TokenizerManager};
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term};
use tracing::debug_span;
@@ -65,7 +66,7 @@ impl SearchIndex {
SearchIndexBuilder::default()
}
pub fn writer(&mut self) -> crate::Result<IndexWriter> {
pub fn writer(&mut self) -> crate::Result<IndexWriter<HashMap<Field, OwnedValue>>> {
Ok(self.index.writer(128 * 1024 * 1024)?)
}
@@ -86,7 +87,7 @@ impl SearchIndex {
];
if let Some(type_) = type_ {
let doc_type_term = Term::from_field_u64(Schema::type_field(), type_);
let doc_type_term = Term::from_field_u64(Schema::discriminant_field(), type_);
let doc_type_query = Box::new(TermQuery::new(doc_type_term, IndexRecordOption::Basic));
subqueries.push((Occur::Must, doc_type_query));
}
publish/src/rdf/language.rs → search/src/language.rs
+11 -3
View File
@@ -1,5 +1,5 @@
use std::sync::LazyLock;
use oxigraph::model::Term;
use oxigraph::model::{Term, TermRef};
use oxilangtag::LanguageTag;
pub const ENGLISH_PRIMARY: &str = "en";
@@ -16,8 +16,8 @@ pub enum LanguageCondition {
}
impl LanguageCondition {
pub fn primary_matches_term(&self, term: &Term) -> bool {
if let Term::Literal(literal) = term {
pub fn primary_matches_term<'a>(&self, term: impl Into<TermRef<'a>>) -> bool {
if let TermRef::Literal(literal) = term.into() {
let tag = literal.language()
.map(LanguageTag::parse_and_normalize)
.and_then(Result::ok);
@@ -35,6 +35,14 @@ impl LanguageCondition {
}
}
pub fn primary_language(&self) -> Option<&str> {
match self {
LanguageCondition::ExactMatchOnly(tag) => Some(tag.primary_language()),
LanguageCondition::ExactMatchOrUntagged(tag) => Some(tag.primary_language()),
_ => None,
}
}
pub fn to_filter_expression(&self, var: &str) -> String {
match self {
LanguageCondition::ExactMatchOnly(language) => format!(r#"FILTER(langMATCHES(LANG(?{var}), "{language}"))"#),
search/src/lib.rs
+22 -1
View File
@@ -1,11 +1,32 @@
mod error;
mod index;
mod schema;
pub mod subtitles;
pub mod rdf;
pub mod language;
use oxigraph::model::NamedNodeRef;
pub use tantivy::TantivyDocument as SearchDocument;
pub use tantivy::doc;
pub use tantivy::schema::document::Value;
pub use tantivy::schema::document::{Document, Value};
pub use tantivy::indexer::IndexWriter;
pub use error::{Result, SearchError};
pub use index::{SearchIndex, SearchIndexBuilder};
pub use schema::Schema;
pub enum DocType {
RdfProperty = 0,
RdfsClass = 1,
SkosConcept = 2,
Person = 3,
}
impl DocType {
pub fn from_named_node(node: NamedNodeRef) -> Option<Self> {
match node.as_str() {
"http://rdaregistry.info/Elements/c/C10004" => Some(DocType::Person),
_ => None,
}
}
}
search/src/rdf.rs
+177
View File
@@ -0,0 +1,177 @@
use std::collections::HashMap;
use oxigraph::model::{Dataset, NamedNode, NamedNodeRef, NamedOrBlankNodeRef, Term, TermRef};
use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use oxigraph::store::Store;
use tantivy::IndexWriter;
use tantivy::schema::{Field, OwnedValue};
use crate::DocType;
use crate::language::LanguageCondition;
use crate::schema::Schema;
use crate::SearchError::InvalidDocType;
fn term_to_named_node(term: &Term) -> Option<&NamedNode> {
if let Term::NamedNode(node) = term {
Some(node)
} else {
None
}
}
fn term_ref_as_named_node(term: TermRef<'_>) -> Option<NamedNodeRef<'_>> {
if let TermRef::NamedNode(node) = term {
Some(node)
} else {
None
}
}
fn term_as_str(term: &Term) -> Option<&str> {
if let Term::Literal(literal) = term {
match literal.datatype() {
xsd::STRING | xsd::NORMALIZED_STRING | rdf::LANG_STRING | rdf::DIR_LANG_STRING => {
Some(literal.value())
}
_ => None,
}
} else {
None
}
}
fn term_ref_as_str(term: TermRef<'_>) -> Option<&str> {
if let TermRef::Literal(literal) = term {
match literal.datatype() {
xsd::STRING | xsd::NORMALIZED_STRING | rdf::LANG_STRING | rdf::DIR_LANG_STRING => {
Some(literal.value())
}
_ => None,
}
} else {
None
}
}
const GIVEN_NAME: NamedNodeRef = NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50292");
const SURNAME: NamedNodeRef = NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50291");
pub fn index_entity(dataset: &Dataset, language: &LanguageCondition, writer: &IndexWriter<HashMap<Field, OwnedValue>>) -> crate::Result<usize> {
let mut subject_type_map = HashMap::new();
for quad in dataset.quads_for_pattern(None, Some(rdf::TYPE), None, None) {
if let NamedOrBlankNodeRef::NamedNode(subject) = quad.subject {
subject_type_map.entry(subject)
.and_modify(|types: &mut Vec<_>| types.push(quad.object))
.or_insert_with(Vec::new);
}
}
let mut counter = 0usize;
for (subject, types) in subject_type_map {
for type_ in types {
if let Some(doc_type) = term_ref_as_named_node(type_).and_then(DocType::from_named_node) {
match doc_type {
DocType::Person => {
let mut doc = index_person(dataset, subject, language);
doc.insert(Schema::discriminant_field(), OwnedValue::U64(doc_type as u64));
doc.insert(Schema::iri_field(), OwnedValue::Str(subject.as_str().to_string()));
writer.add_document(doc)?;
counter += 1;
}
_ => {}
}
}
}
}
Ok(counter)
}
fn index_person(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &LanguageCondition) -> HashMap<Field, OwnedValue> {
let given_name = dataset.quads_for_pattern(Some(subject.into()), Some(GIVEN_NAME), None, None)
.map(|q| q.object)
.filter(|term| language.primary_matches_term(*term))
.filter_map(term_ref_as_str)
.map(String::from)
.map(OwnedValue::Str)
.next()
.unwrap_or_else(|| OwnedValue::Null);
let surname = dataset.quads_for_pattern(Some(subject.into()), Some(SURNAME), None, None)
.map(|q| q.object)
.filter(|term| language.primary_matches_term(*term))
.filter_map(term_ref_as_str)
.map(String::from)
.map(OwnedValue::Str)
.next()
.unwrap_or_else(|| OwnedValue::Null);
HashMap::from_iter([
(Schema::field("given_name", language.primary_language()), given_name),
(Schema::field("surname", language.primary_language()), surname),
])
}
pub fn index_triples(doc_type: DocType, store: Store, language: &LanguageCondition, writer: &IndexWriter<HashMap<Field, OwnedValue>>) -> crate::Result<usize> {
let label_filter = language.to_filter_expression("label");
let description_filter = language.to_filter_expression("description");
let rdf_type = match doc_type {
DocType::RdfsClass => Ok("rdfs:Class"),
DocType::RdfProperty => Ok("rdf:Property"),
DocType::SkosConcept => Ok("skos:Concept"),
_ => Err(InvalidDocType),
}?;
let query = format!(r#"SELECT ?subject ?label ?description WHERE {{
?subject a {rdf_type} ;
rdfs:label ?label .
{label_filter}
OPTIONAL {{ ?subject rdfs:comment ?comment }}
OPTIONAL {{ ?subject skos:definition ?definition }}
BIND(COALESCE(?definition, ?comment) AS ?description)
{description_filter}
}}"#);
let mut sparql = SparqlEvaluator::new()
.with_prefix("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#").unwrap()
.with_prefix("rdfs", "http://www.w3.org/2000/01/rdf-schema#").unwrap()
.with_prefix("skos", "http://www.w3.org/2004/02/skos/core#").unwrap()
.parse_query(&query)?;
sparql.dataset_mut().set_default_graph_as_union();
let mut counter = 0usize;
let query_results = sparql.on_store(&store).execute()?;
if let QueryResults::Solutions(solutions) = query_results {
let discriminant = OwnedValue::U64(doc_type as u64);
let primary_language = language.primary_language();
for solution in solutions.filter_map(Result::ok) {
let mut document = HashMap::with_capacity(4);
document.insert(Schema::discriminant_field(), discriminant.clone());
let subject = solution.get("subject")
.and_then(term_to_named_node)
.map(NamedNode::as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::iri_field(), subject);
let label = solution.get("label")
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::field("label", primary_language), label);
let definition = solution.get("description")
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::field("definition", primary_language), definition);
writer.add_document(document)?;
counter += 1;
}
Ok(counter)
} else {
unreachable!()
}
}
search/src/schema.rs
+15 -5
View File
@@ -24,7 +24,7 @@ impl Schema {
).set_stored();
let mut schema_builder = TantivySchema::builder();
schema_builder.add_u64_field("type", schema::FAST | schema::INDEXED);
schema_builder.add_u64_field("discriminant", schema::FAST | schema::INDEXED);
schema_builder.add_text_field("iri", schema::STORED | schema::STRING);
schema_builder.add_text_field("curie", stored_ngram32.clone());
@@ -34,12 +34,16 @@ impl Schema {
schema_builder.add_text_field("surname:en", stored_ngram32.clone());
schema_builder.add_text_field("given_name:en", stored_ngram32.clone());
schema_builder.add_u64_field("subtitle_start", schema::STORED);
schema_builder.add_u64_field("subtitle_end", schema::STORED);
schema_builder.add_text_field("subtitle:en", stored_en_stem.clone());
schema_builder.build()
})
}
pub fn type_field() -> Field {
Self::schema().get_field("type").unwrap()
pub fn discriminant_field() -> Field {
Self::schema().get_field("discriminant").unwrap()
}
pub fn iri_field() -> Field {
@@ -50,9 +54,15 @@ impl Schema {
Self::schema().get_field("curie").unwrap()
}
pub fn field(name: &str, language: &str) -> Field {
pub fn field(name: &str, language: Option<&str>) -> Field {
let field_name = if let Some(language) = language {
format!("{name}:{language}")
} else {
name.to_string()
};
Schema::schema()
.get_field(&format!("{name}:{language}"))
.get_field(&field_name)
.expect("Field not found in schema")
}
search/src/subtitles.rs
+21
View File
@@ -0,0 +1,21 @@
use subtitler;
use subtitler::SubtitleFormat;
use tantivy::doc;
use crate::{Schema, SearchDocument};
pub fn bytes_to_documents(data: &[u8]) -> crate::Result<Vec<SearchDocument>> {
let subtitle_start = Schema::field("subtitle_start", None);
let subtitle_end = Schema::field("subtitle_end", None);
let subtitle_content = Schema::field("subtitle", Some("en"));
let subtitle_file = subtitler::parse_bytes(data)?;
let subtitles = subtitle_file.subtitles();
let documents = subtitles.iter().map(|subtitle| {
doc!(
subtitle_start => subtitle.start,
subtitle_end => subtitle.end,
subtitle_content => subtitle.text,
)
}).collect::<Vec<_>>();
Ok(documents)
}