gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
alex
2026-08-12 14:11:47 -04:00
parent 79894f654d
commit f852da4f4e
48 changed files with 4379 additions and 779 deletions
Download Patch File Download Diff File Expand all files Collapse all files
publish/src/app.rs
+11 -30
View File
@@ -4,15 +4,13 @@ use crate::rdf::term_helper::{TermHelper, TermHelperMut};
use crate::widget::iri_input::iri_input;
use crate::widget::navigation_area::navigation_area;
use gl_graph::CurieHelper;
use gl_graph::inference::InferenceEngine;
use gl_search::language;
use gl_search::{Schema, SearchDocument, SearchIndex, Value};
use gl_graph::language;
use gl_search::{Schema, SearchIndex};
use http::StatusCode;
use iced::alignment::Horizontal;
use iced::keyboard::{Event, key};
use iced::widget::button::Style;
use iced::widget::grid::Sizing;
use iced::widget::text::Wrapping;
use iced::widget::{
button, center, column, combo_box, container, grid, mouse_area, opaque, operation, pick_list,
row, scrollable, space, stack, table, text, text_input, toggler,
@@ -30,7 +28,6 @@ use oxigraph::model::{
BaseDirection, Dataset, Graph, NamedNode, NamedOrBlankNode, NamedOrBlankNodeRef, Quad, Term,
TripleRef,
};
use oxigraph::store::Store;
use tracing::{debug_span, error, trace};
#[derive(Clone, Debug)]
@@ -52,7 +49,7 @@ pub(crate) enum Message {
HoverRow(QuadKey),
UnhoverRow(QuadKey),
QueryUpdated(String),
SetSearchResults(Vec<SearchDocument>),
SetSearchResults(Vec<()>),
QueryTypeUpdated(LabeledIri),
SearchResultClicked(NamedNode),
DatatypeUpdated(QuadKey, Option<String>),
@@ -97,14 +94,13 @@ struct SearchState {
action: SearchResultClickAction,
entity_selection: Option<LabeledIri>,
query: String,
results: Vec<SearchDocument>,
results: Vec<()>,
}
pub(crate) struct Publisher {
http_client: ClientWithMiddleware,
curie_helper: CurieHelper,
ontology: Ontology,
inference_engine: InferenceEngine,
abbreviated_datatypes: Vec<String>,
window_id: window::Id,
url_input: String,
@@ -135,8 +131,6 @@ impl Publisher {
.expect("Failed to build ontology")
});
let inference_engine = InferenceEngine::new(ontology.store().clone());
let index = SearchIndex::builder()
.with_path("/home/alex/.local/share/org.graphofliberty.desktop/index")
.build()
@@ -172,7 +166,6 @@ impl Publisher {
http_client,
curie_helper,
ontology,
inference_engine,
abbreviated_datatypes,
window_id: id,
url_input: starting_url.to_string(),
@@ -359,7 +352,7 @@ impl Publisher {
.index
.query(category_id, query.as_str(), Schema::all_fields(), 25)
.expect("Unable to complete search");
task = Task::done(Message::SetSearchResults(results));
task = Task::done(Message::SetSearchResults(vec![]));
};
}
Message::SetSearchResults(results) => {
@@ -555,19 +548,6 @@ impl Publisher {
self.show_inferred_properties = value;
}
Message::RunInference => {
let dataset = self
.document
.dataset()
.quads
.iter()
.map(|(_, quad)| quad)
.collect();
let engine = self.inference_engine.clone();
task = Task::perform(
tokio::task::spawn_blocking(move || engine.run(&dataset).unwrap()),
|dataset| Message::SetInferredTriples(dataset.unwrap()),
);
}
Message::SetInferredTriples(dataset) => {
self.inferred_triples = Graph::from_iter(&dataset);
@@ -810,8 +790,8 @@ impl Publisher {
)
.on_select(|selection| Message::QueryTypeUpdated(selection));
let mut columns = vec![table::column(text("CURIE"), |document: &SearchDocument| {
let iri = document
let mut columns = vec![table::column(text("CURIE"), |_| {
/*let iri = document
.get_first(Schema::iri_field())
.and_then(|value| value.as_str())
.unwrap_or_default();
@@ -823,7 +803,8 @@ impl Publisher {
button(text(abbreviated_iri).wrapping(Wrapping::Word))
.on_press(Message::SearchResultClicked(NamedNode::new_unchecked(iri)))
.style(button::text)
.style(button::text)*/
text("")
})];
let fields = if let Some(selection) = search_state.entity_selection.as_ref() {
@@ -834,7 +815,7 @@ impl Publisher {
vec![]
};
for field in fields.into_iter() {
/*for field in fields.into_iter() {
let field_name = field.name.clone();
columns.push(
table::column(text(field.label), move |document: &SearchDocument| {
@@ -854,7 +835,7 @@ impl Publisher {
})
.width(Length::Fixed(256.0)),
);
}
}*/
let results_table = if columns.is_empty() {
None
publish/src/args.rs
+1 -1
View File
@@ -4,7 +4,7 @@ use std::path::PathBuf;
#[derive(Args)]
pub(crate) struct QueryArgs {
#[arg(short, long, value_name = "DATASET PATH")]
pub(crate) dataset_path: PathBuf,
pub(crate) dataset_path: Option<PathBuf>,
#[arg(short, long, value_name = "QUERY PATH")]
pub(crate) query_path: PathBuf,
publish/src/error.rs
+4 -1
View File
@@ -19,6 +19,9 @@ pub(crate) enum Error {
#[error(transparent)]
ParseInt(#[from] std::num::ParseIntError),
#[error(transparent)]
Graph(#[from] gl_graph::Error),
#[error(transparent)]
Search(#[from] gl_search::SearchError),
@@ -33,4 +36,4 @@ pub(crate) enum Error {
#[error(transparent)]
Join(#[from] tokio::task::JoinError),
}
}
publish/src/main.rs
+43 -101
View File
@@ -10,24 +10,22 @@ mod windows;
use crate::app::Publisher;
use crate::args::{AppArgs, Command};
use clap::Parser;
use gl_graph::inference::InferenceEngine;
use gl_search::SearchIndex;
use gl_search::{Schema, SearchIndex};
use iced::futures::StreamExt;
use ldp::middleware::BasicAuthMiddleware;
use ldp::reqwest::{Client, Url};
use ldp::reqwest_middleware::ClientBuilder;
use ldp::traverse::Traverse;
use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer};
use oxigraph::model::{Dataset, GraphName, NamedNode, Quad, Triple};
use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsSerializer};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use oxigraph::store::Store;
use std::fs::File;
use oxigraph::model::Dataset;
use tracing::{debug_span, error, field};
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, fmt};
use gl_graph::index::index_ontology;
use gl_graph::language;
use gl_inference::proto::ontology_client::OntologyClient;
use gl_inference::proto::OntologyQueryRequest;
fn main() -> color_eyre::Result<()> {
let appender = tracing_appender::rolling::never("/tmp", "publisher-log");
@@ -49,94 +47,56 @@ fn main() -> color_eyre::Result<()> {
let args = AppArgs::parse();
match args.command {
Some(Command::Query(args)) => {
let store = Store::open("/home/alex/.local/share/org.graphofliberty.desktop/ontology")?;
let inference_engine = InferenceEngine::new(store.clone());
let graph_name = GraphName::NamedNode(NamedNode::new_unchecked(format!(
"file://{}",
args.dataset_path.to_string_lossy()
)));
let dataset = RdfParser::from_format(RdfFormat::Turtle)
.with_default_graph(graph_name)
.for_reader(File::open(&args.dataset_path)?)
.filter_map(Result::ok)
.collect::<Dataset>();
let dataset_with_inferences = inference_engine.run(&dataset)?;
let raw_query = String::from_utf8(std::fs::read(&args.query_path)?)?;
let mut query = SparqlEvaluator::new().parse_query(&raw_query)?;
query.dataset_mut().set_default_graph_as_union();
match query
.on_queryable_dataset(&dataset_with_inferences)
.execute()?
{
QueryResults::Graph(graph) => {
let mut serializer = RdfSerializer::from_format(RdfFormat::Turtle)
.with_prefix("rdfs", "http://www.w3.org/2000/01/rdf-schema#")?
.with_prefix("rdac", "http://rdaregistry.info/Elements/c/")?
.with_prefix("ldp", "http://www.w3.org/ns/ldp#")?
.with_prefix("fedora", "http://fedora.info/definitions/v4/repository#")?
.with_prefix("lrmer", "http://iflastandards.info/ns/lrm/lrmer/")?
.with_prefix("owl", "http://www.w3.org/2002/07/owl#")?
.with_base_iri("http://fedora.quill.lan/rest/")?
.for_writer(std::io::stdout());
for triple in graph.filter_map(Result::ok) {
serializer.serialize_triple(triple.as_ref())?;
}
serializer.finish()?;
}
QueryResults::Solutions(solutions) => {
let json_serializer =
QueryResultsSerializer::from_format(QueryResultsFormat::Json);
let mut writer = json_serializer.serialize_solutions_to_writer(
std::io::stdout(),
Vec::from_iter(solutions.variables().iter().cloned()),
)?;
for solution in solutions.filter_map(Result::ok) {
writer.serialize(&solution)?;
}
writer.finish()?;
}
QueryResults::Boolean(result) => {
let json_serializer =
QueryResultsSerializer::from_format(QueryResultsFormat::Json);
json_serializer.serialize_boolean_to_writer(std::io::stdout(), result)?;
}
}
let graph = if let Some(dataset_path) = &args.dataset_path {
Some(String::from_utf8(std::fs::read(dataset_path)?)?)
} else { None };
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.name("inference-client")
.build()?;
runtime.block_on(async {
let mut client = OntologyClient::connect("http://[::1]:3000").await.unwrap();
let mut request = OntologyQueryRequest::default();
request.sparql_query = raw_query;
request.turtle = graph;
let response = client.query(request).await.unwrap();
println!("{}", response.get_ref().results);
});
}
Some(Command::Search(args)) => {
/*for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? {
println!("{}", doc.to_json(Schema::schema()));
}*/
for document in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? {
println!("{}", gl_search::to_json(document));
}
}
Some(Command::Reindex) => {
let store = Store::open("/home/alex/.local/share/org.graphofliberty.desktop/ontology")?;
let inference_engine = InferenceEngine::new(store.clone());
let mut writer = index.writer()?;
debug_span!("Clear Index").in_scope(|| {
writer.delete_all_documents()?;
writer.commit()
})?;
{
let span = debug_span!("Index Schema", documents = field::Empty).entered();
let count = gl_search::rdf::index_schema(
store,
&*gl_search::language::ENGLISH_OR_UNTAGGED,
&writer,
)?;
span.record("documents", count);
}
let starting_url = Url::parse("http://fedora.quill.lan/rest/")?;
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.name("repository-traversal")
.name("reindex")
.build()?;
let traversal_task = runtime.spawn(async move {
runtime.block_on(async {
let mut client = OntologyClient::connect("http://[::1]:3000").await.unwrap();
let documents = index_ontology(&mut client, &language::ENGLISH_OR_UNTAGGED).await.unwrap();
debug_span!("Index Ontology", documents = field::Empty).in_scope(|| {
for document in documents {
writer.add_document(document).unwrap();
}
});
});
let mut writer = runtime.block_on(async move {
let http_client = ClientBuilder::new(Client::new())
.with(BasicAuthMiddleware::new(
"fedoraAdmin".to_string(),
@@ -144,7 +104,7 @@ fn main() -> color_eyre::Result<()> {
))
.build();
let mut rdf_source_count = 0usize;
let starting_url = Url::parse("http://fedora.quill.lan/rest/").unwrap();
let mut dataset = Dataset::new();
let mut traversal = Traverse::new(http_client, starting_url, None);
while let Some(result) = traversal.next().await {
@@ -152,30 +112,12 @@ fn main() -> color_eyre::Result<()> {
Ok(rdf_source) => dataset.extend(rdf_source.dataset()),
Err(err) => error!(?err),
}
rdf_source_count += 1;
}
let inferences = inference_engine.run(&dataset).unwrap();
dataset.extend(&inferences);
let span = debug_span!(
"Index Repository",
rdf_sources = field::Empty,
documents = field::Empty
)
.entered();
let document_count = gl_search::rdf::index_entity(
&dataset,
&*gl_search::language::ENGLISH_OR_UNTAGGED,
&writer,
)?;
span.record("rdf_sources", rdf_source_count);
span.record("documents", document_count);
let span = debug_span!("Index Repository", rdf_sources = field::Empty, documents = field::Empty).entered();
Ok::<_, gl_search::SearchError>(writer)
});
})?;
let mut writer = runtime.block_on(traversal_task)??;
debug_span!("Commit").in_scope(|| writer.commit())?;
}
None => {
publish/src/rdf/ontology.rs
+7 -46
View File
@@ -1,7 +1,7 @@
use crate::error;
use crate::rdf::conversion;
use gl_graph::language::LanguageCondition;
use gl_graph::vocab::gl;
use gl_search::language::LanguageCondition;
use iced::futures::TryFutureExt;
use oxigraph::model::vocab::{rdf, rdfs, xsd};
use oxigraph::model::{
@@ -89,7 +89,7 @@ impl OntologyBuilder {
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect::<HashMap<String, String>>();
let mut indexed_by = HashMap::new();
/*let mut indexed_by = HashMap::new();
for quad in store
.quads_for_pattern(None, Some(gl::INDEXED_BY_FIELD), None, None)
.filter_map(Result::ok)
@@ -99,7 +99,7 @@ impl OntologyBuilder {
{
indexed_by.insert(subject, field);
}
}
}*/
Ok(Ontology { store, prefixes })
}
@@ -254,12 +254,13 @@ WHERE {{
}
pub fn category_id(&self, class: &NamedNode) -> Option<u64> {
self.store
None
/*self.store
.quads_for_pattern(Some(class.into()), Some(gl::CATEGORY_ID), None, None)
.filter_map(Result::ok)
.map(conversion::quad_into_term)
.filter_map(conversion::term_into_u64)
.next()
.next()*/
}
pub fn fields_for_class(
@@ -392,44 +393,4 @@ WHERE {{
}
}
}
pub fn template_triples<'a>(
&'a self,
class: NamedNodeRef<'a>,
subject: NamedNodeRef<'_>,
) -> Option<impl Iterator<Item = Triple>> {
if let Some(quad) = self
.store
.quads_for_pattern(
Some(NamedOrBlankNodeRef::NamedNode(class)),
Some(gl::TEMPLATE),
None,
None,
)
.filter_map(Result::ok)
.next()
{
if let Term::BlankNode(blank_node) = quad.object {
let iter = self
.store
.quads_for_pattern(
Some(NamedOrBlankNodeRef::BlankNode(blank_node.as_ref())),
None,
None,
None,
)
.filter_map(Result::ok)
.map(Triple::from)
.map(move |mut triple| {
triple.subject = NamedOrBlankNode::NamedNode(subject.into_owned());
triple
});
Some(iter)
} else {
None
}
} else {
None
}
}
}
}