gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
alex
2026-08-07 18:16:03 -04:00
parent cd47d868c1
commit 36a105edfc
14 changed files with 175 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
publish/src/app.rs
+2 -2
View File
@@ -733,10 +733,10 @@ impl Publisher {
text_input("Query", &search_state.query).on_input(Message::QueryUpdated)
.id("query");
let entities = self.ontology
let entities = Vec::new(); /*self.ontology
.searchable_classes(&*language::ENGLISH_OR_UNTAGGED)
.into_iter()
.collect::<Vec<_>>();
.collect::<Vec<_>>();*/
let type_selector = pick_list(search_state.entity_selection.as_ref(), entities, ToString::to_string)
.on_select(|selection| Message::QueryTypeUpdated(selection));
publish/src/args.rs
+11
View File
@@ -1,5 +1,15 @@
use std::path::PathBuf;
use clap::{Args, Parser, Subcommand};
#[derive(Args)]
pub(crate) struct QueryArgs {
#[arg(short, long, value_name = "DATASET PATH")]
pub(crate) dataset_path: PathBuf,
#[arg(short, long, value_name = "QUERY PATH")]
pub(crate) query_path: PathBuf,
}
#[derive(Args)]
pub(crate) struct SearchArgs {
#[arg(short, long, value_name = "DOC TYPE")]
@@ -11,6 +21,7 @@ pub(crate) struct SearchArgs {
#[derive(Subcommand)]
pub(crate) enum Command {
Query(QueryArgs),
Search(SearchArgs),
Reindex,
}
publish/src/main.rs
+63 -13
View File
@@ -7,18 +7,25 @@ mod navigator;
mod theme;
mod args;
use std::fs::File;
use clap::Parser;
use iced::futures::StreamExt;
use ldp::middleware::BasicAuthMiddleware;
use ldp::reqwest::{Client, Url};
use ldp::reqwest_middleware::ClientBuilder;
use ldp::traverse::Traverse;
use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer};
use oxigraph::model::{Dataset, Graph, GraphName, GraphNameRef, NamedNode, Quad, Triple};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsSerializer};
use oxigraph::store::Store;
use tracing::{debug, debug_span, error, field};
use crate::app::Publisher;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, fmt};
use tracing_subscriber::fmt::format::FmtSpan;
use gl_graph::inference::InferenceEngine;
use gl_search::{Document, Schema, SearchIndex};
use crate::args::{AppArgs, Command};
use crate::rdf::ontology::Ontology;
@@ -31,10 +38,8 @@ fn main() -> color_eyre::Result<()> {
.init();
color_eyre::install()?;
let ontology = Ontology::builder()
.with_path("/home/alex/.local/share/org.graphofliberty.desktop/ontology")
.build()
.expect("Failed to build ontology");
let store = Store::open("/home/alex/.local/share/org.graphofliberty.desktop/ontology")?;
let inference_engine = InferenceEngine::new(store.clone());
let mut index = SearchIndex::builder()
.with_path("/home/alex/.local/share/org.graphofliberty.desktop/index")
@@ -43,11 +48,49 @@ fn main() -> color_eyre::Result<()> {
let args = AppArgs::parse();
match args.command {
Some(Command::Search(args)) => {
for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? {
println!("{}", doc.to_json(Schema::schema()));
Some(Command::Query(args)) => {
let graph_name = GraphName::NamedNode(NamedNode::new_unchecked(format!("file://{}", args.dataset_path.to_string_lossy())));
let mut dataset = RdfParser::from_format(RdfFormat::Turtle)
.with_default_graph(graph_name)
.for_reader(File::open(&args.dataset_path)?)
.filter_map(Result::ok)
.collect::<Dataset>();
let inferences = inference_engine.run(&dataset)?;
dataset.extend(&inferences);
let raw_query = String::from_utf8(std::fs::read(&args.query_path)?)?;
let mut query = SparqlEvaluator::new()
.parse_query(&raw_query)?;
query.dataset_mut().set_default_graph_as_union();
match query.on_queryable_dataset(&dataset).execute()? {
QueryResults::Graph(graph) => {
let mut serializer = RdfSerializer::from_format(RdfFormat::Turtle)
.for_writer(std::io::stdout());
for triple in graph.filter_map(Result::ok) {
serializer.serialize_triple(triple.as_ref())?;
}
serializer.finish()?;
}
QueryResults::Solutions(solutions) => {
let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json);
let mut writer = json_serializer.serialize_solutions_to_writer(std::io::stdout(), Vec::from_iter(solutions.variables().iter().cloned()))?;
for solution in solutions.filter_map(Result::ok) {
writer.serialize(&solution)?;
}
writer.finish()?;
}
QueryResults::Boolean(result) => {
let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json);
json_serializer.serialize_boolean_to_writer(std::io::stdout(), result)?;
}
}
}
Some(Command::Search(args)) => {
/*for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? {
println!("{}", doc.to_json(Schema::schema()));
}*/
}
Some(Command::Reindex) => {
let mut writer = index.writer()?;
debug_span!("Clear Index").in_scope(|| {
@@ -57,7 +100,6 @@ fn main() -> color_eyre::Result<()> {
{
let span = debug_span!("Index Schema", documents = field::Empty).entered();
let store = ontology.store();
let count = gl_search::rdf::index_schema(store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
span.record("documents", count);
}
@@ -76,17 +118,25 @@ fn main() -> color_eyre::Result<()> {
))
.build();
let mut documents = 0usize;
let mut rdf_source_count = 0usize;
let mut dataset = Dataset::new();
let mut traversal = Traverse::new(http_client, starting_url, None);
while let Some(result) = traversal.next().await {
match result {
Ok(rdf_source) => {
documents += gl_search::rdf::index_entity(rdf_source.dataset(), &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
}
Ok(rdf_source) => dataset.extend(rdf_source.dataset()),
Err(err) => error!(?err),
}
rdf_source_count += 1;
}
debug!(documents, "Repository Traversal");
let inferences = inference_engine.run(&dataset).unwrap();
dataset.extend(&inferences);
let span = debug_span!("Index Repository", rdf_sources = field::Empty, documents = field::Empty).entered();
let document_count = gl_search::rdf::index_entity(&dataset, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
span.record("rdf_sources", rdf_source_count);
span.record("documents", document_count);
Ok::<_, gl_search::SearchError>(writer)
});
publish/src/rdf/materialize.rs
-115
View File
@@ -1,115 +0,0 @@
use crate::error;
use gl_graph::vocab::owl;
use oxigraph::model::{Dataset, GraphName, NamedNodeRef, NamedOrBlankNode, NamedOrBlankNodeRef, Term};
use oxigraph::sparql::SparqlEvaluator;
use oxigraph::store::Store;
use tracing::{debug, debug_span};
const INFERENCE_GRAPH: NamedNodeRef = NamedNodeRef::new_unchecked("https://graphofliberty.org/inference");
const RDF_SCHEMA: NamedNodeRef = NamedNodeRef::new_unchecked("http://www.w3.org/2000/01/rdf-schema#");
pub fn same_as(store: &mut Store) -> error::Result<()> {
let _span = debug_span!("Materialize owl:sameAs").entered();
let additional_quads = store
.quads_for_pattern(None, Some(owl::SAME_AS), None, None)
.filter_map(Result::ok)
.fold(Dataset::new(), |mut new_dataset, alias| {
if let NamedOrBlankNode::NamedNode(x) = alias.subject
&& let Term::NamedNode(y) = alias.object
{
for mut quad in store.quads_for_pattern(
Some(NamedOrBlankNodeRef::NamedNode(x.as_ref())),
None,
None,
None,
).filter_map(Result::ok) {
quad.subject = NamedOrBlankNode::NamedNode(y.clone());
quad.graph_name = GraphName::NamedNode(INFERENCE_GRAPH.into_owned());
new_dataset.insert(quad.as_ref());
}
for mut quad in store.quads_for_pattern(
Some(NamedOrBlankNodeRef::NamedNode(y.as_ref())),
None,
None,
None,
).filter_map(Result::ok) {
quad.subject = NamedOrBlankNode::NamedNode(x.clone());
quad.graph_name = GraphName::NamedNode(INFERENCE_GRAPH.into_owned());
new_dataset.insert(quad.as_ref());
}
}
new_dataset
});
let old_size = store.len()?;
store.extend(&additional_quads)?;
let new_size = store.len()?;
if new_size > old_size {
let difference = new_size - old_size;
debug!(?old_size, ?new_size, ?difference, "same_as");
same_as(store)?;
}
Ok(())
}
pub fn super_properties(store: &mut Store) -> error::Result<()> {
let _span = debug_span!("Materialize rdfs:subPropertyOf").entered();
let update = SparqlEvaluator::new()
.with_prefix("rdfs", RDF_SCHEMA.as_str())?
.with_prefix("inference", INFERENCE_GRAPH.as_str())?
.parse_update(r#"INSERT {
GRAPH inference: {
?property rdfs:subPropertyOf ?parent .
?subject ?parent ?object .
}
} WHERE {
GRAPH ?g1 { ?property rdfs:subPropertyOf+ ?parent }
OPTIONAL { GRAPH ?g2 { ?subject ?property ?object } }
}"#)?;
let old_size = store.len()?;
update.on_store(&store).execute()?;
let new_size = store.len()?;
if new_size > old_size {
let difference = new_size - old_size;
debug!(?old_size, ?new_size, ?difference, "super_properties");
super_properties(store)?;
}
Ok(())
}
pub fn super_classes(store: &mut Store) -> error::Result<()> {
let _span = debug_span!("Materialize rdfs:subClassOf").entered();
let update = SparqlEvaluator::new()
.with_prefix("rdfs", RDF_SCHEMA.as_str())?
.with_prefix("inference", INFERENCE_GRAPH.as_str())?
.parse_update(r#"INSERT {
GRAPH inference: {
?class rdfs:subClassOf ?parent .
?item a ?parent .
}
} WHERE {
GRAPH ?g1 { ?class rdfs:subClassOf+ ?parent }
OPTIONAL { GRAPH ?g2 { ?item a ?class } }
}"#)?;
let old_size = store.len()?;
update.on_store(&store).execute()?;
let new_size = store.len()?;
if new_size > old_size {
let difference = new_size - old_size;
debug!(?old_size, ?new_size, ?difference, "super_classes");
super_classes(store)?;
}
Ok(())
}
publish/src/rdf/mod.rs
-1
View File
@@ -1,4 +1,3 @@
pub(crate) mod ontology;
pub(crate) mod term_helper;
pub(crate) mod materialize;
pub(crate) mod conversion;
publish/src/rdf/ontology.rs
+1 -27
View File
@@ -1,5 +1,5 @@
use crate::error;
use crate::rdf::{conversion, materialize};
use crate::rdf::conversion;
use gl_graph::vocab::gl;
use gl_search::language::LanguageCondition;
use iced::futures::TryFutureExt;
@@ -91,13 +91,6 @@ impl OntologyBuilder {
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect::<HashMap<String, String>>();
if self.materialize_inferences {
materialize::same_as(&mut store)?;
materialize::super_properties(&mut store)?;
materialize::super_classes(&mut store)?;
debug_span!("Optimize Ontology").in_scope(|| store.optimize())?;
}
let mut indexed_by = HashMap::new();
for quad in store.quads_for_pattern(None, Some(gl::INDEXED_BY_FIELD), None, None)
.filter_map(Result::ok) {
@@ -297,25 +290,6 @@ WHERE {{
}
}
pub fn searchable_classes(&self, language: &LanguageCondition) -> BTreeSet<LabeledIri> {
self.store.quads_for_pattern(None, Some(rdf::TYPE), Some(gl::SEARCHABLE_CLASS.into()), None)
.filter_map(Result::ok)
.filter_map(|quad| {
let label = self.store.quads_for_pattern(Some(quad.subject.as_ref().into()), Some(rdfs::LABEL), None, None)
.filter_map(Result::ok)
.map(conversion::quad_into_term)
.filter(|term| language.primary_matches_term(term))
.filter_map(conversion::term_into_string)
.next();
if let Some(label) = label && let NamedOrBlankNode::NamedNode(subject) = quad.subject {
Some(LabeledIri {
iri: subject,
label,
})
} else { None }
}).collect()
}
pub fn subclasses_of(&self, class: &NamedNode) -> BTreeSet<NamedNode> {
self.store.quads_for_pattern(None, Some(rdfs::SUB_CLASS_OF), Some(class.into()), None)
.filter_map(Result::ok)