gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
alex
2026-08-12 14:11:47 -04:00
parent 79894f654d
commit f852da4f4e
48 changed files with 4379 additions and 779 deletions
Download Patch File Download Diff File Expand all files Collapse all files
publish/src/main.rs
+43 -101
View File
@@ -10,24 +10,22 @@ mod windows;
use crate::app::Publisher;
use crate::args::{AppArgs, Command};
use clap::Parser;
use gl_graph::inference::InferenceEngine;
use gl_search::SearchIndex;
use gl_search::{Schema, SearchIndex};
use iced::futures::StreamExt;
use ldp::middleware::BasicAuthMiddleware;
use ldp::reqwest::{Client, Url};
use ldp::reqwest_middleware::ClientBuilder;
use ldp::traverse::Traverse;
use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer};
use oxigraph::model::{Dataset, GraphName, NamedNode, Quad, Triple};
use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsSerializer};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use oxigraph::store::Store;
use std::fs::File;
use oxigraph::model::Dataset;
use tracing::{debug_span, error, field};
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, fmt};
use gl_graph::index::index_ontology;
use gl_graph::language;
use gl_inference::proto::ontology_client::OntologyClient;
use gl_inference::proto::OntologyQueryRequest;
fn main() -> color_eyre::Result<()> {
let appender = tracing_appender::rolling::never("/tmp", "publisher-log");
@@ -49,94 +47,56 @@ fn main() -> color_eyre::Result<()> {
let args = AppArgs::parse();
match args.command {
Some(Command::Query(args)) => {
let store = Store::open("/home/alex/.local/share/org.graphofliberty.desktop/ontology")?;
let inference_engine = InferenceEngine::new(store.clone());
let graph_name = GraphName::NamedNode(NamedNode::new_unchecked(format!(
"file://{}",
args.dataset_path.to_string_lossy()
)));
let dataset = RdfParser::from_format(RdfFormat::Turtle)
.with_default_graph(graph_name)
.for_reader(File::open(&args.dataset_path)?)
.filter_map(Result::ok)
.collect::<Dataset>();
let dataset_with_inferences = inference_engine.run(&dataset)?;
let raw_query = String::from_utf8(std::fs::read(&args.query_path)?)?;
let mut query = SparqlEvaluator::new().parse_query(&raw_query)?;
query.dataset_mut().set_default_graph_as_union();
match query
.on_queryable_dataset(&dataset_with_inferences)
.execute()?
{
QueryResults::Graph(graph) => {
let mut serializer = RdfSerializer::from_format(RdfFormat::Turtle)
.with_prefix("rdfs", "http://www.w3.org/2000/01/rdf-schema#")?
.with_prefix("rdac", "http://rdaregistry.info/Elements/c/")?
.with_prefix("ldp", "http://www.w3.org/ns/ldp#")?
.with_prefix("fedora", "http://fedora.info/definitions/v4/repository#")?
.with_prefix("lrmer", "http://iflastandards.info/ns/lrm/lrmer/")?
.with_prefix("owl", "http://www.w3.org/2002/07/owl#")?
.with_base_iri("http://fedora.quill.lan/rest/")?
.for_writer(std::io::stdout());
for triple in graph.filter_map(Result::ok) {
serializer.serialize_triple(triple.as_ref())?;
}
serializer.finish()?;
}
QueryResults::Solutions(solutions) => {
let json_serializer =
QueryResultsSerializer::from_format(QueryResultsFormat::Json);
let mut writer = json_serializer.serialize_solutions_to_writer(
std::io::stdout(),
Vec::from_iter(solutions.variables().iter().cloned()),
)?;
for solution in solutions.filter_map(Result::ok) {
writer.serialize(&solution)?;
}
writer.finish()?;
}
QueryResults::Boolean(result) => {
let json_serializer =
QueryResultsSerializer::from_format(QueryResultsFormat::Json);
json_serializer.serialize_boolean_to_writer(std::io::stdout(), result)?;
}
}
let graph = if let Some(dataset_path) = &args.dataset_path {
Some(String::from_utf8(std::fs::read(dataset_path)?)?)
} else { None };
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.name("inference-client")
.build()?;
runtime.block_on(async {
let mut client = OntologyClient::connect("http://[::1]:3000").await.unwrap();
let mut request = OntologyQueryRequest::default();
request.sparql_query = raw_query;
request.turtle = graph;
let response = client.query(request).await.unwrap();
println!("{}", response.get_ref().results);
});
}
Some(Command::Search(args)) => {
/*for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? {
println!("{}", doc.to_json(Schema::schema()));
}*/
for document in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? {
println!("{}", gl_search::to_json(document));
}
}
Some(Command::Reindex) => {
let store = Store::open("/home/alex/.local/share/org.graphofliberty.desktop/ontology")?;
let inference_engine = InferenceEngine::new(store.clone());
let mut writer = index.writer()?;
debug_span!("Clear Index").in_scope(|| {
writer.delete_all_documents()?;
writer.commit()
})?;
{
let span = debug_span!("Index Schema", documents = field::Empty).entered();
let count = gl_search::rdf::index_schema(
store,
&*gl_search::language::ENGLISH_OR_UNTAGGED,
&writer,
)?;
span.record("documents", count);
}
let starting_url = Url::parse("http://fedora.quill.lan/rest/")?;
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.name("repository-traversal")
.name("reindex")
.build()?;
let traversal_task = runtime.spawn(async move {
runtime.block_on(async {
let mut client = OntologyClient::connect("http://[::1]:3000").await.unwrap();
let documents = index_ontology(&mut client, &language::ENGLISH_OR_UNTAGGED).await.unwrap();
debug_span!("Index Ontology", documents = field::Empty).in_scope(|| {
for document in documents {
writer.add_document(document).unwrap();
}
});
});
let mut writer = runtime.block_on(async move {
let http_client = ClientBuilder::new(Client::new())
.with(BasicAuthMiddleware::new(
"fedoraAdmin".to_string(),
@@ -144,7 +104,7 @@ fn main() -> color_eyre::Result<()> {
))
.build();
let mut rdf_source_count = 0usize;
let starting_url = Url::parse("http://fedora.quill.lan/rest/").unwrap();
let mut dataset = Dataset::new();
let mut traversal = Traverse::new(http_client, starting_url, None);
while let Some(result) = traversal.next().await {
@@ -152,30 +112,12 @@ fn main() -> color_eyre::Result<()> {
Ok(rdf_source) => dataset.extend(rdf_source.dataset()),
Err(err) => error!(?err),
}
rdf_source_count += 1;
}
let inferences = inference_engine.run(&dataset).unwrap();
dataset.extend(&inferences);
let span = debug_span!(
"Index Repository",
rdf_sources = field::Empty,
documents = field::Empty
)
.entered();
let document_count = gl_search::rdf::index_entity(
&dataset,
&*gl_search::language::ENGLISH_OR_UNTAGGED,
&writer,
)?;
span.record("rdf_sources", rdf_source_count);
span.record("documents", document_count);
let span = debug_span!("Index Repository", rdf_sources = field::Empty, documents = field::Empty).entered();
Ok::<_, gl_search::SearchError>(writer)
});
})?;
let mut writer = runtime.block_on(traversal_task)??;
debug_span!("Commit").in_scope(|| writer.commit())?;
}
None => {