gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
alex
2026-08-05 18:33:30 -04:00
parent 7ce5ed118d
commit cd47d868c1
20 changed files with 307 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
publish/src/main.rs
+11 -50
View File
@@ -7,26 +7,20 @@ mod navigator;
mod theme;
mod args;
use std::sync::{Arc, RwLock};
use std::thread;
use clap::Parser;
use color_eyre::eyre::eyre;
use iced::futures::StreamExt;
use iced::Task;
use ldp::middleware::BasicAuthMiddleware;
use ldp::reqwest::{Client, Url};
use ldp::reqwest_middleware::ClientBuilder;
use ldp::traverse::Traverse;
use oxigraph::model::Dataset;
use tracing::{debug, debug_span, error, field};
use crate::app::{Message, Publisher};
use crate::app::Publisher;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, fmt};
use tracing_subscriber::fmt::format::FmtSpan;
use gl_search::{DocType, Document, Schema, SearchIndex};
use gl_search::{Document, Schema, SearchIndex};
use crate::args::{AppArgs, Command};
use crate::rdf::curie::CurieHelper;
use crate::rdf::ontology::Ontology;
fn main() -> color_eyre::Result<()> {
@@ -50,52 +44,25 @@ fn main() -> color_eyre::Result<()> {
let args = AppArgs::parse();
match args.command {
Some(Command::Search(args)) => {
for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 5)? {
for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? {
println!("{}", doc.to_json(Schema::schema()));
}
}
Some(Command::Reindex) => {
let curie_helper = CurieHelper::new(Ontology::prefixes().clone());
let writer = Arc::new(RwLock::new(index.writer()?));
let mut writer = index.writer()?;
debug_span!("Clear Index").in_scope(|| {
let mut writer = writer.write()?;
writer.delete_all_documents()?;
writer.commit()
})?;
let store = ontology.store();
let writer_clone = writer.clone();
let index_classes_thread = thread::spawn(move || {
let span = debug_span!("Index Classes", documents = field::Empty).entered();
let writer = writer_clone.read().unwrap();
let count = gl_search::rdf::index_triples(DocType::RdfsClass, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &*writer)?;
{
let span = debug_span!("Index Schema", documents = field::Empty).entered();
let store = ontology.store();
let count = gl_search::rdf::index_schema(store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
span.record("documents", count);
Ok::<(), gl_search::SearchError>(())
});
let store = ontology.store();
let writer_clone = writer.clone();
let index_properties_thread = thread::spawn(move || {
let span = debug_span!("Index Properties", documents = field::Empty).entered();
let writer = writer_clone.read().unwrap();
let count = gl_search::rdf::index_triples(DocType::RdfProperty, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
span.record("documents", count);
Ok::<(), gl_search::SearchError>(())
});
let store = ontology.store();
let writer_clone = writer.clone();
let index_concepts_thread = thread::spawn(move || {
let span = debug_span!("Index Concepts", documents = field::Empty).entered();
let writer = writer_clone.read().unwrap();
let count = gl_search::rdf::index_triples(DocType::SkosConcept, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
span.record("documents", count);
Ok::<(), gl_search::SearchError>(())
});
}
let starting_url = Url::parse("http://fedora.quill.lan/rest/")?;
let writer_clone = writer.clone();
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.name("repository-traversal")
@@ -114,23 +81,17 @@ fn main() -> color_eyre::Result<()> {
while let Some(result) = traversal.next().await {
match result {
Ok(rdf_source) => {
let writer = writer_clone.read().unwrap();
documents += gl_search::rdf::index_entity(rdf_source.dataset(), &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
}
Err(err) => error!(?err),
}
}
debug!(documents, "Repository Traversal");
Ok::<(), gl_search::SearchError>(())
Ok::<_, gl_search::SearchError>(writer)
});
index_classes_thread.join().unwrap()?;
index_properties_thread.join().unwrap()?;
index_concepts_thread.join().unwrap()?;
runtime.block_on(traversal_task)??;
let mut writer = runtime.block_on(traversal_task)??;
debug_span!("Commit").in_scope(|| {
let mut writer = writer.write()?;
writer.commit()
})?;
}