tools/publish/src/main.rs

mod app;
mod error;
mod rdf;
mod windows;
mod widget;
mod navigator;
mod theme;
mod args;

use std::fs::File;
use clap::Parser;
use iced::futures::StreamExt;
use ldp::middleware::BasicAuthMiddleware;
use ldp::reqwest::{Client, Url};
use ldp::reqwest_middleware::ClientBuilder;
use ldp::traverse::Traverse;
use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer};
use oxigraph::model::{Dataset, Graph, GraphName, GraphNameRef, NamedNode, Quad, Triple};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsSerializer};
use oxigraph::store::Store;
use tracing::{debug, debug_span, error, field};
use crate::app::Publisher;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, fmt};
use tracing_subscriber::fmt::format::FmtSpan;
use gl_graph::inference::InferenceEngine;
use gl_search::{Document, Schema, SearchIndex};
use crate::args::{AppArgs, Command};
use crate::rdf::ontology::Ontology;

fn main() -> color_eyre::Result<()> {
    let appender = tracing_appender::rolling::never("/tmp", "publisher-log");
    tracing_subscriber::registry()
        .with(fmt::layer().with_span_events(FmtSpan::CLOSE).with_writer(appender))
        .with(EnvFilter::from_default_env())
        .init();
    color_eyre::install()?;

    let store = Store::open("/home/alex/.local/share/org.graphofliberty.desktop/ontology")?;
    let inference_engine = InferenceEngine::new(store.clone());

    let mut index = SearchIndex::builder()
        .with_path("/home/alex/.local/share/org.graphofliberty.desktop/index")
        .build()
        .expect("Failed to build search index");

    let args = AppArgs::parse();
    match args.command {
        Some(Command::Query(args)) => {
            let graph_name = GraphName::NamedNode(NamedNode::new_unchecked(format!("file://{}", args.dataset_path.to_string_lossy())));
            let mut dataset = RdfParser::from_format(RdfFormat::Turtle)
                .with_default_graph(graph_name)
                .for_reader(File::open(&args.dataset_path)?)
                .filter_map(Result::ok)
                .collect::<Dataset>();
            let inferences = inference_engine.run(&dataset)?;
            dataset.extend(&inferences);

            let raw_query = String::from_utf8(std::fs::read(&args.query_path)?)?;
            let mut query = SparqlEvaluator::new()
                .parse_query(&raw_query)?;
            query.dataset_mut().set_default_graph_as_union();

            match query.on_queryable_dataset(&dataset).execute()? {
                QueryResults::Graph(graph) => {
                    let mut serializer = RdfSerializer::from_format(RdfFormat::Turtle)
                        .for_writer(std::io::stdout());
                    for triple in graph.filter_map(Result::ok) {
                        serializer.serialize_triple(triple.as_ref())?;
                    }
                    serializer.finish()?;
                }
                QueryResults::Solutions(solutions) => {
                    let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json);
                    let mut writer = json_serializer.serialize_solutions_to_writer(std::io::stdout(), Vec::from_iter(solutions.variables().iter().cloned()))?;
                    for solution in solutions.filter_map(Result::ok) {
                        writer.serialize(&solution)?;
                    }
                    writer.finish()?;
                }
                QueryResults::Boolean(result) => {
                    let json_serializer = QueryResultsSerializer::from_format(QueryResultsFormat::Json);
                    json_serializer.serialize_boolean_to_writer(std::io::stdout(), result)?;
                }
            }
        }
        Some(Command::Search(args)) => {
            /*for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? {
                println!("{}", doc.to_json(Schema::schema()));
            }*/
        }
        Some(Command::Reindex) => {
            let mut writer = index.writer()?;
            debug_span!("Clear Index").in_scope(|| {
                writer.delete_all_documents()?;
                writer.commit()
            })?;

            {
                let span = debug_span!("Index Schema", documents = field::Empty).entered();
                let count = gl_search::rdf::index_schema(store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
                span.record("documents", count);
            }

            let starting_url = Url::parse("http://fedora.quill.lan/rest/")?;
            let runtime = tokio::runtime::Builder::new_multi_thread()
                .enable_all()
                .name("repository-traversal")
                .build()?;

            let traversal_task = runtime.spawn(async move {
                let http_client = ClientBuilder::new(Client::new())
                    .with(BasicAuthMiddleware::new(
                        "fedoraAdmin".to_string(),
                        Some("fedoraAdmin".to_string()),
                    ))
                    .build();

                let mut rdf_source_count = 0usize;
                let mut dataset = Dataset::new();
                let mut traversal = Traverse::new(http_client, starting_url, None);
                while let Some(result) = traversal.next().await {
                    match result {
                        Ok(rdf_source) => dataset.extend(rdf_source.dataset()),
                        Err(err) => error!(?err),
                    }
                    rdf_source_count += 1;
                }

                let inferences = inference_engine.run(&dataset).unwrap();
                dataset.extend(&inferences);

                let span = debug_span!("Index Repository", rdf_sources = field::Empty, documents = field::Empty).entered();
                let document_count = gl_search::rdf::index_entity(&dataset, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
                span.record("rdf_sources", rdf_source_count);
                span.record("documents", document_count);

                Ok::<_, gl_search::SearchError>(writer)
            });

            let mut writer = runtime.block_on(traversal_task)??;
            debug_span!("Commit").in_scope(|| {
                writer.commit()
            })?;
        }
        None => {
            iced::daemon(Publisher::new, Publisher::update, Publisher::view)
                .title(Publisher::title)
                .subscription(Publisher::subscription)
                .run()?;
        }
    }

    Ok(())
}