tools/graph/src/indexer.rs

use rayon::iter::ParallelIterator;
use std::collections::{HashMap, HashSet};
use oxigraph::model::{Graph, NamedNode, TermRef, NamedOrBlankNodeRef, NamedNodeRef};
use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsParser, SliceQueryResultsParserOutput};
use rayon::iter::IntoParallelRefIterator;
use gl_inference::tonic::transport::Channel;
use gl_inference::proto::ontology_client::OntologyClient;
use gl_inference::proto::OntologyQueryRequest;
use gl_search::{Field, OwnedValue, Schema};
use crate::class::Class;
use crate::{helpers, vocab, CurieHelper};
use crate::helpers::{term_as_str, term_to_named_node};
use crate::language::LanguageCondition;

pub struct Indexer<'a> {
    language: LanguageCondition,
    curie_helper: &'a CurieHelper,
}

impl<'a> Indexer<'a> {
    pub fn new(language: LanguageCondition, curie_helper: &'a CurieHelper) -> Self {
        Self {
            language,
            curie_helper,
        }
    }

    fn extract_string(&self, graph: &Graph, subject: NamedNodeRef, predicate: NamedNodeRef) -> OwnedValue {
        graph.object_for_subject_predicate(subject, predicate)
            .filter(|term| self.language.primary_matches_term(*term))
            .and_then(helpers::term_ref_as_str)
            .map(String::from)
            .map(OwnedValue::Str)
            .unwrap_or_else(|| OwnedValue::Null)
    }

    fn person(&self, graph: &Graph, subject: NamedNodeRef) -> HashMap<Field, OwnedValue> {
        HashMap::from_iter([
            (
                Schema::field("given_name", self.language.primary_language()),
                self.extract_string(graph, subject, vocab::rdaad::GIVEN_NAME),
            ),
            (
                Schema::field("surname", self.language.primary_language()),
                self.extract_string(graph, subject, vocab::rdaad::SURNAME),
            ),
        ])
    }

    fn corporate_body(&self, graph: &Graph, subject: NamedNodeRef) -> HashMap<Field, OwnedValue> {
        HashMap::from_iter([
            (
                Schema::field("corporate_name", self.language.primary_language()),
                self.extract_string(graph, subject, vocab::rdaad::NAME_OF_CORPORATE_BODY),
            ),
        ])
    }

    fn work(&self, graph: &Graph, subject: NamedNodeRef) -> HashMap<Field, OwnedValue> {
        HashMap::from_iter([
            (
                Schema::field("title", self.language.primary_language()),
                self.extract_string(graph, subject, vocab::rdawd::TITLE_OF_WORK),
            ),
        ])
    }

    fn expression(&self, graph: &Graph, subject: NamedNodeRef) -> HashMap<Field, OwnedValue> {
        HashMap::from_iter([
            (
                Schema::field("title", self.language.primary_language()),
                self.extract_string(graph, subject, vocab::rdaed::TITLE_OF_EXPRESSION),
            ),
        ])
    }

    pub fn graph(&self, graph: &Graph) -> crate::Result<Vec<HashMap<Field, OwnedValue>>> {
        let mut entities_and_types = HashSet::new();
        let triples = graph.triples_for_predicate(vocab::rdf::TYPE);
        for triple in triples {
            if let NamedOrBlankNodeRef::NamedNode(subject) = triple.subject &&
                let TermRef::NamedNode(class) = triple.object {
                entities_and_types.insert((subject, class));
            }
        }

        let results = entities_and_types.par_iter()
            .filter_map(|(subject, class)| {
                Class::try_from_named_node(*class).and_then(|class| {
                    match class {
                        Class::RdfProperty => None,
                        Class::RdfsClass => None,
                        Class::SkosConcept => None,
                        Class::Work => Some(self.work(graph, *subject)),
                        Class::Person => Some(self.person(graph, *subject)),
                        Class::CorporateBody => Some(self.corporate_body(graph, *subject)),
                        Class::Expression => Some(self.expression(graph, *subject)),
                        Class::Manifestation => Some(self.person(graph, *subject)),
                    }.map(|mut document| {
                        document.insert(Schema::discriminant_field(), OwnedValue::from(class as u64));
                        document.insert(Schema::iri_field(), OwnedValue::Str(subject.as_str().to_string()));

                        let curie = self.curie_helper.abbreviate(None, subject.as_str())
                            .map(OwnedValue::Str)
                            .unwrap_or(OwnedValue::Null);
                        document.insert(Schema::curie_field(), curie);

                        document
                    })
                })
            }).collect();

        Ok(results)
    }

    pub async fn ontology(&self, client: &mut OntologyClient<Channel>) -> crate::Result<Vec<HashMap<Field, OwnedValue>>> {
        let label_filter = self.language.to_filter_expression("label");
        let description_filter = self.language.to_filter_expression("description");

        let mut request = OntologyQueryRequest::default();
        request.sparql_query = Some(format!(r#"PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>

SELECT ?class ?subject ?label ?description WHERE {{
    VALUES ?class {{ rdf:Property rdfs:Class skos:Concept }}
    ?subject a ?class ;
    	rdfs:label ?label .
    {label_filter}

    OPTIONAL {{ ?subject rdfs:comment ?comment }}
    OPTIONAL {{ ?subject skos:definition ?definition }}
    BIND(COALESCE(?definition, ?comment) AS ?description)
    {description_filter}
}}"#));

        let response = client.query(request).await?;
        let parser_output = QueryResultsParser::from_format(QueryResultsFormat::Json)
            .for_slice(&response.get_ref().results)?;

        let mut results = Vec::new();
        if let SliceQueryResultsParserOutput::Solutions(solutions) = parser_output {
            let primary_language = self.language.primary_language();
            let label_field = Schema::field("label", primary_language);
            let definition_field = Schema::field("definition", primary_language);
            for solution in solutions.filter_map(Result::ok) {
                let mut document = HashMap::with_capacity(4);

                let discriminant = solution
                    .get("class")
                    .and_then(term_to_named_node)
                    .and_then(Class::try_from_named_node)
                    .map(|doc_type| doc_type as u64)
                    .map(OwnedValue::U64)
                    .unwrap_or(OwnedValue::Null);
                document.insert(Schema::discriminant_field(), discriminant);

                let subject_str = solution
                    .get("subject")
                    .and_then(term_to_named_node)
                    .map(NamedNode::as_str);

                let curie = subject_str
                    .and_then(|subject| self.curie_helper.abbreviate(None, subject))
                    .map(OwnedValue::Str)
                    .unwrap_or(OwnedValue::Null);
                document.insert(Schema::curie_field(), curie);

                let subject = subject_str
                    .map(OwnedValue::from)
                    .unwrap_or(OwnedValue::Null);
                document.insert(Schema::iri_field(), subject);

                let label = solution
                    .get("label")
                    .and_then(term_as_str)
                    .map(OwnedValue::from)
                    .unwrap_or(OwnedValue::Null);
                document.insert(label_field, label);

                let definition = solution
                    .get("description")
                    .and_then(term_as_str)
                    .map(OwnedValue::from)
                    .unwrap_or(OwnedValue::Null);
                document.insert(definition_field, definition);

                results.push(document);
            }
        }
        Ok(results)
    }
}