From 72a39e5baaf604db16cd81f1fe0edf2805cecfb07d9484f3fec9bff8cb18d8a7 Mon Sep 17 00:00:00 2001 From: Alex Wied <2+alex@noreply.code.graphofliberty.org> Date: Sun, 16 Aug 2026 15:50:10 -0400 Subject: [PATCH] . --- Cargo.lock | 122 ++++++++++++------------- Cargo.toml | 1 + graph/Cargo.toml | 1 + graph/src/curie.rs | 1 - graph/src/error.rs | 6 ++ graph/src/helpers.rs | 8 -- graph/src/index.rs | 95 -------------------- graph/src/indexer.rs | 175 ++++++++++++++++++++++++++++++++++++ graph/src/language.rs | 1 + graph/src/lib.rs | 5 +- inference/src/service.rs | 12 ++- proto/inference.proto | 8 +- publish/src/error.rs | 6 ++ publish/src/main.rs | 26 ++++-- publish/src/rdf/ontology.rs | 2 +- search/src/lib.rs | 1 + search/src/schema.rs | 2 +- 17 files changed, 290 insertions(+), 182 deletions(-) delete mode 100644 graph/src/index.rs create mode 100644 graph/src/indexer.rs diff --git a/Cargo.lock b/Cargo.lock index 45c6512..d8046ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -640,9 +640,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -984,7 +984,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "cryoglyph" version = "0.1.0" -source = "git+https://github.com/iced-rs/cryoglyph.git?rev=53ba3e879539d19ed8162942126a977ec896cc3b#53ba3e879539d19ed8162942126a977ec896cc3b" +source = "git+https://github.com/iced-rs/cryoglyph.git?rev=f4e7e4eb84dc1d2f335a98c67d8694640d53a433#f4e7e4eb84dc1d2f335a98c67d8694640d53a433" dependencies = [ "cosmic-text", "etagere", @@ -1259,9 +1259,9 @@ dependencies = [ [[package]] name = "error-code" -version = "3.3.2" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" +checksum = "0b5343afd4a8365a643ac588dab4cf234a190c7f6c88c9f6dd6ffe00837661b7" [[package]] name = "etagere" @@ -1337,9 +1337,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "fixedbitset" @@ -1632,6 +1632,7 @@ dependencies = [ "gl-search", "oxigraph", "oxilangtag", + "rayon", "thiserror 2.0.20", "tracing", ] @@ -1997,7 +1998,7 @@ dependencies = [ [[package]] name = "iced" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "iced_core", "iced_debug", @@ -2012,7 +2013,7 @@ dependencies = [ [[package]] name = "iced_core" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "bitflags 2.13.1", "bytes", @@ -2031,7 +2032,7 @@ dependencies = [ [[package]] name = "iced_debug" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "iced_core", "iced_futures", @@ -2041,7 +2042,7 @@ dependencies = [ [[package]] name = "iced_futures" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "futures", "iced_core", @@ -2055,7 +2056,7 @@ dependencies = [ [[package]] name = "iced_graphics" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "bitflags 2.13.1", "bytemuck", @@ -2072,7 +2073,7 @@ dependencies = [ [[package]] name = "iced_program" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "iced_graphics", "iced_runtime", @@ -2081,7 +2082,7 @@ dependencies = [ [[package]] name = "iced_renderer" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "iced_graphics", "iced_tiny_skia", @@ -2093,7 +2094,7 @@ dependencies = [ [[package]] name = "iced_runtime" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "bytes", "iced_core", @@ -2104,7 +2105,7 @@ dependencies = [ [[package]] name = "iced_tiny_skia" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "bytemuck", "cosmic-text", @@ -2120,7 +2121,7 @@ dependencies = [ [[package]] name = "iced_wgpu" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "bitflags 2.13.1", "bytemuck", @@ -2137,7 +2138,7 @@ dependencies = [ [[package]] name = "iced_widget" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "iced_renderer", "iced_runtime", @@ -2151,7 +2152,7 @@ dependencies = [ [[package]] name = "iced_winit" version = "0.15.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4" +source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157" dependencies = [ "arboard", "iced_debug", @@ -2170,9 +2171,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -2184,9 +2185,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -2197,9 +2198,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2211,16 +2212,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -2231,15 +2233,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" dependencies = [ "displaydoc", "icu_locale_core", @@ -2502,9 +2504,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" +checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" dependencies = [ "bitflags 2.13.1", "libc", @@ -2541,9 +2543,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "litrs" @@ -3548,9 +3550,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "plain" @@ -3595,9 +3597,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -3889,9 +3891,9 @@ checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" [[package]] name = "rangemap" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" [[package]] name = "raw-window-handle" @@ -5122,9 +5124,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -5590,9 +5592,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -5734,9 +5736,9 @@ dependencies = [ [[package]] name = "wayland-backend" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016ccf01d1c58b6f8999612813e17c9b2390f7d70671428869913310f83f54b8" +checksum = "38a91b4eaddff87b1cd1074985e3713da4af2c49742d1b356b2c01670a67a078" dependencies = [ "cc", "downcast-rs 1.2.1", @@ -6430,9 +6432,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "x11-dl" @@ -6661,9 +6663,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -6672,9 +6674,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" dependencies = [ "yoke", "zerofrom", @@ -6683,13 +6685,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "47402523226a02bfe5230160dc3ccc089aa6f6f19e7fcbb4e6f824bbb1b4aa62" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -6756,9 +6758,9 @@ dependencies = [ [[package]] name = "zvariant_utils" -version = "4.0.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629d80ece222cad20fe0e8741be493c4ab166acf3b85341bdc2cdbcfd8f3c2d6" +checksum = "6b84ebb462416c27cdb97f2e7f5f0ccc844da1fe2ecc7121e1b690b41318bf42" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index a378090..5356ab6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ parse_link_header = "0.4" pin-project-lite = "0.2" prost = "0.14" rand = "0.10" +rayon = "1.12" rfd = "0.17" slotmap = "1.1" subtitler = "2.6" diff --git a/graph/Cargo.toml b/graph/Cargo.toml index 90871be..60a82db 100644 --- a/graph/Cargo.toml +++ b/graph/Cargo.toml @@ -9,5 +9,6 @@ gl-search.workspace = true oxigraph.workspace = true oxilangtag.workspace = true +rayon.workspace = true thiserror.workspace = true tracing.workspace = true \ No newline at end of file diff --git a/graph/src/curie.rs b/graph/src/curie.rs index 24211ea..3020ba7 100644 --- a/graph/src/curie.rs +++ b/graph/src/curie.rs @@ -1,4 +1,3 @@ -use oxigraph::model::{GraphNameRef, NamedNodeRef}; use std::collections::BTreeMap; use std::sync::LazyLock; diff --git a/graph/src/error.rs b/graph/src/error.rs index ef0a2a9..3e49a4f 100644 --- a/graph/src/error.rs +++ b/graph/src/error.rs @@ -16,6 +16,12 @@ pub enum Error { #[error(transparent)] QueryEvaluation(#[from] oxigraph::sparql::QueryEvaluationError), + #[error(transparent)] + QueryResultsSyntax(#[from] oxigraph::sparql::results::QueryResultsSyntaxError), + #[error(transparent)] UpdateEvaluation(#[from] oxigraph::sparql::UpdateEvaluationError), + + #[error(transparent)] + TonicStatus(#[from] gl_inference::tonic::Status), } diff --git a/graph/src/helpers.rs b/graph/src/helpers.rs index 9a5c6f0..bdd5146 100644 --- a/graph/src/helpers.rs +++ b/graph/src/helpers.rs @@ -10,14 +10,6 @@ pub(crate) fn term_to_named_node(term: &Term) -> Option<&NamedNode> { } } -pub(crate) fn term_ref_as_named_node(term: TermRef<'_>) -> Option> { - if let TermRef::NamedNode(node) = term { - Some(node) - } else { - None - } -} - pub(crate) fn term_as_str(term: &Term) -> Option<&str> { if let Term::Literal(literal) = term { match literal.datatype() { diff --git a/graph/src/index.rs b/graph/src/index.rs deleted file mode 100644 index 5158b1c..0000000 --- a/graph/src/index.rs +++ /dev/null @@ -1,95 +0,0 @@ -use std::collections::HashMap; -use oxigraph::model::NamedNode; -use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsParser, SliceQueryResultsParserOutput}; -use gl_inference::tonic::transport::Channel; -use gl_inference::proto::ontology_client::OntologyClient; -use gl_inference::proto::OntologyQueryRequest; -use gl_search::{Field, OwnedValue, Schema}; -use crate::class::Class; -use crate::{curie, CurieHelper}; -use crate::helpers::{term_as_str, term_to_named_node}; -use crate::language::LanguageCondition; - -pub async fn index_ontology( - client: &mut OntologyClient, - language: &LanguageCondition, -) -> crate::Result>> { - let curie_helper = CurieHelper::new((&*curie::PREFIXES).clone()); - - let label_filter = language.to_filter_expression("label"); - let description_filter = language.to_filter_expression("description"); - - let mut request = OntologyQueryRequest::default(); - request.sparql_query = Some(format!(r#"PREFIX rdf: -PREFIX rdfs: -PREFIX skos: - -SELECT ?class ?subject ?label ?description WHERE {{ - VALUES ?class {{ rdf:Property rdfs:Class skos:Concept }} - ?subject a ?class ; - rdfs:label ?label . - {label_filter} - - OPTIONAL {{ ?subject rdfs:comment ?comment }} - OPTIONAL {{ ?subject skos:definition ?definition }} - BIND(COALESCE(?definition, ?comment) AS ?description) - {description_filter} -}}"#)); - - let response = client.query(request).await.unwrap(); - let parser_output = QueryResultsParser::from_format(QueryResultsFormat::Json) - .for_slice(&response.get_ref().results) - .unwrap(); - - let mut results = Vec::new(); - if let SliceQueryResultsParserOutput::Solutions(solutions) = parser_output { - let primary_language = language.primary_language(); - let label_field = Schema::field("label", primary_language); - let definition_field = Schema::field("definition", primary_language); - for solution in solutions.filter_map(Result::ok) { - let mut document = HashMap::with_capacity(4); - - let discriminant = solution - .get("class") - .and_then(term_to_named_node) - .and_then(Class::try_from_named_node) - .map(|doc_type| doc_type as u64) - .map(OwnedValue::U64) - .unwrap_or(OwnedValue::Null); - document.insert(Schema::discriminant_field(), discriminant); - - let subject_str = solution - .get("subject") - .and_then(term_to_named_node) - .map(NamedNode::as_str); - - let curie = subject_str - .and_then(|subject| curie_helper.abbreviate(None, subject)) - .map(OwnedValue::Str) - .unwrap_or(OwnedValue::Null); - document.insert(Schema::curie_field(), curie); - - let subject = subject_str - .map(OwnedValue::from) - .unwrap_or(OwnedValue::Null); - document.insert(Schema::iri_field(), subject); - - let label = solution - .get("label") - .and_then(term_as_str) - .map(OwnedValue::from) - .unwrap_or(OwnedValue::Null); - document.insert(label_field, label); - - let definition = solution - .get("description") - .and_then(term_as_str) - .map(OwnedValue::from) - .unwrap_or(OwnedValue::Null); - document.insert(definition_field, definition); - - results.push(document); - } - } - Ok(results) -} \ No newline at end of file diff --git a/graph/src/indexer.rs b/graph/src/indexer.rs new file mode 100644 index 0000000..7d86532 --- /dev/null +++ b/graph/src/indexer.rs @@ -0,0 +1,175 @@ +use rayon::iter::ParallelIterator; +use std::collections::{HashMap, HashSet}; +use oxigraph::model::{Graph, NamedNode, TermRef, NamedOrBlankNodeRef, NamedNodeRef}; +use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsParser, SliceQueryResultsParserOutput}; +use rayon::iter::IntoParallelRefIterator; +use gl_inference::tonic::transport::Channel; +use gl_inference::proto::ontology_client::OntologyClient; +use gl_inference::proto::OntologyQueryRequest; +use gl_search::{Field, OwnedValue, Schema}; +use crate::class::Class; +use crate::{helpers, vocab, CurieHelper}; +use crate::helpers::{term_as_str, term_to_named_node}; +use crate::language::LanguageCondition; + +pub struct Indexer<'a> { + language: LanguageCondition, + curie_helper: &'a CurieHelper, +} + +impl<'a> Indexer<'a> { + pub fn new(language: LanguageCondition, curie_helper: &'a CurieHelper) -> Self { + Self { + language, + curie_helper, + } + } + + fn extract_string(&self, graph: &Graph, subject: NamedNodeRef, predicate: NamedNodeRef) -> OwnedValue { + graph.object_for_subject_predicate(subject, predicate) + .filter(|term| self.language.primary_matches_term(*term)) + .and_then(helpers::term_ref_as_str) + .map(String::from) + .map(OwnedValue::Str) + .unwrap_or_else(|| OwnedValue::Null) + } + + fn person(&self, graph: &Graph, subject: NamedNodeRef) -> HashMap { + HashMap::from_iter([ + ( + Schema::field("given_name", self.language.primary_language()), + self.extract_string(graph, subject, vocab::rdaad::GIVEN_NAME), + ), + ( + Schema::field("surname", self.language.primary_language()), + self.extract_string(graph, subject, vocab::rdaad::SURNAME), + ), + ]) + } + + fn corporate_body(&self, graph: &Graph, subject: NamedNodeRef) -> HashMap { + HashMap::from_iter([ + ( + Schema::field("corporate_name", self.language.primary_language()), + self.extract_string(graph, subject, vocab::rdaad::NAME_OF_CORPORATE_BODY), + ), + ]) + } + + pub fn graph(&self, graph: &Graph) -> crate::Result>> { + let mut entities_and_types = HashSet::new(); + let triples = graph.triples_for_predicate(vocab::rdf::TYPE); + for triple in triples { + if let NamedOrBlankNodeRef::NamedNode(subject) = triple.subject && + let TermRef::NamedNode(class) = triple.object { + entities_and_types.insert((subject, class)); + } + } + + let results = entities_and_types.par_iter() + .filter_map(|(subject, class)| { + Class::try_from_named_node(*class).and_then(|class| { + match class { + Class::RdfProperty => None, + Class::RdfsClass => None, + Class::SkosConcept => None, + Class::Work => Some(self.person(graph, *subject)), + Class::Person => Some(self.person(graph, *subject)), + Class::CorporateBody => Some(self.corporate_body(graph, *subject)), + Class::Expression => Some(self.person(graph, *subject)), + Class::Manifestation => Some(self.person(graph, *subject)), + }.map(|mut document| { + document.insert(Schema::discriminant_field(), OwnedValue::from(class as u64)); + document.insert(Schema::iri_field(), OwnedValue::Str(subject.as_str().to_string())); + + let curie = self.curie_helper.abbreviate(None, subject.as_str()) + .map(OwnedValue::Str) + .unwrap_or(OwnedValue::Null); + document.insert(Schema::curie_field(), curie); + + document + }) + }) + }).collect(); + + Ok(results) + } + + pub async fn ontology(&self, client: &mut OntologyClient) -> crate::Result>> { + let label_filter = self.language.to_filter_expression("label"); + let description_filter = self.language.to_filter_expression("description"); + + let mut request = OntologyQueryRequest::default(); + request.sparql_query = Some(format!(r#"PREFIX rdf: +PREFIX rdfs: +PREFIX skos: + +SELECT ?class ?subject ?label ?description WHERE {{ + VALUES ?class {{ rdf:Property rdfs:Class skos:Concept }} + ?subject a ?class ; + rdfs:label ?label . + {label_filter} + + OPTIONAL {{ ?subject rdfs:comment ?comment }} + OPTIONAL {{ ?subject skos:definition ?definition }} + BIND(COALESCE(?definition, ?comment) AS ?description) + {description_filter} +}}"#)); + + let response = client.query(request).await?; + let parser_output = QueryResultsParser::from_format(QueryResultsFormat::Json) + .for_slice(&response.get_ref().results)?; + + let mut results = Vec::new(); + if let SliceQueryResultsParserOutput::Solutions(solutions) = parser_output { + let primary_language = self.language.primary_language(); + let label_field = Schema::field("label", primary_language); + let definition_field = Schema::field("definition", primary_language); + for solution in solutions.filter_map(Result::ok) { + let mut document = HashMap::with_capacity(4); + + let discriminant = solution + .get("class") + .and_then(term_to_named_node) + .and_then(Class::try_from_named_node) + .map(|doc_type| doc_type as u64) + .map(OwnedValue::U64) + .unwrap_or(OwnedValue::Null); + document.insert(Schema::discriminant_field(), discriminant); + + let subject_str = solution + .get("subject") + .and_then(term_to_named_node) + .map(NamedNode::as_str); + + let curie = subject_str + .and_then(|subject| self.curie_helper.abbreviate(None, subject)) + .map(OwnedValue::Str) + .unwrap_or(OwnedValue::Null); + document.insert(Schema::curie_field(), curie); + + let subject = subject_str + .map(OwnedValue::from) + .unwrap_or(OwnedValue::Null); + document.insert(Schema::iri_field(), subject); + + let label = solution + .get("label") + .and_then(term_as_str) + .map(OwnedValue::from) + .unwrap_or(OwnedValue::Null); + document.insert(label_field, label); + + let definition = solution + .get("description") + .and_then(term_as_str) + .map(OwnedValue::from) + .unwrap_or(OwnedValue::Null); + document.insert(definition_field, definition); + + results.push(document); + } + } + Ok(results) + } +} \ No newline at end of file diff --git a/graph/src/language.rs b/graph/src/language.rs index 492ae4b..e236941 100644 --- a/graph/src/language.rs +++ b/graph/src/language.rs @@ -10,6 +10,7 @@ pub static ENGLISH_OR_UNTAGGED: LazyLock = LazyLock::new(|| { ) }); +#[derive(Clone)] pub enum LanguageCondition { ExactMatchOnly(LanguageTag), ExactMatchOrUntagged(LanguageTag), diff --git a/graph/src/lib.rs b/graph/src/lib.rs index 2a25969..dc0d661 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -1,5 +1,4 @@ mod curie; -//mod materialize; mod error; pub mod vocab; @@ -7,7 +6,7 @@ pub mod class; pub mod category; pub mod language; mod helpers; -pub mod index; +pub mod indexer; pub use curie::{CurieHelper, PREFIXES}; -pub use error::{Error, Result}; +pub use error::{Error, Result}; \ No newline at end of file diff --git a/inference/src/service.rs b/inference/src/service.rs index f790198..2f3a4a7 100644 --- a/inference/src/service.rs +++ b/inference/src/service.rs @@ -28,7 +28,7 @@ impl OntologyService { #[tonic::async_trait] impl Ontology for OntologyService { async fn load(&self, request: Request) -> Result, Status> { - let span = debug_span!("Load Ontology", input_size = field::Empty, old_size = field::Empty, new_size = field::Empty).entered(); + let span = debug_span!("Load Ontology", old_size = field::Empty, input_size = field::Empty, inferences = field::Empty, new_size = field::Empty).entered(); let request = request.get_ref(); let mut response = OntologyLoadResponse::default(); @@ -53,13 +53,21 @@ impl Ontology for OntologyService { }); self.ontology.extend(quads).unwrap(); + response.inferences = if request.infer { + crate::logic::infer(0, &self.ontology, ONTOLOGY_GRAPH) + .map_err(|err| Status::internal(err.to_string()))? + .to_u64() + .unwrap_or(u64::MAX) + } else { 0 }; + response.new_size = self.ontology.len() .map_err(|err| Status::internal(err.to_string()))? .to_u64() .unwrap_or(u64::MAX); - span.record("input_size", response.input_size); span.record("old_size", response.old_size); + span.record("input_size", response.input_size); + span.record("inferences", response.inferences); span.record("new_size", response.new_size); Ok(Response::new(response)) diff --git a/proto/inference.proto b/proto/inference.proto index 16af230..490f344 100644 --- a/proto/inference.proto +++ b/proto/inference.proto @@ -5,12 +5,14 @@ import "google/protobuf/empty.proto"; message OntologyLoadRequest { string path = 1; + bool infer = 2; } message OntologyLoadResponse { - uint64 input_size = 1; - uint64 old_size = 2; - uint64 new_size = 3; + uint64 old_size = 1; + uint64 input_size = 2; + uint64 inferences = 3; + uint64 new_size = 4; } message OntologyClearResponse { diff --git a/publish/src/error.rs b/publish/src/error.rs index e85c417..86be1d8 100644 --- a/publish/src/error.rs +++ b/publish/src/error.rs @@ -42,4 +42,10 @@ pub(crate) enum Error { #[error(transparent)] TonicTransport(#[from] tonic::transport::Error), + + #[error(transparent)] + TonicStatus(#[from] tonic::Status), + + #[error(transparent)] + Tantivy(#[from] gl_search::tantivy::TantivyError), } \ No newline at end of file diff --git a/publish/src/main.rs b/publish/src/main.rs index adb7583..4644558 100644 --- a/publish/src/main.rs +++ b/publish/src/main.rs @@ -19,15 +19,14 @@ use ldp::reqwest_middleware::ClientBuilder; use ldp::traverse::Traverse; use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer}; use oxigraph::model::{Dataset, Graph, Triple, TripleRef}; -use tonic::Status; -use tracing::{debug, debug_span, error, field, Instrument}; +use tracing::{debug_span, error, field, Instrument}; use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, fmt}; use url::Url; -use gl_graph::index::index_ontology; -use gl_graph::language; +use gl_graph::indexer::Indexer; +use gl_graph::{language, CurieHelper}; use gl_inference::proto::ontology_client::OntologyClient; use gl_inference::proto::OntologyQueryRequest; @@ -75,7 +74,7 @@ fn main() -> color_eyre::Result<()> { }); } Some(Command::Search(args)) => { - for document in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? { + for document in index.query(args.discriminant, &args.query, Schema::all_fields(), 5000)? { println!("{}", gl_search::to_json(document)); } } @@ -92,8 +91,11 @@ fn main() -> color_eyre::Result<()> { .build()?; runtime.block_on(async { + let curie_helper = CurieHelper::new(gl_graph::PREFIXES.clone()); + let indexer = Indexer::new(language::ENGLISH_OR_UNTAGGED.clone(), &curie_helper); + let mut client = OntologyClient::connect("http://[::1]:3000").await?; - let documents = index_ontology(&mut client, &language::ENGLISH_OR_UNTAGGED).await?; + let documents = indexer.ontology(&mut client).await?; debug_span!("Index Ontology", documents = field::Empty).in_scope(|| { for document in documents { writer.add_document(document).unwrap(); @@ -126,7 +128,10 @@ fn main() -> color_eyre::Result<()> { let span = debug_span!("Index Repository", rdf_sources = rdf_source_count, triples = field::Empty, documents = field::Empty); let triple_count = async { - let mut client = OntologyClient::connect("http://[::1]:3000").await.unwrap(); + let curie_helper = CurieHelper::new(gl_graph::PREFIXES.clone()); + let indexer = Indexer::new(language::ENGLISH_OR_UNTAGGED.clone(), &curie_helper); + + let mut client = OntologyClient::connect("http://[::1]:3000").await?; let mut request = OntologyQueryRequest::default(); let mut output_buffer = Vec::new(); @@ -143,13 +148,18 @@ fn main() -> color_eyre::Result<()> { request.prefixes = HashMap::new(); request.base = None; - let response = client.query(request).await.unwrap(); + let response = client.query(request).await?; let graph_with_inferences = RdfParser::from_format(RdfFormat::Turtle) .for_slice(&response.get_ref().results) .filter_map(Result::ok) .map(Triple::from) .collect::(); + let documents = indexer.graph(&graph_with_inferences)?; + for document in documents { + writer.add_document(document)?; + } + Ok::(graph_with_inferences.len()) }.instrument(span.clone()).await?; span.record("triples", triple_count); diff --git a/publish/src/rdf/ontology.rs b/publish/src/rdf/ontology.rs index f9fcd57..c6e94c7 100644 --- a/publish/src/rdf/ontology.rs +++ b/publish/src/rdf/ontology.rs @@ -6,7 +6,7 @@ use iced::futures::TryFutureExt; use oxigraph::model::vocab::{rdf, rdfs, xsd}; use oxigraph::model::{ Dataset, Graph, GraphNameRef, LiteralRef, NamedNode, NamedNodeRef, NamedOrBlankNode, - NamedOrBlankNodeRef, Term, TermRef, Triple, TripleRef, + NamedOrBlankNodeRef, TermRef, Triple, TripleRef, }; use oxigraph::sparql::{QueryResults, SparqlEvaluator}; use oxigraph::store::Store; diff --git a/search/src/lib.rs b/search/src/lib.rs index 4d18652..de9972a 100644 --- a/search/src/lib.rs +++ b/search/src/lib.rs @@ -2,6 +2,7 @@ mod error; mod index; mod schema; pub mod subtitles; +pub use tantivy; pub use tantivy::indexer::IndexWriter; pub use tantivy::schema::Field; pub use tantivy::schema::document::OwnedValue; diff --git a/search/src/schema.rs b/search/src/schema.rs index 9cf5f1d..f58e22f 100644 --- a/search/src/schema.rs +++ b/search/src/schema.rs @@ -74,4 +74,4 @@ impl Schema { pub fn all_fields() -> Vec { Self::schema().fields().map(|(field, _)| field).collect() } -} +} \ No newline at end of file