gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
alex
2026-08-16 15:50:10 -04:00
parent 8dcaa5b61f
commit 72a39e5baa
17 changed files with 290 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+62 -60
View File
@@ -640,9 +640,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.4.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e"
checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -984,7 +984,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "cryoglyph"
version = "0.1.0"
source = "git+https://github.com/iced-rs/cryoglyph.git?rev=53ba3e879539d19ed8162942126a977ec896cc3b#53ba3e879539d19ed8162942126a977ec896cc3b"
source = "git+https://github.com/iced-rs/cryoglyph.git?rev=f4e7e4eb84dc1d2f335a98c67d8694640d53a433#f4e7e4eb84dc1d2f335a98c67d8694640d53a433"
dependencies = [
"cosmic-text",
"etagere",
@@ -1259,9 +1259,9 @@ dependencies = [
[[package]]
name = "error-code"
version = "3.3.2"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59"
checksum = "0b5343afd4a8365a643ac588dab4cf234a190c7f6c88c9f6dd6ffe00837661b7"
[[package]]
name = "etagere"
@@ -1337,9 +1337,9 @@ dependencies = [
[[package]]
name = "find-msvc-tools"
version = "0.1.10"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de"
checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890"
[[package]]
name = "fixedbitset"
@@ -1632,6 +1632,7 @@ dependencies = [
"gl-search",
"oxigraph",
"oxilangtag",
"rayon",
"thiserror 2.0.20",
"tracing",
]
@@ -1997,7 +1998,7 @@ dependencies = [
[[package]]
name = "iced"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"iced_core",
"iced_debug",
@@ -2012,7 +2013,7 @@ dependencies = [
[[package]]
name = "iced_core"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"bitflags 2.13.1",
"bytes",
@@ -2031,7 +2032,7 @@ dependencies = [
[[package]]
name = "iced_debug"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"iced_core",
"iced_futures",
@@ -2041,7 +2042,7 @@ dependencies = [
[[package]]
name = "iced_futures"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"futures",
"iced_core",
@@ -2055,7 +2056,7 @@ dependencies = [
[[package]]
name = "iced_graphics"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"bitflags 2.13.1",
"bytemuck",
@@ -2072,7 +2073,7 @@ dependencies = [
[[package]]
name = "iced_program"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"iced_graphics",
"iced_runtime",
@@ -2081,7 +2082,7 @@ dependencies = [
[[package]]
name = "iced_renderer"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"iced_graphics",
"iced_tiny_skia",
@@ -2093,7 +2094,7 @@ dependencies = [
[[package]]
name = "iced_runtime"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"bytes",
"iced_core",
@@ -2104,7 +2105,7 @@ dependencies = [
[[package]]
name = "iced_tiny_skia"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"bytemuck",
"cosmic-text",
@@ -2120,7 +2121,7 @@ dependencies = [
[[package]]
name = "iced_wgpu"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"bitflags 2.13.1",
"bytemuck",
@@ -2137,7 +2138,7 @@ dependencies = [
[[package]]
name = "iced_widget"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"iced_renderer",
"iced_runtime",
@@ -2151,7 +2152,7 @@ dependencies = [
[[package]]
name = "iced_winit"
version = "0.15.0-dev"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2b275718d19a5cf306537e1d2417a1f5e9d94ef4"
source = "git+https://github.com/iced-rs/iced.git?branch=master#2cffa99b395d84fe469b44dccb56bbacd2f1a157"
dependencies = [
"arboard",
"iced_debug",
@@ -2170,9 +2171,9 @@ dependencies = [
[[package]]
name = "icu_collections"
version = "2.2.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"
dependencies = [
"displaydoc",
"potential_utf",
@@ -2184,9 +2185,9 @@ dependencies = [
[[package]]
name = "icu_locale_core"
version = "2.2.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
dependencies = [
"displaydoc",
"litemap",
@@ -2197,9 +2198,9 @@ dependencies = [
[[package]]
name = "icu_normalizer"
version = "2.2.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f"
dependencies = [
"icu_collections",
"icu_normalizer_data",
@@ -2211,16 +2212,17 @@ dependencies = [
[[package]]
name = "icu_normalizer_data"
version = "2.2.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0"
[[package]]
name = "icu_properties"
version = "2.2.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148"
dependencies = [
"displaydoc",
"icu_collections",
"icu_locale_core",
"icu_properties_data",
@@ -2231,15 +2233,15 @@ dependencies = [
[[package]]
name = "icu_properties_data"
version = "2.2.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
[[package]]
name = "icu_provider"
version = "2.2.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
dependencies = [
"displaydoc",
"icu_locale_core",
@@ -2502,9 +2504,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libredox"
version = "0.1.19"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa"
checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a"
dependencies = [
"bitflags 2.13.1",
"libc",
@@ -2541,9 +2543,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "litemap"
version = "0.8.2"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
[[package]]
name = "litrs"
@@ -3548,9 +3550,9 @@ dependencies = [
[[package]]
name = "pkg-config"
version = "0.3.33"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
[[package]]
name = "plain"
@@ -3595,9 +3597,9 @@ dependencies = [
[[package]]
name = "potential_utf"
version = "0.1.5"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
dependencies = [
"zerovec",
]
@@ -3889,9 +3891,9 @@ checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08"
[[package]]
name = "rangemap"
version = "1.7.1"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d"
[[package]]
name = "raw-window-handle"
@@ -5122,9 +5124,9 @@ dependencies = [
[[package]]
name = "tinystr"
version = "0.8.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
dependencies = [
"displaydoc",
"zerovec",
@@ -5590,9 +5592,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.24.0"
version = "1.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
dependencies = [
"getrandom 0.4.3",
"js-sys",
@@ -5734,9 +5736,9 @@ dependencies = [
[[package]]
name = "wayland-backend"
version = "0.3.16"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "016ccf01d1c58b6f8999612813e17c9b2390f7d70671428869913310f83f54b8"
checksum = "38a91b4eaddff87b1cd1074985e3713da4af2c49742d1b356b2c01670a67a078"
dependencies = [
"cc",
"downcast-rs 1.2.1",
@@ -6430,9 +6432,9 @@ dependencies = [
[[package]]
name = "writeable"
version = "0.6.3"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
[[package]]
name = "x11-dl"
@@ -6661,9 +6663,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
[[package]]
name = "zerotrie"
version = "0.2.4"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
dependencies = [
"displaydoc",
"yoke",
@@ -6672,9 +6674,9 @@ dependencies = [
[[package]]
name = "zerovec"
version = "0.11.6"
version = "0.11.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8"
dependencies = [
"yoke",
"zerofrom",
@@ -6683,13 +6685,13 @@ dependencies = [
[[package]]
name = "zerovec-derive"
version = "0.11.3"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
checksum = "47402523226a02bfe5230160dc3ccc089aa6f6f19e7fcbb4e6f824bbb1b4aa62"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"syn 3.0.3",
]
[[package]]
@@ -6756,9 +6758,9 @@ dependencies = [
[[package]]
name = "zvariant_utils"
version = "4.0.0"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "629d80ece222cad20fe0e8741be493c4ab166acf3b85341bdc2cdbcfd8f3c2d6"
checksum = "6b84ebb462416c27cdb97f2e7f5f0ccc844da1fe2ecc7121e1b690b41318bf42"
dependencies = [
"proc-macro2",
"quote",
Cargo.toml
+1
View File
@@ -28,6 +28,7 @@ parse_link_header = "0.4"
pin-project-lite = "0.2"
prost = "0.14"
rand = "0.10"
rayon = "1.12"
rfd = "0.17"
slotmap = "1.1"
subtitler = "2.6"
graph/Cargo.toml
+1
View File
@@ -9,5 +9,6 @@ gl-search.workspace = true
oxigraph.workspace = true
oxilangtag.workspace = true
rayon.workspace = true
thiserror.workspace = true
tracing.workspace = true
graph/src/curie.rs
-1
View File
@@ -1,4 +1,3 @@
use oxigraph::model::{GraphNameRef, NamedNodeRef};
use std::collections::BTreeMap;
use std::sync::LazyLock;
graph/src/error.rs
+6
View File
@@ -16,6 +16,12 @@ pub enum Error {
#[error(transparent)]
QueryEvaluation(#[from] oxigraph::sparql::QueryEvaluationError),
#[error(transparent)]
QueryResultsSyntax(#[from] oxigraph::sparql::results::QueryResultsSyntaxError),
#[error(transparent)]
UpdateEvaluation(#[from] oxigraph::sparql::UpdateEvaluationError),
#[error(transparent)]
TonicStatus(#[from] gl_inference::tonic::Status),
}
graph/src/helpers.rs
-8
View File
@@ -10,14 +10,6 @@ pub(crate) fn term_to_named_node(term: &Term) -> Option<&NamedNode> {
}
}
pub(crate) fn term_ref_as_named_node(term: TermRef<'_>) -> Option<NamedNodeRef<'_>> {
if let TermRef::NamedNode(node) = term {
Some(node)
} else {
None
}
}
pub(crate) fn term_as_str(term: &Term) -> Option<&str> {
if let Term::Literal(literal) = term {
match literal.datatype() {
graph/src/index.rs
-95
View File
@@ -1,95 +0,0 @@
use std::collections::HashMap;
use oxigraph::model::NamedNode;
use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsParser, SliceQueryResultsParserOutput};
use gl_inference::tonic::transport::Channel;
use gl_inference::proto::ontology_client::OntologyClient;
use gl_inference::proto::OntologyQueryRequest;
use gl_search::{Field, OwnedValue, Schema};
use crate::class::Class;
use crate::{curie, CurieHelper};
use crate::helpers::{term_as_str, term_to_named_node};
use crate::language::LanguageCondition;
pub async fn index_ontology(
client: &mut OntologyClient<Channel>,
language: &LanguageCondition,
) -> crate::Result<Vec<HashMap<Field, OwnedValue>>> {
let curie_helper = CurieHelper::new((&*curie::PREFIXES).clone());
let label_filter = language.to_filter_expression("label");
let description_filter = language.to_filter_expression("description");
let mut request = OntologyQueryRequest::default();
request.sparql_query = Some(format!(r#"PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
SELECT ?class ?subject ?label ?description WHERE {{
VALUES ?class {{ rdf:Property rdfs:Class skos:Concept }}
?subject a ?class ;
rdfs:label ?label .
{label_filter}
OPTIONAL {{ ?subject rdfs:comment ?comment }}
OPTIONAL {{ ?subject skos:definition ?definition }}
BIND(COALESCE(?definition, ?comment) AS ?description)
{description_filter}
}}"#));
let response = client.query(request).await.unwrap();
let parser_output = QueryResultsParser::from_format(QueryResultsFormat::Json)
.for_slice(&response.get_ref().results)
.unwrap();
let mut results = Vec::new();
if let SliceQueryResultsParserOutput::Solutions(solutions) = parser_output {
let primary_language = language.primary_language();
let label_field = Schema::field("label", primary_language);
let definition_field = Schema::field("definition", primary_language);
for solution in solutions.filter_map(Result::ok) {
let mut document = HashMap::with_capacity(4);
let discriminant = solution
.get("class")
.and_then(term_to_named_node)
.and_then(Class::try_from_named_node)
.map(|doc_type| doc_type as u64)
.map(OwnedValue::U64)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::discriminant_field(), discriminant);
let subject_str = solution
.get("subject")
.and_then(term_to_named_node)
.map(NamedNode::as_str);
let curie = subject_str
.and_then(|subject| curie_helper.abbreviate(None, subject))
.map(OwnedValue::Str)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::curie_field(), curie);
let subject = subject_str
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::iri_field(), subject);
let label = solution
.get("label")
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(label_field, label);
let definition = solution
.get("description")
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(definition_field, definition);
results.push(document);
}
}
Ok(results)
}
graph/src/indexer.rs
+175
View File
@@ -0,0 +1,175 @@
use rayon::iter::ParallelIterator;
use std::collections::{HashMap, HashSet};
use oxigraph::model::{Graph, NamedNode, TermRef, NamedOrBlankNodeRef, NamedNodeRef};
use oxigraph::sparql::results::{QueryResultsFormat, QueryResultsParser, SliceQueryResultsParserOutput};
use rayon::iter::IntoParallelRefIterator;
use gl_inference::tonic::transport::Channel;
use gl_inference::proto::ontology_client::OntologyClient;
use gl_inference::proto::OntologyQueryRequest;
use gl_search::{Field, OwnedValue, Schema};
use crate::class::Class;
use crate::{helpers, vocab, CurieHelper};
use crate::helpers::{term_as_str, term_to_named_node};
use crate::language::LanguageCondition;
pub struct Indexer<'a> {
language: LanguageCondition,
curie_helper: &'a CurieHelper,
}
impl<'a> Indexer<'a> {
pub fn new(language: LanguageCondition, curie_helper: &'a CurieHelper) -> Self {
Self {
language,
curie_helper,
}
}
fn extract_string(&self, graph: &Graph, subject: NamedNodeRef, predicate: NamedNodeRef) -> OwnedValue {
graph.object_for_subject_predicate(subject, predicate)
.filter(|term| self.language.primary_matches_term(*term))
.and_then(helpers::term_ref_as_str)
.map(String::from)
.map(OwnedValue::Str)
.unwrap_or_else(|| OwnedValue::Null)
}
fn person(&self, graph: &Graph, subject: NamedNodeRef) -> HashMap<Field, OwnedValue> {
HashMap::from_iter([
(
Schema::field("given_name", self.language.primary_language()),
self.extract_string(graph, subject, vocab::rdaad::GIVEN_NAME),
),
(
Schema::field("surname", self.language.primary_language()),
self.extract_string(graph, subject, vocab::rdaad::SURNAME),
),
])
}
fn corporate_body(&self, graph: &Graph, subject: NamedNodeRef) -> HashMap<Field, OwnedValue> {
HashMap::from_iter([
(
Schema::field("corporate_name", self.language.primary_language()),
self.extract_string(graph, subject, vocab::rdaad::NAME_OF_CORPORATE_BODY),
),
])
}
pub fn graph(&self, graph: &Graph) -> crate::Result<Vec<HashMap<Field, OwnedValue>>> {
let mut entities_and_types = HashSet::new();
let triples = graph.triples_for_predicate(vocab::rdf::TYPE);
for triple in triples {
if let NamedOrBlankNodeRef::NamedNode(subject) = triple.subject &&
let TermRef::NamedNode(class) = triple.object {
entities_and_types.insert((subject, class));
}
}
let results = entities_and_types.par_iter()
.filter_map(|(subject, class)| {
Class::try_from_named_node(*class).and_then(|class| {
match class {
Class::RdfProperty => None,
Class::RdfsClass => None,
Class::SkosConcept => None,
Class::Work => Some(self.person(graph, *subject)),
Class::Person => Some(self.person(graph, *subject)),
Class::CorporateBody => Some(self.corporate_body(graph, *subject)),
Class::Expression => Some(self.person(graph, *subject)),
Class::Manifestation => Some(self.person(graph, *subject)),
}.map(|mut document| {
document.insert(Schema::discriminant_field(), OwnedValue::from(class as u64));
document.insert(Schema::iri_field(), OwnedValue::Str(subject.as_str().to_string()));
let curie = self.curie_helper.abbreviate(None, subject.as_str())
.map(OwnedValue::Str)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::curie_field(), curie);
document
})
})
}).collect();
Ok(results)
}
pub async fn ontology(&self, client: &mut OntologyClient<Channel>) -> crate::Result<Vec<HashMap<Field, OwnedValue>>> {
let label_filter = self.language.to_filter_expression("label");
let description_filter = self.language.to_filter_expression("description");
let mut request = OntologyQueryRequest::default();
request.sparql_query = Some(format!(r#"PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
SELECT ?class ?subject ?label ?description WHERE {{
VALUES ?class {{ rdf:Property rdfs:Class skos:Concept }}
?subject a ?class ;
rdfs:label ?label .
{label_filter}
OPTIONAL {{ ?subject rdfs:comment ?comment }}
OPTIONAL {{ ?subject skos:definition ?definition }}
BIND(COALESCE(?definition, ?comment) AS ?description)
{description_filter}
}}"#));
let response = client.query(request).await?;
let parser_output = QueryResultsParser::from_format(QueryResultsFormat::Json)
.for_slice(&response.get_ref().results)?;
let mut results = Vec::new();
if let SliceQueryResultsParserOutput::Solutions(solutions) = parser_output {
let primary_language = self.language.primary_language();
let label_field = Schema::field("label", primary_language);
let definition_field = Schema::field("definition", primary_language);
for solution in solutions.filter_map(Result::ok) {
let mut document = HashMap::with_capacity(4);
let discriminant = solution
.get("class")
.and_then(term_to_named_node)
.and_then(Class::try_from_named_node)
.map(|doc_type| doc_type as u64)
.map(OwnedValue::U64)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::discriminant_field(), discriminant);
let subject_str = solution
.get("subject")
.and_then(term_to_named_node)
.map(NamedNode::as_str);
let curie = subject_str
.and_then(|subject| self.curie_helper.abbreviate(None, subject))
.map(OwnedValue::Str)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::curie_field(), curie);
let subject = subject_str
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::iri_field(), subject);
let label = solution
.get("label")
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(label_field, label);
let definition = solution
.get("description")
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(definition_field, definition);
results.push(document);
}
}
Ok(results)
}
}
graph/src/language.rs
+1
View File
@@ -10,6 +10,7 @@ pub static ENGLISH_OR_UNTAGGED: LazyLock<LanguageCondition> = LazyLock::new(|| {
)
});
#[derive(Clone)]
pub enum LanguageCondition {
ExactMatchOnly(LanguageTag<String>),
ExactMatchOrUntagged(LanguageTag<String>),
graph/src/lib.rs
+2 -3
View File
@@ -1,5 +1,4 @@
mod curie;
//mod materialize;
mod error;
pub mod vocab;
@@ -7,7 +6,7 @@ pub mod class;
pub mod category;
pub mod language;
mod helpers;
pub mod index;
pub mod indexer;
pub use curie::{CurieHelper, PREFIXES};
pub use error::{Error, Result};
pub use error::{Error, Result};
inference/src/service.rs
+10 -2
View File
@@ -28,7 +28,7 @@ impl OntologyService {
#[tonic::async_trait]
impl Ontology for OntologyService {
async fn load(&self, request: Request<OntologyLoadRequest>) -> Result<Response<OntologyLoadResponse>, Status> {
let span = debug_span!("Load Ontology", input_size = field::Empty, old_size = field::Empty, new_size = field::Empty).entered();
let span = debug_span!("Load Ontology", old_size = field::Empty, input_size = field::Empty, inferences = field::Empty, new_size = field::Empty).entered();
let request = request.get_ref();
let mut response = OntologyLoadResponse::default();
@@ -53,13 +53,21 @@ impl Ontology for OntologyService {
});
self.ontology.extend(quads).unwrap();
response.inferences = if request.infer {
crate::logic::infer(0, &self.ontology, ONTOLOGY_GRAPH)
.map_err(|err| Status::internal(err.to_string()))?
.to_u64()
.unwrap_or(u64::MAX)
} else { 0 };
response.new_size = self.ontology.len()
.map_err(|err| Status::internal(err.to_string()))?
.to_u64()
.unwrap_or(u64::MAX);
span.record("input_size", response.input_size);
span.record("old_size", response.old_size);
span.record("input_size", response.input_size);
span.record("inferences", response.inferences);
span.record("new_size", response.new_size);
Ok(Response::new(response))
proto/inference.proto
+5 -3
View File
@@ -5,12 +5,14 @@ import "google/protobuf/empty.proto";
message OntologyLoadRequest {
string path = 1;
bool infer = 2;
}
message OntologyLoadResponse {
uint64 input_size = 1;
uint64 old_size = 2;
uint64 new_size = 3;
uint64 old_size = 1;
uint64 input_size = 2;
uint64 inferences = 3;
uint64 new_size = 4;
}
message OntologyClearResponse {
publish/src/error.rs
+6
View File
@@ -42,4 +42,10 @@ pub(crate) enum Error {
#[error(transparent)]
TonicTransport(#[from] tonic::transport::Error),
#[error(transparent)]
TonicStatus(#[from] tonic::Status),
#[error(transparent)]
Tantivy(#[from] gl_search::tantivy::TantivyError),
}
publish/src/main.rs
+18 -8
View File
@@ -19,15 +19,14 @@ use ldp::reqwest_middleware::ClientBuilder;
use ldp::traverse::Traverse;
use oxigraph::io::{RdfFormat, RdfParser, RdfSerializer};
use oxigraph::model::{Dataset, Graph, Triple, TripleRef};
use tonic::Status;
use tracing::{debug, debug_span, error, field, Instrument};
use tracing::{debug_span, error, field, Instrument};
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, fmt};
use url::Url;
use gl_graph::index::index_ontology;
use gl_graph::language;
use gl_graph::indexer::Indexer;
use gl_graph::{language, CurieHelper};
use gl_inference::proto::ontology_client::OntologyClient;
use gl_inference::proto::OntologyQueryRequest;
@@ -75,7 +74,7 @@ fn main() -> color_eyre::Result<()> {
});
}
Some(Command::Search(args)) => {
for document in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? {
for document in index.query(args.discriminant, &args.query, Schema::all_fields(), 5000)? {
println!("{}", gl_search::to_json(document));
}
}
@@ -92,8 +91,11 @@ fn main() -> color_eyre::Result<()> {
.build()?;
runtime.block_on(async {
let curie_helper = CurieHelper::new(gl_graph::PREFIXES.clone());
let indexer = Indexer::new(language::ENGLISH_OR_UNTAGGED.clone(), &curie_helper);
let mut client = OntologyClient::connect("http://[::1]:3000").await?;
let documents = index_ontology(&mut client, &language::ENGLISH_OR_UNTAGGED).await?;
let documents = indexer.ontology(&mut client).await?;
debug_span!("Index Ontology", documents = field::Empty).in_scope(|| {
for document in documents {
writer.add_document(document).unwrap();
@@ -126,7 +128,10 @@ fn main() -> color_eyre::Result<()> {
let span = debug_span!("Index Repository", rdf_sources = rdf_source_count, triples = field::Empty, documents = field::Empty);
let triple_count = async {
let mut client = OntologyClient::connect("http://[::1]:3000").await.unwrap();
let curie_helper = CurieHelper::new(gl_graph::PREFIXES.clone());
let indexer = Indexer::new(language::ENGLISH_OR_UNTAGGED.clone(), &curie_helper);
let mut client = OntologyClient::connect("http://[::1]:3000").await?;
let mut request = OntologyQueryRequest::default();
let mut output_buffer = Vec::new();
@@ -143,13 +148,18 @@ fn main() -> color_eyre::Result<()> {
request.prefixes = HashMap::new();
request.base = None;
let response = client.query(request).await.unwrap();
let response = client.query(request).await?;
let graph_with_inferences = RdfParser::from_format(RdfFormat::Turtle)
.for_slice(&response.get_ref().results)
.filter_map(Result::ok)
.map(Triple::from)
.collect::<Graph>();
let documents = indexer.graph(&graph_with_inferences)?;
for document in documents {
writer.add_document(document)?;
}
Ok::<usize, crate::error::Error>(graph_with_inferences.len())
}.instrument(span.clone()).await?;
span.record("triples", triple_count);
publish/src/rdf/ontology.rs
+1 -1
View File
@@ -6,7 +6,7 @@ use iced::futures::TryFutureExt;
use oxigraph::model::vocab::{rdf, rdfs, xsd};
use oxigraph::model::{
Dataset, Graph, GraphNameRef, LiteralRef, NamedNode, NamedNodeRef, NamedOrBlankNode,
NamedOrBlankNodeRef, Term, TermRef, Triple, TripleRef,
NamedOrBlankNodeRef, TermRef, Triple, TripleRef,
};
use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use oxigraph::store::Store;
search/src/lib.rs
+1
View File
@@ -2,6 +2,7 @@ mod error;
mod index;
mod schema;
pub mod subtitles;
pub use tantivy;
pub use tantivy::indexer::IndexWriter;
pub use tantivy::schema::Field;
pub use tantivy::schema::document::OwnedValue;
search/src/schema.rs
+1 -1
View File
@@ -74,4 +74,4 @@ impl Schema {
pub fn all_fields() -> Vec<Field> {
Self::schema().fields().map(|(field, _)| field).collect()
}
}
}