gl/tools
SHA256
1
0
Fork 0
Code Actions Activity

.

Browse Source
This commit is contained in:
alex
2026-08-05 18:33:30 -04:00
parent 7ce5ed118d
commit cd47d868c1
20 changed files with 307 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+10
View File
@@ -1580,6 +1580,14 @@ version = "0.32.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
[[package]]
name = "gl-graph"
version = "0.1.0"
dependencies = [
"oxigraph",
"tracing",
]
[[package]]
name = "gl-publisher"
version = "0.1.0"
@@ -1587,6 +1595,7 @@ dependencies = [
"clap",
"color-eyre",
"csv",
"gl-graph",
"gl-search",
"http",
"iced",
@@ -1608,6 +1617,7 @@ name = "gl-search"
version = "0.1.0"
dependencies = [
"futures",
"gl-graph",
"oxigraph",
"oxilangtag",
"subtitler",
Cargo.toml
+2
View File
@@ -1,11 +1,13 @@
[workspace]
resolver = "2"
members = [
"graph",
"publish",
"search",
]
[workspace.dependencies]
gl-graph = { path = "graph" }
gl-search = { path = "search" }
ldp = { path = "../../ldp/ldp", features = ["keyed"] }
graph/Cargo.toml
+8
View File
@@ -0,0 +1,8 @@
[package]
name = "gl-graph"
version = "0.1.0"
edition = "2024"
[dependencies]
oxigraph.workspace = true
tracing.workspace = true
graph/src/curie.rs
+87
View File
@@ -0,0 +1,87 @@
use std::collections::BTreeMap;
use std::sync::LazyLock;
use oxigraph::model::{GraphNameRef, NamedNodeRef};
use tracing::debug;
const ONTOLOGY_PREFIX: &str = "https://graphofliberty.org/2026/04/ont/";
const ONTOLOGY_GRAPH_NAME: GraphNameRef = GraphNameRef::NamedNode(NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont"));
pub static PREFIXES: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
BTreeMap::from_iter([
("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"),
("rdfs", "http://www.w3.org/2000/01/rdf-schema#"),
("owl", "http://www.w3.org/2002/07/owl#"),
("xsd", "http://www.w3.org/2001/XMLSchema#"),
("ldp", "http://www.w3.org/ns/ldp#"),
("dc", "http://purl.org/dc/elements/1.1/"),
("posix", "http://www.w3.org/ns/posix/stat#"),
(
"ebucore",
"http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#",
),
("prov", "http://www.w3.org/ns/prov#"),
("locid", "http://id.loc.gov/vocabulary/identifiers/"),
("loclang", "http://id.loc.gov/vocabulary/languages/"),
("premis", "http://www.loc.gov/premis/rdf/v1#"),
("premis3", "http://www.loc.gov/premis/rdf/v3/"),
("dcterms", "http://purl.org/dc/terms/"),
("fedora", "http://fedora.info/definitions/v4/repository#"),
("rdaa", "http://rdaregistry.info/Elements/a/"),
("rdac", "http://rdaregistry.info/Elements/c/"),
("rdae", "http://rdaregistry.info/Elements/e/"),
("rdai", "http://rdaregistry.info/Elements/i/"),
("rdam", "http://rdaregistry.info/Elements/m/"),
("rdan", "http://rdaregistry.info/Elements/n/"),
("rdap", "http://rdaregistry.info/Elements/p/"),
("rdaf", "http://rdaregistry.info/Elements/rof/"),
("rdat", "http://rdaregistry.info/Elements/t/"),
("rdau", "http://rdaregistry.info/Elements/u/"),
("rdaw", "http://rdaregistry.info/Elements/w/"),
("rdax", "http://rdaregistry.info/Elements/x/"),
("rdaco", "http://rdaregistry.info/termList/RDAContentType/"),
("rdact", "http://rdaregistry.info/termList/RDACarrierType/"),
("rdamt", "http://rdaregistry.info/termList/RDAMediaType/"),
("rdaft", "http://rdaregistry.info/termList/fileType/"),
("schema", "https://schema.org/"),
("gl", "http://fedora.quill.lan/rest/"),
("glo", ONTOLOGY_PREFIX),
].map(|(k, v)| (k.to_string(), v.to_string())))
});
#[derive(Clone)]
pub struct CurieHelper {
prefixes: BTreeMap<String, String>,
}
impl CurieHelper {
pub fn new(prefixes: BTreeMap<String, String>) -> Self {
Self {
prefixes,
}
}
pub fn abbreviate(&self, base: Option<&str>, iri: &str) -> Option<String> {
if let Some(base) = base && let Some(local_name) = iri.strip_prefix(base) {
return Some(format!(":{local_name}"));
}
for (name, base) in &self.prefixes {
if let Some(local_name) = iri.strip_prefix(base) {
return Some(format!("{name}:{local_name}"));
}
}
None
}
pub fn expand(&self, base: Option<&str>, abbreviated_iri: &str) -> Option<String> {
let (prefix, name) = abbreviated_iri.split_once(':')?;
if prefix == "" && let Some(base) = base {
Some(format!("{base}{name}"))
} else {
self.prefixes
.get(prefix)
.map(|base| format!("{base}{name}"))
}
}
}
graph/src/lib.rs
+4
View File
@@ -0,0 +1,4 @@
mod curie;
pub mod vocab;
pub use curie::{PREFIXES, CurieHelper};
graph/src/vocab.rs
+73
View File
@@ -0,0 +1,73 @@
pub use oxigraph::model::vocab::rdf;
pub use oxigraph::model::vocab::rdfs;
pub mod gl {
use oxigraph::model::NamedNodeRef;
pub const TEMPLATE: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/template");
pub const INDEXED_BY_FIELD: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/indexedByField");
pub const CATEGORY_ID: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/categoryId");
pub const SEARCHABLE_CLASS: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/SearchableClass");
pub const ENTITY: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/Entity");
pub const ASSOCIATED_PROPERTY: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/associatedProperty");
pub const READ_ONLY: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/readOnly");
}
pub mod skos {
use oxigraph::model::NamedNodeRef;
pub const CONCEPT: NamedNodeRef =
NamedNodeRef::new_unchecked("http://www.w3.org/2004/02/skos/core#Concept");
}
pub mod owl {
use oxigraph::model::NamedNodeRef;
pub const SAME_AS: NamedNodeRef =
NamedNodeRef::new_unchecked("http://www.w3.org/2002/07/owl#sameAs");
}
pub mod rdac {
use oxigraph::model::NamedNodeRef;
pub const PERSON: NamedNodeRef =
NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10004");
pub const CORPORATE_BODY: NamedNodeRef =
NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10005");
pub const WORK: NamedNodeRef =
NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10001");
pub const EXPRESSION: NamedNodeRef =
NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10006");
pub const MANIFESTATION: NamedNodeRef =
NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10007");
}
pub mod rdaad {
use oxigraph::model::NamedNodeRef;
pub const GIVEN_NAME: NamedNodeRef =
NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50292");
pub const SURNAME: NamedNodeRef =
NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50291");
pub const NAME_OF_CORPORATE_BODY: NamedNodeRef =
NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50032");
}
publish/Cargo.toml
+2 -2
View File
@@ -4,15 +4,15 @@ version = "0.1.0"
edition = "2024"
[dependencies]
#gl-ldp.workspace = true
gl-graph.workspace = true
gl-search.workspace = true
ldp.workspace = true
clap.workspace = true
color-eyre.workspace = true
csv = "1.4"
http.workspace = true
iced.workspace = true
ldp.workspace = true
rfd.workspace = true
oxigraph.workspace = true
oxilangtag.workspace = true
publish/src/app.rs
+4 -4
View File
@@ -1,12 +1,11 @@
use crate::navigator::Navigator;
use crate::rdf::curie::CurieHelper;
use crate::rdf::ontology::{LabeledIri, Ontology};
use crate::rdf::term_helper::{TermHelper, TermHelperMut};
use crate::rdf::vocab::rda;
use crate::widget::iri_input::iri_input;
use crate::widget::navigation_area::navigation_area;
use gl_graph::vocab::rdac;
use gl_search::language;
use gl_search::{IndexWriter, Schema, SearchDocument, SearchIndex, Value};
use gl_search::{Schema, SearchDocument, SearchIndex, Value};
use http::StatusCode;
use iced::alignment::Horizontal;
use iced::keyboard::{Event, key};
@@ -25,6 +24,7 @@ use oxigraph::io::RdfFormat;
use oxigraph::model::vocab::{rdf, rdfs};
use oxigraph::model::{BaseDirection, Dataset, NamedNode, NamedOrBlankNode, Quad, Term};
use tracing::{debug_span, error, trace};
use gl_graph::{vocab, CurieHelper};
#[derive(Clone, Debug)]
pub(crate) enum Message {
@@ -816,7 +816,7 @@ impl Publisher {
.collect();
let body: Element<Message> = if self.show_new_document_buttons {
let subclasses = self.ontology.subclasses_of(&rda::ENTITY.into_owned());
let subclasses = Vec::new(); // TODO self.ontology.subclasses_of(vocab::rda::ENTITY.into_owned());
let labeled_subclasses = subclasses.iter().map(|iri| self.ontology.info(iri, &*language::ENGLISH_OR_UNTAGGED));
column![self.view_new_entity_buttons(labeled_subclasses)].into()
} else {
publish/src/main.rs
+11 -50
View File
@@ -7,26 +7,20 @@ mod navigator;
mod theme;
mod args;
use std::sync::{Arc, RwLock};
use std::thread;
use clap::Parser;
use color_eyre::eyre::eyre;
use iced::futures::StreamExt;
use iced::Task;
use ldp::middleware::BasicAuthMiddleware;
use ldp::reqwest::{Client, Url};
use ldp::reqwest_middleware::ClientBuilder;
use ldp::traverse::Traverse;
use oxigraph::model::Dataset;
use tracing::{debug, debug_span, error, field};
use crate::app::{Message, Publisher};
use crate::app::Publisher;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, fmt};
use tracing_subscriber::fmt::format::FmtSpan;
use gl_search::{DocType, Document, Schema, SearchIndex};
use gl_search::{Document, Schema, SearchIndex};
use crate::args::{AppArgs, Command};
use crate::rdf::curie::CurieHelper;
use crate::rdf::ontology::Ontology;
fn main() -> color_eyre::Result<()> {
@@ -50,52 +44,25 @@ fn main() -> color_eyre::Result<()> {
let args = AppArgs::parse();
match args.command {
Some(Command::Search(args)) => {
for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 5)? {
for doc in index.query(args.discriminant, &args.query, Schema::all_fields(), 500000)? {
println!("{}", doc.to_json(Schema::schema()));
}
}
Some(Command::Reindex) => {
let curie_helper = CurieHelper::new(Ontology::prefixes().clone());
let writer = Arc::new(RwLock::new(index.writer()?));
let mut writer = index.writer()?;
debug_span!("Clear Index").in_scope(|| {
let mut writer = writer.write()?;
writer.delete_all_documents()?;
writer.commit()
})?;
let store = ontology.store();
let writer_clone = writer.clone();
let index_classes_thread = thread::spawn(move || {
let span = debug_span!("Index Classes", documents = field::Empty).entered();
let writer = writer_clone.read().unwrap();
let count = gl_search::rdf::index_triples(DocType::RdfsClass, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &*writer)?;
{
let span = debug_span!("Index Schema", documents = field::Empty).entered();
let store = ontology.store();
let count = gl_search::rdf::index_schema(store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
span.record("documents", count);
Ok::<(), gl_search::SearchError>(())
});
let store = ontology.store();
let writer_clone = writer.clone();
let index_properties_thread = thread::spawn(move || {
let span = debug_span!("Index Properties", documents = field::Empty).entered();
let writer = writer_clone.read().unwrap();
let count = gl_search::rdf::index_triples(DocType::RdfProperty, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
span.record("documents", count);
Ok::<(), gl_search::SearchError>(())
});
let store = ontology.store();
let writer_clone = writer.clone();
let index_concepts_thread = thread::spawn(move || {
let span = debug_span!("Index Concepts", documents = field::Empty).entered();
let writer = writer_clone.read().unwrap();
let count = gl_search::rdf::index_triples(DocType::SkosConcept, store, &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
span.record("documents", count);
Ok::<(), gl_search::SearchError>(())
});
}
let starting_url = Url::parse("http://fedora.quill.lan/rest/")?;
let writer_clone = writer.clone();
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.name("repository-traversal")
@@ -114,23 +81,17 @@ fn main() -> color_eyre::Result<()> {
while let Some(result) = traversal.next().await {
match result {
Ok(rdf_source) => {
let writer = writer_clone.read().unwrap();
documents += gl_search::rdf::index_entity(rdf_source.dataset(), &*gl_search::language::ENGLISH_OR_UNTAGGED, &writer)?;
}
Err(err) => error!(?err),
}
}
debug!(documents, "Repository Traversal");
Ok::<(), gl_search::SearchError>(())
Ok::<_, gl_search::SearchError>(writer)
});
index_classes_thread.join().unwrap()?;
index_properties_thread.join().unwrap()?;
index_concepts_thread.join().unwrap()?;
runtime.block_on(traversal_task)??;
let mut writer = runtime.block_on(traversal_task)??;
debug_span!("Commit").in_scope(|| {
let mut writer = writer.write()?;
writer.commit()
})?;
}
publish/src/rdf/curie.rs
-39
View File
@@ -1,39 +0,0 @@
use std::collections::BTreeMap;
#[derive(Clone)]
pub struct CurieHelper {
prefixes: BTreeMap<String, String>,
}
impl CurieHelper {
pub fn new(prefixes: BTreeMap<String, String>) -> Self {
Self {
prefixes,
}
}
pub fn abbreviate(&self, base: Option<&str>, iri: &str) -> Option<String> {
if let Some(base) = base && let Some(local_name) = iri.strip_prefix(base) {
return Some(format!(":{local_name}"));
}
for (name, base) in &self.prefixes {
if let Some(local_name) = iri.strip_prefix(base) {
return Some(format!("{name}:{local_name}"));
}
}
None
}
pub fn expand(&self, base: Option<&str>, abbreviated_iri: &str) -> Option<String> {
let (prefix, name) = abbreviated_iri.split_once(':')?;
if prefix == "" && let Some(base) = base {
Some(format!("{base}{name}"))
} else {
self.prefixes
.get(prefix)
.map(|base| format!("{base}{name}"))
}
}
}
publish/src/rdf/materialize.rs
+2 -2
View File
@@ -1,9 +1,9 @@
use crate::error;
use gl_graph::vocab::owl;
use oxigraph::model::{Dataset, GraphName, NamedNodeRef, NamedOrBlankNode, NamedOrBlankNodeRef, Term};
use oxigraph::sparql::SparqlEvaluator;
use oxigraph::store::Store;
use tracing::{debug, debug_span};
use crate::error;
use crate::rdf::vocab::owl;
const INFERENCE_GRAPH: NamedNodeRef = NamedNodeRef::new_unchecked("https://graphofliberty.org/inference");
const RDF_SCHEMA: NamedNodeRef = NamedNodeRef::new_unchecked("http://www.w3.org/2000/01/rdf-schema#");
publish/src/rdf/mod.rs
-2
View File
@@ -1,6 +1,4 @@
pub(crate) mod ontology;
pub(crate) mod term_helper;
pub mod vocab;
pub(crate) mod materialize;
pub(crate) mod conversion;
pub(crate) mod curie;
publish/src/rdf/ontology.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use crate::error;
use crate::rdf::vocab::gl;
use crate::rdf::{conversion, materialize};
use gl_graph::vocab::gl;
use gl_search::language::LanguageCondition;
use iced::futures::TryFutureExt;
use oxigraph::model::vocab::{rdf, rdfs, xsd};
publish/src/rdf/vocab.rs
-38
View File
@@ -1,38 +0,0 @@
pub mod gl {
use oxigraph::model::NamedNodeRef;
pub const TEMPLATE: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/template");
pub const INDEXED_BY_FIELD: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/indexedByField");
pub const CATEGORY_ID: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/categoryId");
pub const SEARCHABLE_CLASS: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/SearchableClass");
pub const ENTITY: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/Entity");
pub const ASSOCIATED_PROPERTY: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/associatedProperty");
pub const READ_ONLY: NamedNodeRef =
NamedNodeRef::new_unchecked("https://graphofliberty.org/2026/04/ont/readOnly");
}
pub mod owl {
use oxigraph::model::NamedNodeRef;
pub const SAME_AS: NamedNodeRef =
NamedNodeRef::new_unchecked("http://www.w3.org/2002/07/owl#sameAs");
}
pub mod rda {
use oxigraph::model::NamedNodeRef;
pub const ENTITY: NamedNodeRef =
NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/c/C10013");
}
publish/src/widget/iri_input.rs
+2 -2
View File
@@ -1,4 +1,4 @@
use iced::{alignment, keyboard, widget, Element, Event, Length, Rectangle, Size};
use iced::{keyboard, widget, Element, Event, Length, Rectangle, Size};
use iced::advanced::{mouse, text, Shell, renderer};
use iced::advanced::layout::{Limits, Node};
use iced::advanced::{Layout, Widget};
@@ -6,7 +6,7 @@ use iced::advanced::widget::{tree, Tree, Operation};
use iced::clipboard::Content;
use iced::mouse::{Cursor, Interaction};
use iced::widget::text_input::{Catalog, Status, Style, StyleFn};
use crate::rdf::curie::CurieHelper;
use gl_graph::CurieHelper;
pub struct State {
control: bool,
search/Cargo.toml
+2 -1
View File
@@ -5,10 +5,11 @@ edition = "2024"
publish = false
[dependencies]
gl-graph.workspace = true
futures.workspace = true
tantivy.workspace = true
thiserror.workspace = true
#poppler-rs = "0.26.0-alpha.0"
subtitler.workspace = true
tracing.workspace = true
search/src/error.rs
-3
View File
@@ -23,7 +23,4 @@ pub enum SearchError {
#[error(transparent)]
SparqlSyntax(#[from] oxigraph::sparql::SparqlSyntaxError),
#[error("The provided DocType is not valid for this operation")]
InvalidDocType,
}
search/src/lib.rs
+29 -3
View File
@@ -12,6 +12,7 @@ pub use tantivy::schema::document::{Document, Value};
pub use tantivy::indexer::IndexWriter;
pub use error::{Result, SearchError};
use gl_graph::vocab;
pub use index::{SearchIndex, SearchIndexBuilder};
pub use schema::Schema;
@@ -20,12 +21,37 @@ pub enum DocType {
RdfsClass = 1,
SkosConcept = 2,
Person = 3,
CorporateBody = 4,
Work = 5,
Expression = 6,
Manifestation = 7,
}
impl DocType {
pub fn from_named_node(node: NamedNodeRef) -> Option<Self> {
match node.as_str() {
"http://rdaregistry.info/Elements/c/C10004" => Some(DocType::Person),
pub fn to_named_node(&self) -> NamedNodeRef<'_> {
match self {
DocType::RdfProperty => vocab::rdf::PROPERTY,
DocType::RdfsClass => vocab::rdfs::CLASS,
DocType::SkosConcept => vocab::skos::CONCEPT,
DocType::Person => vocab::rdac::PERSON,
DocType::CorporateBody => vocab::rdac::CORPORATE_BODY,
DocType::Work => vocab::rdac::WORK,
DocType::Expression => vocab::rdac::EXPRESSION,
DocType::Manifestation => vocab::rdac::MANIFESTATION,
}
}
pub fn from_named_node<'a>(node: impl Into<NamedNodeRef<'a>>) -> Option<Self> {
let node = node.into();
match node {
vocab::rdf::PROPERTY => Some(DocType::RdfProperty),
vocab::rdfs::CLASS => Some(DocType::RdfsClass),
vocab::skos::CONCEPT => Some(DocType::SkosConcept),
vocab::rdac::PERSON => Some(DocType::Person),
vocab::rdac::CORPORATE_BODY => Some(DocType::CorporateBody),
vocab::rdac::WORK => Some(DocType::Work),
vocab::rdac::EXPRESSION => Some(DocType::Expression),
vocab::rdac::MANIFESTATION => Some(DocType::Manifestation),
_ => None,
}
}
search/src/rdf.rs
+68 -33
View File
@@ -5,10 +5,11 @@ use oxigraph::sparql::{QueryResults, SparqlEvaluator};
use oxigraph::store::Store;
use tantivy::IndexWriter;
use tantivy::schema::{Field, OwnedValue};
use tracing::debug;
use gl_graph::{vocab, CurieHelper};
use crate::DocType;
use crate::language::LanguageCondition;
use crate::schema::Schema;
use crate::SearchError::InvalidDocType;
fn term_to_named_node(term: &Term) -> Option<&NamedNode> {
if let Term::NamedNode(node) = term {
@@ -52,16 +53,15 @@ fn term_ref_as_str(term: TermRef<'_>) -> Option<&str> {
}
}
const GIVEN_NAME: NamedNodeRef = NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50292");
const SURNAME: NamedNodeRef = NamedNodeRef::new_unchecked("http://rdaregistry.info/Elements/a/datatype/P50291");
pub fn index_entity(dataset: &Dataset, language: &LanguageCondition, writer: &IndexWriter<HashMap<Field, OwnedValue>>) -> crate::Result<usize> {
let curie_helper = CurieHelper::new((&*gl_graph::PREFIXES).clone());
let mut subject_type_map = HashMap::new();
for quad in dataset.quads_for_pattern(None, Some(rdf::TYPE), None, None) {
if let NamedOrBlankNodeRef::NamedNode(subject) = quad.subject {
subject_type_map.entry(subject)
.and_modify(|types: &mut Vec<_>| types.push(quad.object))
.or_insert_with(Vec::new);
.or_insert_with(|| vec![quad.object]);
}
}
@@ -69,16 +69,24 @@ pub fn index_entity(dataset: &Dataset, language: &LanguageCondition, writer: &In
for (subject, types) in subject_type_map {
for type_ in types {
if let Some(doc_type) = term_ref_as_named_node(type_).and_then(DocType::from_named_node) {
match doc_type {
DocType::Person => {
let mut doc = index_person(dataset, subject, language);
doc.insert(Schema::discriminant_field(), OwnedValue::U64(doc_type as u64));
doc.insert(Schema::iri_field(), OwnedValue::Str(subject.as_str().to_string()));
writer.add_document(doc)?;
counter += 1;
}
_ => {}
}
let mut doc = match doc_type {
DocType::Person => index_person(dataset, subject, language),
DocType::CorporateBody => index_corporate_body(dataset, subject, language),
_ => continue
};
let subject_str = subject.as_str();
doc.insert(Schema::discriminant_field(), OwnedValue::U64(doc_type as u64));
doc.insert(Schema::iri_field(), OwnedValue::Str(subject_str.to_string()));
let curie = curie_helper.abbreviate(None, subject_str)
.map(OwnedValue::Str)
.unwrap_or(OwnedValue::Null);
doc.insert(Schema::curie_field(), curie);
writer.add_document(doc)?;
counter += 1;
}
}
}
@@ -87,7 +95,7 @@ pub fn index_entity(dataset: &Dataset, language: &LanguageCondition, writer: &In
}
fn index_person(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &LanguageCondition) -> HashMap<Field, OwnedValue> {
let given_name = dataset.quads_for_pattern(Some(subject.into()), Some(GIVEN_NAME), None, None)
let given_name = dataset.quads_for_pattern(Some(subject.into()), Some(vocab::rdaad::GIVEN_NAME), None, None)
.map(|q| q.object)
.filter(|term| language.primary_matches_term(*term))
.filter_map(term_ref_as_str)
@@ -96,7 +104,7 @@ fn index_person(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &Languag
.next()
.unwrap_or_else(|| OwnedValue::Null);
let surname = dataset.quads_for_pattern(Some(subject.into()), Some(SURNAME), None, None)
let surname = dataset.quads_for_pattern(Some(subject.into()), Some(vocab::rdaad::SURNAME), None, None)
.map(|q| q.object)
.filter(|term| language.primary_matches_term(*term))
.filter_map(term_ref_as_str)
@@ -111,18 +119,30 @@ fn index_person(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &Languag
])
}
pub fn index_triples(doc_type: DocType, store: Store, language: &LanguageCondition, writer: &IndexWriter<HashMap<Field, OwnedValue>>) -> crate::Result<usize> {
fn index_corporate_body(dataset: &Dataset, subject: NamedNodeRef<'_>, language: &LanguageCondition) -> HashMap<Field, OwnedValue> {
let name = dataset.quads_for_pattern(Some(subject.into()), Some(vocab::rdaad::NAME_OF_CORPORATE_BODY), None, None)
.map(|q| q.object)
.filter(|term| language.primary_matches_term(*term))
.filter_map(term_ref_as_str)
.map(String::from)
.map(OwnedValue::Str)
.next()
.unwrap_or_else(|| OwnedValue::Null);
HashMap::from_iter([
(Schema::field("corporate_name", language.primary_language()), name),
])
}
pub fn index_schema(store: Store, language: &LanguageCondition, writer: &IndexWriter<HashMap<Field, OwnedValue>>) -> crate::Result<usize> {
let curie_helper = CurieHelper::new((&*gl_graph::PREFIXES).clone());
let label_filter = language.to_filter_expression("label");
let description_filter = language.to_filter_expression("description");
let rdf_type = match doc_type {
DocType::RdfsClass => Ok("rdfs:Class"),
DocType::RdfProperty => Ok("rdf:Property"),
DocType::SkosConcept => Ok("skos:Concept"),
_ => Err(InvalidDocType),
}?;
let query = format!(r#"SELECT ?subject ?label ?description WHERE {{
?subject a {rdf_type} ;
let query = format!(r#"SELECT ?class ?subject ?label ?description WHERE {{
VALUES ?class {{ rdf:Property rdfs:Class skos:Concept }}
?subject a ?class ;
rdfs:label ?label .
{label_filter}
@@ -141,17 +161,32 @@ pub fn index_triples(doc_type: DocType, store: Store, language: &LanguageConditi
let mut counter = 0usize;
let query_results = sparql.on_store(&store).execute()?;
if let QueryResults::Solutions(solutions) = query_results {
let discriminant = OwnedValue::U64(doc_type as u64);
let primary_language = language.primary_language();
let label_field = Schema::field("label", primary_language);
let definition_field = Schema::field("definition", primary_language);
for solution in solutions.filter_map(Result::ok) {
let mut document = HashMap::with_capacity(4);
document.insert(Schema::discriminant_field(), discriminant.clone());
let subject = solution.get("subject")
let discriminant = solution.get("class")
.and_then(term_to_named_node)
.map(NamedNode::as_str)
.map(OwnedValue::from)
.and_then(DocType::from_named_node)
.map(|doc_type| doc_type as u64)
.map(OwnedValue::U64)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::discriminant_field(), discriminant);
let subject_str = solution.get("subject")
.and_then(term_to_named_node)
.map(NamedNode::as_str);
let curie = subject_str
.and_then(|subject| curie_helper.abbreviate(None, subject))
.map(OwnedValue::Str)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::curie_field(), curie);
let subject = subject_str.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::iri_field(), subject);
@@ -159,13 +194,13 @@ pub fn index_triples(doc_type: DocType, store: Store, language: &LanguageConditi
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::field("label", primary_language), label);
document.insert(label_field, label);
let definition = solution.get("description")
.and_then(term_as_str)
.map(OwnedValue::from)
.unwrap_or(OwnedValue::Null);
document.insert(Schema::field("definition", primary_language), definition);
document.insert(definition_field, definition);
writer.add_document(document)?;
counter += 1;
search/src/schema.rs
+1
View File
@@ -33,6 +33,7 @@ impl Schema {
schema_builder.add_text_field("surname:en", stored_ngram32.clone());
schema_builder.add_text_field("given_name:en", stored_ngram32.clone());
schema_builder.add_text_field("corporate_name:en", stored_ngram32.clone());
schema_builder.add_u64_field("subtitle_start", schema::STORED);
schema_builder.add_u64_field("subtitle_end", schema::STORED);