Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions contexts/aggregator-description.jsonld
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
"@id": "aggr:tokenExpiry",
"@type": "xsd:dateTime"
},
"transformation_catalog": {
"@id": "aggr:transformationCatalog",
"deployment_catalog": {
"@id": "aggr:deploymentCatalog",
"@type": "@id"
},
"service_collection_endpoint": {
Expand Down
4 changes: 2 additions & 2 deletions contexts/aggregator-server-description.jsonld
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@
"@id": "aggr:clientIdentifier",
"@type": "@id"
},
"transformation_catalog": {
"@id": "aggr:transformationCatalog",
"deployment_catalog": {
"@id": "aggr:deploymentCatalog",
"@type": "@id"
}
}
Expand Down
25 changes: 4 additions & 21 deletions contexts/aggregator.jsonld
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
"@context": {
"aggr": "https://w3id.org/aggregator#",
"fno": "https://w3id.org/function/ontology#",
"fnoc": "https://fno.io/vocabulary/composition/0.1.0/",
"dcat": "http://www.w3.org/ns/dcat#",
"dct": "http://purl.org/dc/terms/",
"prov": "http://www.w3.org/ns/prov#",
Expand All @@ -25,25 +24,21 @@
"Service": "aggr:Service",
"ServiceRequest": "aggr:ServiceRequest",
"ServiceCollection": "aggr:ServiceCollection",
"TransformationCatalog": "aggr:TransformationCatalog",
"DeploymentCatalog": "aggr:DeploymentCatalog",
"ProvenanceLog": "aggr:ProvenanceLog",

"hasService": { "@id": "aggr:hasService", "@type": "@id" },
"hasTransformation": { "@id": "aggr:hasTransformation", "@type": "@id" },
"hasAppliedFunction": { "@id": "aggr:hasAppliedFunction", "@type": "@id" },
"hasDeploymentFunction": { "@id": "aggr:hasDeploymentFunction", "@type": "@id" },
"hasAlgorithm": { "@id": "aggr:hasAlgorithm", "@type": "@id" },
"hasMapping": { "@id": "aggr:hasMapping", "@type": "@id" },
"hasComposition": { "@id": "aggr:hasComposition", "@type": "@id" },
"hasActivity": { "@id": "aggr:hasActivity", "@type": "@id" },
"performs": { "@id": "aggr:performs", "@type": "@id" },
"applies": { "@id": "aggr:applies", "@type": "@id" },
"deploymentFunction": { "@id": "aggr:deploymentFunction", "@type": "@id" },
"implements": { "@id": "aggr:implements", "@type": "@id" },
"forOutput": { "@id": "aggr:forOutput", "@type": "@id" },
"fromAggregator": { "@id": "aggr:fromAggregator", "@type": "@id" },
"fromServer": { "@id": "aggr:fromServer", "@type": "@id" },
"fromService": { "@id": "aggr:fromService", "@type": "@id" },
"provenanceLog": { "@id": "aggr:provenanceLog", "@type": "@id" },
"transformationCatalog": { "@id": "aggr:transformationCatalog", "@type": "@id" },
"deploymentCatalog": { "@id": "aggr:deploymentCatalog", "@type": "@id" },
"status": "aggr:status",
"statusDetail": "aggr:statusDetail",
"createdAt": { "@id": "aggr:createdAt", "@type": "xsd:dateTime" },
Expand All @@ -57,18 +52,6 @@
"executes": { "@id": "fno:executes", "@type": "@id" },
"uses": { "@id": "fno:uses", "@type": "@id" },

"applies_function": { "@id": "fnoc:applies", "@type": "@id" },
"parameterBindings": { "@id": "fnoc:parameterBindings", "@type": "@id", "@container": "@list" },
"boundParameter": { "@id": "fnoc:boundParameter", "@type": "@id" },
"boundToTerm": "fnoc:boundToTerm",
"composition": { "@id": "fnoc:composition", "@type": "@id" },
"mapFrom": "fnoc:mapFrom",
"mapTo": "fnoc:mapTo",
"constituentFunction": { "@id": "fnoc:constituentFunction", "@type": "@id" },
"functionParameter": { "@id": "fnoc:functionParameter", "@type": "@id" },
"functionOutput": { "@id": "fnoc:functionOutput", "@type": "@id" },
"composedOf": "fnoc:composedOf",

"title": "dct:title",
"description": "dct:description",
"format": { "@id": "dct:format", "@type": "@id" },
Expand Down
977 changes: 207 additions & 770 deletions dist/index.html

Large diffs are not rendered by default.

822 changes: 147 additions & 675 deletions dist/spec.bs

Large diffs are not rendered by default.

244 changes: 244 additions & 0 deletions provenance-detailed-draft.bs
Original file line number Diff line number Diff line change
@@ -0,0 +1,244 @@
Provenance Draft
================

Data-model draft
----------------

Activity {#data-model-service-activity}
---------------------------------------

An Activity represents service output provenance information. Each activity:
- **MUST** be a `prov:Activity`.
- **MUST** be associated with the service using `prov:associatedWith`.
- **MUST** include a start time using `prov:startedAtTime`.
- **MAY** include an end time using `prov:endedAtTime`.

The activity **MAY** provide additional metadata such as `prov:qualifiedAssociation`, or any other relevant provenance
information. An activity **MAY** additionally be described as an `fno:Execution` when FnO accurately describes the
service's activity, but this specification does not require that modeling choice.

The generated dataset produced by an activity either *is* the logical dataset served by the
service, or it is a distinct version of that logical dataset, depending on whether the service
retains version history.

In either case, the generated dataset:
- **MUST** be typed as a `dcat:Dataset`.
- **MUST** be typed as a `prov:Entity`.
- **MUST** link to the activity using `prov:wasGeneratedBy`.
- **SHOULD** include `prov:wasGeneratedAtTime`.

If the service retains version history, the generated dataset additionally:
- **MUST** specify it is a version of the service dataset using `dcat:isVersionOf`.

If the service retains version history, the logical dataset:
- **MUST** declare the latest known version using `dcat:hasCurrentVersion`.
- **MUST** declare other known versions using `dcat:hasVersion`.

The following example shows an activity that produces a new version of a service dataset.

<div class="example">
```turtle
@base <https://aggregator.example.org/agg1/services/sparql-query-view-service/provenance#> .
@prefix svc: <https://aggregator.example.org/agg1/services/sparql-query-view-service#> .
@prefix dcat: <http://www.w3.org/ns/dcat#> .
@prefix prov: <http://www.w3.org/ns/prov#> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .

<activity-1>
a prov:Activity ;
prov:associatedWith <https://example.org/agg1/services/sparql-query-view-service> ;
prov:startedAtTime "2025-01-02T12:00:00Z"^^xsd:dateTime ;
prov:endedAtTime "2025-01-02T12:02:00Z"^^xsd:dateTime .

<dataset-1>
a dcat:Dataset ;
a prov:Entity ;
dcat:isVersionOf svc:dataset ;
prov:wasGeneratedBy <activity-1> ;
prov:wasGeneratedAtTime "2025-01-02T12:02:00Z"^^xsd:dateTime .
```
</div>

With the logical dataset referencing the generated datasets.

<div class="example">
```turtle
@base : <https://aggregator.example.org/agg1/services/sparql-query-view-service> .
@prefix provlog: <https://aggregator.example.org/agg1/services/sparql-query-view-service/provenance#> .
@prefix dcat: <http://www.w3.org/ns/dcat#> .
@prefix prov: <http://www.w3.org/ns/prov#> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .

<#dataset>
a dcat:Dataset ;
dcat:distribution <#distribution-xml> ,
<#distribution-json> ;
dcat:hasCurrentVersion provlog:dataset-1 ;
dcat:hasVersion provlog:dataset-0 ,
provlog:dataset-1 .
```
</div>

Provenance Log {#data-model-provenance-log}
-------------------------------------------

A Provenance Log consolidates service output provenance information by referencing activities.

The Provenance Log:
- **MUST** be described as an `aggr:ProvenanceLog`.
- **MUST** reference one or more activities using `aggr:hasActivity`.

The Service:
- **MUST** link its provenance log using `aggr:provenanceLog`.


<div class="example">
```turtle
@base <https://aggregator.example.org/agg1/services/sparql-query-view-service/provenance>
@prefix aggr: <https://w3id.org/aggregator#> .

<>
a aggr:ProvenanceLog ;
aggr:hasActivity :activity-0 .
```
</div>

Service-management draft
------------------------

Service Provenance {#service-provenance}
----------------------------------------

This section describes how an Aggregator **SHOULD** store, expose, and maintain provenance metadata of service outputs.
An Aggregator is not required to manage provenance data. However, if it does, it **MUST** conform to this specification.
A service’s `aggr:ProvenanceLog` **MUST** reference **AT LEAST** the most recent `prov:Activity`
([[#data-model-service-activity]]).

Aggregators **MAY** publish a services' Provenance Log as an RDF Document whose base resource, identified by the document's
own retrieval URL, **MUST** be described as an `aggr:ProvenanceLog` ([[#data-model-provenance-log]]). Aggregators **MUST**
publish this document, at any deployment-specific URL of their choosing using the `GET` method. The endpoint **MUST**
be advertised in the Service Description through the `aggr:provenanceLog` predicate. The endpoint **SHOULD** support HTTP
content negotiation, allowing clients to request different RDF serializations (e.g., Turtle, JSON-LD). The activities that
are defined in the provenance log **MAY** be embedded in the Provenance Log document or **MAY** be hosted on separate endpoints
as individual RDF documents.

If the Provenance Log keeps a history log of multiple executions, it **SHOULD** be subject to a retention policy
defining how long execution records remain included in the log. This prevents unbounded growth of the log over time
and ensures manageable storage and query performance. If the Provenance Log only retains the most recent execution, versioning
**MAY** be omitted and the execution can directly reference the dataset from the Service Description.

<div class="example">
This example shows a Service Description for a service that updates its dataset on a daily basis.
In this example, the retained executions produce new versions of the logical service dataset, so that dataset is modeled
as a versioned dataset and references concrete dataset versions that are described in the Provenance Log.
```turtle
@base <https://aggregator.example/agg1/services/service1> .
@prefix dcat: <http://www.w3.org/ns/dcat#> .
@prefix dct: <http://purl.org/dc/terms/> .
@prefix freq: <http://purl.org/cld/freq/> .
@prefix aggr: <https://w3id.org/aggregator#> .
@prefix prov: <http://www.w3.org/ns/prov#> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .

<#dataset>
a dcat:Dataset ;
dcat:distribution <#distribution> ;
dcat:hasCurrentVersion <https://aggregator.example/agg1/services/service1/provenance#dataset-1> ;
dcat:hasVersion
<https://aggregator.example/agg1/services/service1/provenance#dataset-0>,
<https://aggregator.example/agg1/services/service1/provenance#dataset-1> ;
dct:accrualPeriodicity freq:daily ;
dct:modified "2025-01-02T12:02:00Z"^^xsd:dateTime .

<>
a aggr:Service ;
a dcat:DataService ;
a prov:SoftwareAgent ;
dcat:servesDataset <#dataset> ;
aggr:provenanceLog <https://aggregator.example/agg1/services/service1/provenance> ;
aggr:deploymentFunction <https://aggregator.example/deployments#QueryViewDeployment> .

<#distribution>
a dcat:Distribution ;
dcat:accessURL <https://aggregator.example/agg1/services/service1/output> ;
dcat:accessService <> .
```
</div>

<div class="example">
The Provenance Log below illustrates two activities that each produce a new version of the logical service dataset. In this
non-normative example, the activities are additionally modeled as `fno:Execution` resources; this is optional.
```turtle
@base <https://aggregator.example/agg1/services/service1/provenance> .
@prefix aggr: <https://w3id.org/aggregator#> .
@prefix fno: <https://w3id.org/function/ontology#> .
@prefix prov: <http://www.w3.org/ns/prov#> .
@prefix dcat: <http://www.w3.org/ns/dcat#> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
@prefix trans: <https://aggregator.example/transformations#> .

<>
a aggr:ProvenanceLog ;
aggr:hasActivity <#execution-0>, <#execution-1> .

<#execution-0>
a fno:Execution ;
a prov:Activity ;
fno:executes trans:QueryView ;
trans:query "SELECT * WHERE { ?s ?p ?o }" ;
trans:source "http://example.org/source1" ;
trans:result <#dataset-0> ;
prov:wasAssociatedWith <https://aggregator.example/agg1/services/service1> ;
prov:qualifiedAssociation [
a prov:Association ;
prov:agent <https://aggregator.example/agg1/services/service1> ;
prov:hadPlan trans:QueryView .
] ;
prov:used trans:QueryView ;
prov:startedAtTime "2025-01-01T12:00:00Z"^^xsd:dateTime ;
prov:endedAtTime "2025-01-01T12:02:00Z"^^xsd:dateTime .

<#execution-1>
a fno:Execution ;
a prov:Activity ;
fno:executes transQueryView ;
trans:query "SELECT * WHERE { ?s ?p ?o }" ;
trans:source "http://example.org/source1" ;
trans:result <#dataset-1> ;
prov:wasAssociatedWith <https://aggregator.example/agg1/services/service1> ;
prov:qualifiedAssociation [
a prov:Association ;
prov:agent <https://aggregator.example/agg1/services/service1> ;
prov:hadPlan <https://aggregator.example/agg1/services/service1/transformations#QueryView> .
] ;
prov:used <https://aggregator.example/agg1/services/service1/transformations#QueryView> ;
prov:invalidated <#dataset-0> ;
prov:startedAtTime "2025-01-02T12:00:00Z"^^xsd:dateTime ;
prov:endedAtTime "2025-01-02T12:02:00Z"^^xsd:dateTime .

<#dataset-0>
a dcat:Dataset ;
prov:wasGeneratedBy <#execution-0> ;
prov:wasInvalidatedBy <#execution-1> ;
dcat:isVersionOf <https://aggregator.example/agg1/services/service1#dataset> .

<#dataset-1>
a dcat:Dataset ;
prov:wasGeneratedBy <#execution-1> ;
dcat:isVersionOf <https://aggregator.example/agg1/services/service1#dataset> ;
dcat:previousVersion <#dataset-0> .
```
</div>

### Continuous updates ### {#continuous-updates}

For services that continuously update their data (e.g., event-driven or streaming updates) the execution **MUST** omit
`prov:endedAtTime`. All newly generated datasets **MUST** reference the same ongoing `prov:Activity`. Creating a new
`dcat:Dataset` for every minor change may lead to excessive provenance granularity and operational overhead.
Therefore, an Aggregator Server **MUST** determine an appropriate provenance granularity level based on its use case,
performance constraints, and retention policy. The server **MAY** choose, for example, to:
- Apply time-based grouping (e.g., hourly or daily snapshots)
- Only materialize provenance when used externally (e.g., when dataset was requested)

The chosen strategy **SHOULD** be documented and consistently applied to ensure predictable provenance semantics.

13 changes: 6 additions & 7 deletions sections/01-introduction.bs
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
Introduction {#introduction}
====================

This specification defines the Aggregator Protocol, an HTTP-based interface that lets a client create and manage Aggregator
Instances and configure Aggregator Services that execute data transformations. Transformations are described and discovered
using the Function Ontology (FnO) [[!FNO]]; a service is configured by referencing a transformation (a `fno:Function`) and
providing its parameters (using an `fno:AppliedFunction`), after which the client can retrieve the derived result from the
output defined by that function.
This specification defines the Aggregator Protocol, an HTTP-based interface that lets a client create and manage Aggregator
Instances and deploy Aggregator Services. Deployment interfaces are described using the Function Ontology (FnO) [[!FNO]];
a service is deployed by referencing a deployment function and providing its input parameters. A deployed Service is a
`dcat:DataService` that exposes datasets.

Clients start from the Aggregator Server Description at the server base URL to discover the management endpoint, supported
management flows, and the server’s transformation catalog ([[#aggregator-server-metadata]]). Using the management
management flows, and the server’s deployment catalog ([[#aggregator-server-metadata]]). Using the management
endpoint, a client creates (or manages) an Aggregator Instance ([[#aggregator-management]]) and then follows the
instance’s Aggregator Description to find the instance’s service collection and transformations endpoint
instance’s Aggregator Description to find the instance’s service collection and deployment catalog
([[#aggregator-description]] and [[#aggregator-service-management]]).

Management operations are authenticated and authorized unless a specific management flow explicitly allows unauthenticated
Expand Down
11 changes: 5 additions & 6 deletions sections/02-definitions.bs
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ Core Roles and Components {#definitions-core}

- **Aggregator Instance** (or “Aggregator”): A user- or tenant-scoped logical instance created via the management endpoint. An instance exposes an Aggregator Description and an authenticated management API for configuring services.

- **Aggregator Service** (or “Service”): A configured data processing pipeline inside an Aggregator Instance. A service is
represented as an FnO Implementation (`fno:Implementation`) that implements an FnO AppliedFunction (`fno:AppliedFunction`),
which defines inputs and outputs (see [[#aggregator-service-management]]).
- **Aggregator Service** (or “Service”): A deployed `dcat:DataService` and `prov:SoftwareAgent` inside an Aggregator
Instance. It serves one or more datasets and links to the FnO deployment function used to create it (see
[[#aggregator-service-management]]).

- **Transformation Catalog:** A catalog that describes the FnO Functions supported by the Aggregator Server and the FnO AppliedFunctions hosted on an Aggregator
Instance (see [[#server-level-transformation-catalog]] and [[#instance-level-transformation-catalog]]).
- **Deployment Catalog:** A catalog of FnO deployment functions supported by an Aggregator Server or Aggregator
Instance (see [[#server-level-deployment-catalog]] and [[#instance-level-deployment-catalog]]).

Identity, Authorization, and Tokens {#definitions-security}
----------------------------------------------------------
Expand Down Expand Up @@ -63,7 +63,6 @@ The following namespace prefixes are used throughout this specification and **SH
- `aggr:` → `https://w3id.org/aggregator#`
- `dct:` → `http://purl.org/dc/terms/`
- `fno:` → `https://w3id.org/function/ontology#`
- `fnoc:` → `https://fno.io/vocabulary/composition/0.1.0/`
- `dcat:` → `http://www.w3.org/ns/dcat#`
- `rdf:` → `http://www.w3.org/1999/02/22-rdf-syntax-ns#`
- `rdfs:` → `http://www.w3.org/2000/01/rdf-schema#`
Expand Down
Loading