Skip to content

metadata_store

Attributes

Classes

MetadataArchive

Bases: BaseArchive[ARCHIVE_CONFIG_CLS], Generic[ARCHIVE_CONFIG_CLS]

Base class for data archiv implementationss.

Source code in src/kiara/registries/metadata/metadata_store/__init__.py 
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
class MetadataArchive(BaseArchive[ARCHIVE_CONFIG_CLS], Generic[ARCHIVE_CONFIG_CLS]):
    """Base class for data archiv implementationss."""

    @classmethod
    def supported_item_types(cls) -> Iterable[str]:
        """This archive type only supports storing data."""

        return ["metadata"]

    def __init__(
        self,
        archive_name: str,
        archive_config: ARCHIVE_CONFIG_CLS,
        force_read_only: bool = False,
    ):
        super().__init__(
            archive_name=archive_name,
            archive_config=archive_config,
            force_read_only=force_read_only,
        )
        self._schema_stored_cache: Dict[str, Any] = {}
        self._schema_stored_item: Dict[str, uuid.UUID] = {}

    def find_metadata_item_with_hash(
        self, item_hash: str, key: Union[str, None] = None
    ) -> Union[Tuple[str, Mapping[str, Any]], None]:
        """Return the key of the metadata item with the specified hash."""

        return self._retrieve_metadata_item_with_hash(item_hash=item_hash, key=key)

    @abc.abstractmethod
    def _retrieve_metadata_item_with_hash(
        self, item_hash: str, key: Union[str, None] = None
    ) -> Union[Tuple[str, Mapping[str, Any]], None]:
        pass

    def find_matching_metadata_items(
        self,
        matcher: "MetadataMatcher",
        metadata_item_result_fields: Union[Iterable[str], None] = None,
        reference_item_result_fields: Union[Iterable[str], None] = None,
    ) -> Generator[Tuple[Any, ...], None, None]:
        return self._find_matching_metadata_and_ref_items(
            matcher=matcher,
            metadata_item_result_fields=metadata_item_result_fields,
            reference_item_result_fields=reference_item_result_fields,
        )

    @abc.abstractmethod
    def _find_matching_metadata_and_ref_items(
        self,
        matcher: "MetadataMatcher",
        metadata_item_result_fields: Union[Iterable[str], None] = None,
        reference_item_result_fields: Union[Iterable[str], None] = None,
    ) -> Generator[Tuple[Any, ...], None, None]:
        pass

    def retrieve_metadata_item(
        self,
        metadata_item_key: str,
        reference_type: Union[str, None] = None,
        reference_key: Union[str, None] = None,
        reference_id: Union[str, None] = None,
    ) -> Union[Tuple[str, Mapping[str, Any]], None]:
        """Return the model type and model data for the specified metadata item.

        If more than one item matches, an exception is raised.

        Arguments:
            metadata_item_key: The key of the metadata item to retrieve.
            reference_type: The type of the referenced item.
            reference_key: The key of the referenced item.
            reference_id: The id of the referenced item.
        """

        if reference_id and (not reference_type or not reference_key):
            raise ValueError(
                "If reference_id is set, reference_key & reference_type must be set as well."
            )
        if reference_type and reference_key:
            if reference_id is None:
                raise KiaraException(
                    msg="reference_id must set also if reference_type is set."
                )
            result = self._retrieve_referenced_metadata_item_data(
                key=metadata_item_key,
                reference_type=reference_type,
                reference_key=reference_key,
                reference_id=reference_id,
            )
            if result is None:
                return None
            else:
                return result
        else:
            raise NotImplementedError(
                "Retrieving metadata item without reference not implemented yet."
            )

    @abc.abstractmethod
    def _retrieve_referenced_metadata_item_data(
        self, key: str, reference_type: str, reference_key: str, reference_id: str
    ) -> Union[Tuple[str, Mapping[str, Any]], None]:
        """Return the model type id and model data for the specified referenced metadata item."""

Functions

supported_item_types() -> Iterable[str] classmethod

This archive type only supports storing data.

Source code in src/kiara/registries/metadata/metadata_store/__init__.py 
28
29
30
31
32
@classmethod
def supported_item_types(cls) -> Iterable[str]:
    """This archive type only supports storing data."""

    return ["metadata"]
find_metadata_item_with_hash(item_hash: str, key: Union[str, None] = None) -> Union[Tuple[str, Mapping[str, Any]], None]

Return the key of the metadata item with the specified hash.

Source code in src/kiara/registries/metadata/metadata_store/__init__.py 
48
49
50
51
52
53
def find_metadata_item_with_hash(
    self, item_hash: str, key: Union[str, None] = None
) -> Union[Tuple[str, Mapping[str, Any]], None]:
    """Return the key of the metadata item with the specified hash."""

    return self._retrieve_metadata_item_with_hash(item_hash=item_hash, key=key)
find_matching_metadata_items(matcher: MetadataMatcher, metadata_item_result_fields: Union[Iterable[str], None] = None, reference_item_result_fields: Union[Iterable[str], None] = None) -> Generator[Tuple[Any, ...], None, None]
Source code in src/kiara/registries/metadata/metadata_store/__init__.py 
61
62
63
64
65
66
67
68
69
70
71
def find_matching_metadata_items(
    self,
    matcher: "MetadataMatcher",
    metadata_item_result_fields: Union[Iterable[str], None] = None,
    reference_item_result_fields: Union[Iterable[str], None] = None,
) -> Generator[Tuple[Any, ...], None, None]:
    return self._find_matching_metadata_and_ref_items(
        matcher=matcher,
        metadata_item_result_fields=metadata_item_result_fields,
        reference_item_result_fields=reference_item_result_fields,
    )
retrieve_metadata_item(metadata_item_key: str, reference_type: Union[str, None] = None, reference_key: Union[str, None] = None, reference_id: Union[str, None] = None) -> Union[Tuple[str, Mapping[str, Any]], None]

Return the model type and model data for the specified metadata item.

If more than one item matches, an exception is raised.

Parameters:

Name Type Description Default
metadata_item_key str

The key of the metadata item to retrieve.

 required
reference_type Union[str, None]

The type of the referenced item.

 None
reference_key Union[str, None]

The key of the referenced item.

 None
reference_id Union[str, None]

The id of the referenced item.

 None
Source code in src/kiara/registries/metadata/metadata_store/__init__.py 
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
def retrieve_metadata_item(
    self,
    metadata_item_key: str,
    reference_type: Union[str, None] = None,
    reference_key: Union[str, None] = None,
    reference_id: Union[str, None] = None,
) -> Union[Tuple[str, Mapping[str, Any]], None]:
    """Return the model type and model data for the specified metadata item.

    If more than one item matches, an exception is raised.

    Arguments:
        metadata_item_key: The key of the metadata item to retrieve.
        reference_type: The type of the referenced item.
        reference_key: The key of the referenced item.
        reference_id: The id of the referenced item.
    """

    if reference_id and (not reference_type or not reference_key):
        raise ValueError(
            "If reference_id is set, reference_key & reference_type must be set as well."
        )
    if reference_type and reference_key:
        if reference_id is None:
            raise KiaraException(
                msg="reference_id must set also if reference_type is set."
            )
        result = self._retrieve_referenced_metadata_item_data(
            key=metadata_item_key,
            reference_type=reference_type,
            reference_key=reference_key,
            reference_id=reference_id,
        )
        if result is None:
            return None
        else:
            return result
    else:
        raise NotImplementedError(
            "Retrieving metadata item without reference not implemented yet."
        )

MetadataStore

Bases: MetadataArchive

Source code in src/kiara/registries/metadata/metadata_store/__init__.py 
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
class MetadataStore(MetadataArchive):
    def __init__(
        self,
        archive_name: str,
        archive_config: ARCHIVE_CONFIG_CLS,
        force_read_only: bool = False,
    ):
        super().__init__(
            archive_name=archive_name,
            archive_config=archive_config,
            force_read_only=force_read_only,
        )

    @classmethod
    def _is_writeable(cls) -> bool:
        return True

    @abc.abstractmethod
    def _store_metadata_schema(
        self, model_schema_hash: str, model_type_id: str, model_schema: str
    ):
        """Store the metadata schema for the specified model."""

    def store_metadata_item(
        self,
        key: str,
        item: KiaraMetadata,
        reference_item_type: Union[str, None] = None,
        reference_item_key: Union[str, None] = None,
        reference_item_id: Union[str, None] = None,
        replace_existing_references: bool = False,
        allow_multiple_references: bool = False,
        store: Union[str, uuid.UUID, None] = None,
    ) -> uuid.UUID:
        """Store a metadata item into the store.

        If `reference_item_type` and `reference_item_id` are set, the stored metadata item will
        be linked to the stored metadata item, to enable lokoups later on.
        """

        if store:
            raise NotImplementedError(
                "Cannot store metadata item with store, not implemented yet."
            )

        # TODO: check if already stored
        model_type = item.model_type_id
        model_schema_hash = str(item.get_schema_cid())

        if model_schema_hash not in self._schema_stored_cache.keys():
            model_item_schema = item.model_json_schema()
            model_item_schema_str = json.dumps(model_item_schema)

            self._store_metadata_schema(
                model_schema_hash=model_schema_hash,
                model_type_id=model_type,
                model_schema=model_item_schema_str,
            )
            self._schema_stored_cache[model_schema_hash] = model_item_schema

        # data = item.model_dump()
        data_json = item.model_dump_json()
        data_hash = str(item.instance_cid)

        metadata_item_id = self._schema_stored_item.get(data_hash, None)
        if not metadata_item_id:
            metadata_item_id = self._store_metadata_item(
                key=key,
                value_json=data_json,
                value_hash=data_hash,
                model_type_id=model_type,
                model_schema_hash=model_schema_hash,
            )
            self._schema_stored_item[data_hash] = metadata_item_id

        if (reference_item_id and not reference_item_type) or (
            reference_item_type and not reference_item_id
        ):
            raise ValueError(
                "If reference_item_id is set, reference_item_type must be set as well."
            )

        if reference_item_type:
            assert reference_item_id is not None
            assert reference_item_key is not None
            self._store_metadata_reference(
                reference_item_type=reference_item_type,
                reference_item_key=reference_item_key,
                reference_item_id=reference_item_id,
                metadata_item_id=str(metadata_item_id),
                replace_existing_references=replace_existing_references,
                allow_multiple_references=allow_multiple_references,
            )

        return metadata_item_id

    @abc.abstractmethod
    def _store_metadata_reference(
        self,
        reference_item_type: str,
        reference_item_key: str,
        reference_item_id: str,
        metadata_item_id: str,
        replace_existing_references: bool = False,
        allow_multiple_references: bool = False,
    ) -> None:
        pass

    @abc.abstractmethod
    def _store_metadata_item(
        self,
        key: str,
        value_json: str,
        value_hash: str,
        model_type_id: str,
        model_schema_hash: str,
    ) -> uuid.UUID:
        pass

    def store_metadata_and_ref_items(
        self, items: Generator[Tuple[Any, ...], None, None]
    ):
        return self._store_metadata_and_ref_items(items)

    @abc.abstractmethod
    def _store_metadata_and_ref_items(
        self, items: Generator[Tuple[Any, ...], None, None]
    ):
        pass

Functions

store_metadata_item(key: str, item: KiaraMetadata, reference_item_type: Union[str, None] = None, reference_item_key: Union[str, None] = None, reference_item_id: Union[str, None] = None, replace_existing_references: bool = False, allow_multiple_references: bool = False, store: Union[str, uuid.UUID, None] = None) -> uuid.UUID

Store a metadata item into the store.

If reference_item_type and reference_item_id are set, the stored metadata item will be linked to the stored metadata item, to enable lokoups later on.

Source code in src/kiara/registries/metadata/metadata_store/__init__.py 
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
def store_metadata_item(
    self,
    key: str,
    item: KiaraMetadata,
    reference_item_type: Union[str, None] = None,
    reference_item_key: Union[str, None] = None,
    reference_item_id: Union[str, None] = None,
    replace_existing_references: bool = False,
    allow_multiple_references: bool = False,
    store: Union[str, uuid.UUID, None] = None,
) -> uuid.UUID:
    """Store a metadata item into the store.

    If `reference_item_type` and `reference_item_id` are set, the stored metadata item will
    be linked to the stored metadata item, to enable lokoups later on.
    """

    if store:
        raise NotImplementedError(
            "Cannot store metadata item with store, not implemented yet."
        )

    # TODO: check if already stored
    model_type = item.model_type_id
    model_schema_hash = str(item.get_schema_cid())

    if model_schema_hash not in self._schema_stored_cache.keys():
        model_item_schema = item.model_json_schema()
        model_item_schema_str = json.dumps(model_item_schema)

        self._store_metadata_schema(
            model_schema_hash=model_schema_hash,
            model_type_id=model_type,
            model_schema=model_item_schema_str,
        )
        self._schema_stored_cache[model_schema_hash] = model_item_schema

    # data = item.model_dump()
    data_json = item.model_dump_json()
    data_hash = str(item.instance_cid)

    metadata_item_id = self._schema_stored_item.get(data_hash, None)
    if not metadata_item_id:
        metadata_item_id = self._store_metadata_item(
            key=key,
            value_json=data_json,
            value_hash=data_hash,
            model_type_id=model_type,
            model_schema_hash=model_schema_hash,
        )
        self._schema_stored_item[data_hash] = metadata_item_id

    if (reference_item_id and not reference_item_type) or (
        reference_item_type and not reference_item_id
    ):
        raise ValueError(
            "If reference_item_id is set, reference_item_type must be set as well."
        )

    if reference_item_type:
        assert reference_item_id is not None
        assert reference_item_key is not None
        self._store_metadata_reference(
            reference_item_type=reference_item_type,
            reference_item_key=reference_item_key,
            reference_item_id=reference_item_id,
            metadata_item_id=str(metadata_item_id),
            replace_existing_references=replace_existing_references,
            allow_multiple_references=allow_multiple_references,
        )

    return metadata_item_id
store_metadata_and_ref_items(items: Generator[Tuple[Any, ...], None, None])
Source code in src/kiara/registries/metadata/metadata_store/__init__.py 
250
251
252
253
def store_metadata_and_ref_items(
    self, items: Generator[Tuple[Any, ...], None, None]
):
    return self._store_metadata_and_ref_items(items)