file
file.import
Documentation
Import an external file into a kiara session.
Origin
Authors Markus Binsteiner (markus@frkl.io)
Context
Tags core
Labels package: kiara_modules.core
References source_repo:
https://github.com/DHARPA-Project/kia…
documentation:
https://dharpa.org/kiara_modules.core/
module_doc:
https://dharpa.org/kiara_modules.core…
source_url:
https://github.com/DHARPA-Project/kia…
Module config
Field Type Description Required
─────────────────────────────────────────────────────
constants object Value no
constants for
this module.
defaults object Value defaults no
for this
module.
source_profile string The name of yes
the source
profile. Used
to distinguish
different
input
categories for
the same input
type.
source_type string The type of yes
the source to
import from.
Module config -- no config --
Python class
class_name DefaultFileImportModule
module_name kiara_modules.core.file
full_name kiara_modules.core.file.DefaultFileI…
Processing source code ─────────────────────────────────────────────────────
def process(self, inputs: ValueSet, outputs: Value…
source_profile: str = self.get_config_value("s…
source_type: str = self.get_config_value("sour…
source = inputs.get_value_data(source_profile)
if self.get_target_value_type() == "any":
output_key: str = "value_item"
else:
output_key = self.get_target_value_type()
func_name = f"import_from__{source_profile}__{…
if not hasattr(self, func_name):
raise Exception(
f"Can't import '{source_type}' value: …
)
func = getattr(self, func_name)
# TODO: check signature?
result = func(source)
# schema = ValueSchema(type=self.get_target_va…
# value_lineage = ValueLineage.from_module_and…
# module=self, output_name=output_key, inp…
# )
# value: Value = self._kiara.data_registry.reg…
# value_data=result, value_schema=schema, …
# )
outputs.set_value(output_key, result)
─────────────────────────────────────────────────────
file.load
Documentation
Load a file and its metadata.
This module does not read or load the content of a
file, but contains the path to the local
representation/version of the file so it can be
read by a subsequent process.
Origin
Authors Markus Binsteiner (markus@frkl.io)
Context
Tags core
Labels package: kiara_modules.core
References source_repo:
https://github.com/DHARPA-Project/kia…
documentation:
https://dharpa.org/kiara_modules.core/
module_doc:
https://dharpa.org/kiara_modules.core…
source_url:
https://github.com/DHARPA-Project/kia…
Module config
Field Type Description Required
─────────────────────────────────────────────────────
constants object Value constants for no
this module.
defaults object Value defaults for no
this module.
Module config -- no config --
Python class
class_name LoadLocalFileModule
module_name kiara_modules.core.file
full_name kiara_modules.core.file.LoadLocalFil…
Processing source code ─────────────────────────────────────────────────────
def process(self, inputs: ValueSet, outputs: Value…
base_path = inputs.get_value_data("base_path")
rel_path = inputs.get_value_data("rel_path")
path = os.path.join(base_path, rel_path)
file_model = KiaraFile.load_file(path)
outputs.set_value("file", file_model)
─────────────────────────────────────────────────────
Documentation
-- n/a --
Origin
Authors Markus Binsteiner (markus@frkl.io)
Context
Tags core
Labels package: kiara_modules.core
References source_repo:
https://github.com/DHARPA-Project/kia…
documentation:
https://dharpa.org/kiara_modules.core/
module_doc:
https://dharpa.org/kiara_modules.core…
source_url:
https://github.com/DHARPA-Project/kia…
Module config
Field Type Description Required
─────────────────────────────────────────────────────
constants object Value constants no
for this module.
defaults object Value defaults for no
this module.
value_type string The data type this yes
module will be
used for.
Module config -- no config --
Python class
class_name FileMetadataModule
module_name kiara_modules.core.file
full_name kiara_modules.core.file.FileMetadata…
Processing source code ─────────────────────────────────────────────────────
def process(self, inputs: ValueSet, outputs: Value…
input_name = self.value_type
if input_name == "any":
input_name = "value_item"
value = inputs.get_value_obj(input_name)
if self.value_type != "any" and value.type_nam…
raise KiaraProcessingException(
f"Can't extract metadata for value of …
)
# TODO: if type 'any', validate that the data …
outputs.set_value("metadata_item_schema", self…
metadata = self.extract_metadata(value)
if isinstance(metadata, BaseModel):
metadata = metadata.dict(exclude_none=True)
# TODO: validate metadata?
outputs.set_value("metadata_item", metadata)
─────────────────────────────────────────────────────
file.store
Documentation
Save a file to disk.
Origin
Authors Markus Binsteiner (markus@frkl.io)
Context
Tags core
Labels package: kiara_modules.core
References source_repo:
https://github.com/DHARPA-Project/kia…
documentation:
https://dharpa.org/kiara_modules.core/
module_doc:
https://dharpa.org/kiara_modules.core…
source_url:
https://github.com/DHARPA-Project/kia…
Module config
Field Type Description Required
─────────────────────────────────────────────────────
constants object Value constants no
for this module.
defaults object Value defaults for no
this module.
value_type string The type of the yes
value to save.
Module config -- no config --
Python class
class_name StoreFileTypeModule
module_name kiara_modules.core.file
full_name kiara_modules.core.file.StoreFileTyp…
Processing source code ─────────────────────────────────────────────────────
def process(self, inputs: ValueSet, outputs: Value…
value_id: str = inputs.get_value_data("value_i…
if not value_id:
raise KiaraProcessingException("No value i…
field_name = self.get_config_value("value_type…
if field_name == "any":
field_name = "value_item"
value_obj: Value = inputs.get_value_obj(field_…
base_path: str = inputs.get_value_data("base_p…
result = self.store_value(value=value_obj, bas…
if isinstance(result, typing.Mapping):
load_config = result
result_value = value_obj
elif isinstance(result, tuple):
load_config = result[0]
if result[1]:
result_value = result[1]
else:
result_value = value_obj
else:
raise KiaraProcessingException(
f"Invalid result type for 'store_value…
)
load_config["value_id"] = value_id
lc = LoadConfig(**load_config)
if lc.base_path_input_name and lc.base_path_in…
raise KiaraProcessingException(
f"Invalid load config: base path '{lc.…
)
outputs.set_values(
metadata=None, lineage=None, **{"load_conf…
)
─────────────────────────────────────────────────────