Skip to content

tokens

Attributes

log = structlog.getLogger() module-attribute

Classes

TokenizeTextConfig

Bases: KiaraModuleConfig

Source code in kiara_plugin/language_processing/modules/tokens.py 
32
33
34
35
36
37
38
39
40
class TokenizeTextConfig(KiaraModuleConfig):

    filter_non_alpha: bool = Field(
        description="Whether to filter out non alpha tokens.", default=True
    )
    min_token_length: int = Field(description="The minimum token length.", default=3)
    to_lowercase: bool = Field(
        description="Whether to lowercase the tokens.", default=True
    )

Attributes

filter_non_alpha: bool = Field(description='Whether to filter out non alpha tokens.', default=True) class-attribute
min_token_length: int = Field(description='The minimum token length.', default=3) class-attribute
to_lowercase: bool = Field(description='Whether to lowercase the tokens.', default=True) class-attribute

TokenizeTextModule

Bases: KiaraModule

Tokenize a string.

Source code in kiara_plugin/language_processing/modules/tokens.py 
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
class TokenizeTextModule(KiaraModule):
    """Tokenize a string."""

    _config_cls = TokenizeTextConfig
    _module_type_name = "tokenize.string"

    def create_inputs_schema(
        self,
    ) -> ValueSetSchema:

        inputs = {"text": {"type": "string", "doc": "The text to tokenize."}}

        return inputs

    def create_outputs_schema(
        self,
    ) -> ValueSetSchema:

        outputs = {
            "token_list": {
                "type": "list",
                "doc": "The tokenized version of the input text.",
            }
        }
        return outputs

    def process(self, inputs: ValueMap, outputs: ValueMap) -> None:

        import nltk

        # TODO: module-independent caching?
        # language = inputs.get_value_data("language")
        #
        text = inputs.get_value_data("text")
        tokenized = nltk.word_tokenize(text)

        result = tokenized
        if self.get_config_value("min_token_length") > 0:
            result = (
                x
                for x in tokenized
                if len(x) >= self.get_config_value("min_token_length")
            )

        if self.get_config_value("filter_non_alpha"):
            result = (x for x in result if x.isalpha())

        if self.get_config_value("to_lowercase"):
            result = (x.lower() for x in result)

        outputs.set_value("token_list", list(result))

Attributes

_config_cls = TokenizeTextConfig class-attribute

Functions

create_inputs_schema() -> ValueSetSchema
Source code in kiara_plugin/language_processing/modules/tokens.py 
49
50
51
52
53
54
55
def create_inputs_schema(
    self,
) -> ValueSetSchema:

    inputs = {"text": {"type": "string", "doc": "The text to tokenize."}}

    return inputs
create_outputs_schema() -> ValueSetSchema
Source code in kiara_plugin/language_processing/modules/tokens.py 
57
58
59
60
61
62
63
64
65
66
67
def create_outputs_schema(
    self,
) -> ValueSetSchema:

    outputs = {
        "token_list": {
            "type": "list",
            "doc": "The tokenized version of the input text.",
        }
    }
    return outputs
process(inputs: ValueMap, outputs: ValueMap) -> None
Source code in kiara_plugin/language_processing/modules/tokens.py 
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:

    import nltk

    # TODO: module-independent caching?
    # language = inputs.get_value_data("language")
    #
    text = inputs.get_value_data("text")
    tokenized = nltk.word_tokenize(text)

    result = tokenized
    if self.get_config_value("min_token_length") > 0:
        result = (
            x
            for x in tokenized
            if len(x) >= self.get_config_value("min_token_length")
        )

    if self.get_config_value("filter_non_alpha"):
        result = (x for x in result if x.isalpha())

    if self.get_config_value("to_lowercase"):
        result = (x.lower() for x in result)

    outputs.set_value("token_list", list(result))

TokenizeTextArrayeModule

Bases: KiaraModule

Split sentences into words or words into characters. In other words, this operation establishes the word boundaries (i.e., tokens) a very helpful way of finding patterns. It is also the typical step prior to stemming and lemmatization

Source code in kiara_plugin/language_processing/modules/tokens.py 
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
class TokenizeTextArrayeModule(KiaraModule):
    """Split sentences into words or words into characters.
    In other words, this operation establishes the word boundaries (i.e., tokens) a very helpful way of finding patterns. It is also the typical step prior to stemming and lemmatization
    """

    _module_type_name = "tokenize.texts_array"

    KIARA_METADATA = {
        "tags": ["tokenize", "tokens"],
    }

    def create_inputs_schema(
        self,
    ) -> ValueSetSchema:

        return {
            "texts_array": {
                "type": "array",
                "doc": "An array of text items to be tokenized.",
            },
            "tokenize_by_word": {
                "type": "boolean",
                "doc": "Whether to tokenize by word (default), or character.",
                "default": True,
            },
        }

    def create_outputs_schema(
        self,
    ) -> ValueSetSchema:

        return {
            "tokens_array": {
                "type": "array",
                "doc": "The tokenized content, as an array of lists of strings.",
            }
        }

    def process(self, inputs: ValueMap, outputs: ValueMap):

        pass

        import nltk
        import polars as pl
        import pyarrow as pa

        array: KiaraArray = inputs.get_value_data("texts_array")
        # tokenize_by_word: bool = inputs.get_value_data("tokenize_by_word")

        column: pa.ChunkedArray = array.arrow_array

        # warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning)

        def word_tokenize(word):
            result = nltk.word_tokenize(word)
            return result

        series = pl.Series(name="tokens", values=column)
        result = series.apply(word_tokenize)

        result_array = result.to_arrow()

        # TODO: remove this cast once the array data type can handle non-chunked arrays
        chunked = pa.chunked_array(result_array)
        outputs.set_values(tokens_array=chunked)

Attributes

KIARA_METADATA = {'tags': ['tokenize', 'tokens']} class-attribute

Functions

create_inputs_schema() -> ValueSetSchema
Source code in kiara_plugin/language_processing/modules/tokens.py 
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def create_inputs_schema(
    self,
) -> ValueSetSchema:

    return {
        "texts_array": {
            "type": "array",
            "doc": "An array of text items to be tokenized.",
        },
        "tokenize_by_word": {
            "type": "boolean",
            "doc": "Whether to tokenize by word (default), or character.",
            "default": True,
        },
    }
create_outputs_schema() -> ValueSetSchema
Source code in kiara_plugin/language_processing/modules/tokens.py 
123
124
125
126
127
128
129
130
131
132
def create_outputs_schema(
    self,
) -> ValueSetSchema:

    return {
        "tokens_array": {
            "type": "array",
            "doc": "The tokenized content, as an array of lists of strings.",
        }
    }
process(inputs: ValueMap, outputs: ValueMap)
Source code in kiara_plugin/language_processing/modules/tokens.py 
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def process(self, inputs: ValueMap, outputs: ValueMap):

    pass

    import nltk
    import polars as pl
    import pyarrow as pa

    array: KiaraArray = inputs.get_value_data("texts_array")
    # tokenize_by_word: bool = inputs.get_value_data("tokenize_by_word")

    column: pa.ChunkedArray = array.arrow_array

    # warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning)

    def word_tokenize(word):
        result = nltk.word_tokenize(word)
        return result

    series = pl.Series(name="tokens", values=column)
    result = series.apply(word_tokenize)

    result_array = result.to_arrow()

    # TODO: remove this cast once the array data type can handle non-chunked arrays
    chunked = pa.chunked_array(result_array)
    outputs.set_values(tokens_array=chunked)

AssembleStopwordsModule

Bases: KiaraModule

Create a list of stopwords from one or multiple sources.

This will download nltk stopwords if necessary, and merge all input lists into a single, sorted list without duplicates.

Source code in kiara_plugin/language_processing/modules/tokens.py 
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
class AssembleStopwordsModule(KiaraModule):
    """Create a list of stopwords from one or multiple sources.

    This will download nltk stopwords if necessary, and merge all input lists into a single, sorted list without duplicates.
    """

    _module_type_name = "create.stopwords_list"

    def create_inputs_schema(
        self,
    ) -> ValueSetSchema:

        return {
            "languages": {
                "type": "list",
                "doc": "A list of languages, will be used to retrieve language-specific stopword from nltk.",
                "optional": True,
            },
            "stopword_list": {
                "type": "list",
                "doc": "A list of additional, custom stopwords.",
                "optional": True,
            },
        }

    def create_outputs_schema(
        self,
    ) -> ValueSetSchema:

        return {
            "stopwords_list": {
                "type": "list",
                "doc": "A sorted list of unique stopwords.",
            }
        }

    def process(self, inputs: ValueMap, outputs: ValueMap):

        stopwords = set()
        _languages = inputs.get_value_obj("languages")

        if _languages.is_set:
            all_stopwords = get_stopwords()
            languages: ListModel = _languages.data

            for language in languages.list_data:

                if language not in all_stopwords.fileids():
                    raise KiaraProcessingException(
                        f"Invalid language: {language}. Available: {', '.join(all_stopwords.fileids())}."
                    )
                stopwords.update(get_stopwords().words(language))

        _stopword_lists = inputs.get_value_obj("stopword_list")
        if _stopword_lists.is_set:
            stopword_lists: ListModel = _stopword_lists.data
            for stopword_list in stopword_lists.list_data:
                if isinstance(stopword_list, str):
                    stopwords.add(stopword_list)
                else:
                    stopwords.update(stopword_list)

        outputs.set_value("stopwords_list", sorted(stopwords))

Functions

create_inputs_schema() -> ValueSetSchema
Source code in kiara_plugin/language_processing/modules/tokens.py 
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
def create_inputs_schema(
    self,
) -> ValueSetSchema:

    return {
        "languages": {
            "type": "list",
            "doc": "A list of languages, will be used to retrieve language-specific stopword from nltk.",
            "optional": True,
        },
        "stopword_list": {
            "type": "list",
            "doc": "A list of additional, custom stopwords.",
            "optional": True,
        },
    }
create_outputs_schema() -> ValueSetSchema
Source code in kiara_plugin/language_processing/modules/tokens.py 
188
189
190
191
192
193
194
195
196
197
def create_outputs_schema(
    self,
) -> ValueSetSchema:

    return {
        "stopwords_list": {
            "type": "list",
            "doc": "A sorted list of unique stopwords.",
        }
    }
process(inputs: ValueMap, outputs: ValueMap)
Source code in kiara_plugin/language_processing/modules/tokens.py 
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
def process(self, inputs: ValueMap, outputs: ValueMap):

    stopwords = set()
    _languages = inputs.get_value_obj("languages")

    if _languages.is_set:
        all_stopwords = get_stopwords()
        languages: ListModel = _languages.data

        for language in languages.list_data:

            if language not in all_stopwords.fileids():
                raise KiaraProcessingException(
                    f"Invalid language: {language}. Available: {', '.join(all_stopwords.fileids())}."
                )
            stopwords.update(get_stopwords().words(language))

    _stopword_lists = inputs.get_value_obj("stopword_list")
    if _stopword_lists.is_set:
        stopword_lists: ListModel = _stopword_lists.data
        for stopword_list in stopword_lists.list_data:
            if isinstance(stopword_list, str):
                stopwords.add(stopword_list)
            else:
                stopwords.update(stopword_list)

    outputs.set_value("stopwords_list", sorted(stopwords))

RemoveStopwordsModule

Bases: KiaraModule

Remove stopwords from an array of token-lists.

Source code in kiara_plugin/language_processing/modules/tokens.py 
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
class RemoveStopwordsModule(KiaraModule):
    """Remove stopwords from an array of token-lists."""

    _module_type_name = "remove_stopwords.from.tokens_array"

    def create_inputs_schema(
        self,
    ) -> ValueSetSchema:

        # TODO: do something smart and check whether languages are already downloaded, if so, display selection in doc
        inputs: Dict[str, Dict[str, Any]] = {
            "tokens_array": {
                "type": "array",
                "doc": "An array of string lists (a list of tokens).",
            },
            "languages": {
                "type": "list",
                # "doc": f"A list of language names to use default stopword lists for. Available: {', '.join(get_stopwords().fileids())}.",
                "doc": "A list of language names to use default stopword lists for.",
                "optional": True,
            },
            "additional_stopwords": {
                "type": "list",
                "doc": "A list of additional, custom stopwords.",
                "optional": True,
            },
        }
        return inputs

    def create_outputs_schema(
        self,
    ) -> ValueSetSchema:

        outputs = {
            "tokens_array": {
                "type": "array",
                "doc": "An array of string lists, with the stopwords removed.",
            }
        }
        return outputs

    def process(self, inputs: ValueMap, outputs: ValueMap) -> None:

        import pyarrow as pa

        custom_stopwords = inputs.get_value_data("additional_stopwords")

        if inputs.get_value_obj("languages").is_set:
            _languages: ListModel = inputs.get_value_data("languages")
            languages = _languages.list_data
        else:
            languages = []

        stopwords = set()
        if languages:
            for language in languages:
                if language not in get_stopwords().fileids():
                    raise KiaraProcessingException(
                        f"Invalid language: {language}. Available: {', '.join(get_stopwords().fileids())}."
                    )
                stopwords.update(get_stopwords().words(language))

        if custom_stopwords:
            stopwords.update(custom_stopwords)

        orig_array = inputs.get_value_obj("tokens_array")  # type: ignore

        if not stopwords:
            outputs.set_value("tokens_array", orig_array)
            return

        # if hasattr(orig_array, "to_pylist"):
        #     token_lists = orig_array.to_pylist()

        tokens_array = orig_array.data.arrow_array

        # TODO: use vaex for this
        result = []
        for token_list in tokens_array:

            cleaned_list = [x for x in token_list.as_py() if x.lower() not in stopwords]
            result.append(cleaned_list)

        outputs.set_value("tokens_array", pa.chunked_array(pa.array(result)))

Functions

create_inputs_schema() -> ValueSetSchema
Source code in kiara_plugin/language_processing/modules/tokens.py 
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
def create_inputs_schema(
    self,
) -> ValueSetSchema:

    # TODO: do something smart and check whether languages are already downloaded, if so, display selection in doc
    inputs: Dict[str, Dict[str, Any]] = {
        "tokens_array": {
            "type": "array",
            "doc": "An array of string lists (a list of tokens).",
        },
        "languages": {
            "type": "list",
            # "doc": f"A list of language names to use default stopword lists for. Available: {', '.join(get_stopwords().fileids())}.",
            "doc": "A list of language names to use default stopword lists for.",
            "optional": True,
        },
        "additional_stopwords": {
            "type": "list",
            "doc": "A list of additional, custom stopwords.",
            "optional": True,
        },
    }
    return inputs
create_outputs_schema() -> ValueSetSchema
Source code in kiara_plugin/language_processing/modules/tokens.py 
257
258
259
260
261
262
263
264
265
266
267
def create_outputs_schema(
    self,
) -> ValueSetSchema:

    outputs = {
        "tokens_array": {
            "type": "array",
            "doc": "An array of string lists, with the stopwords removed.",
        }
    }
    return outputs
process(inputs: ValueMap, outputs: ValueMap) -> None
Source code in kiara_plugin/language_processing/modules/tokens.py 
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:

    import pyarrow as pa

    custom_stopwords = inputs.get_value_data("additional_stopwords")

    if inputs.get_value_obj("languages").is_set:
        _languages: ListModel = inputs.get_value_data("languages")
        languages = _languages.list_data
    else:
        languages = []

    stopwords = set()
    if languages:
        for language in languages:
            if language not in get_stopwords().fileids():
                raise KiaraProcessingException(
                    f"Invalid language: {language}. Available: {', '.join(get_stopwords().fileids())}."
                )
            stopwords.update(get_stopwords().words(language))

    if custom_stopwords:
        stopwords.update(custom_stopwords)

    orig_array = inputs.get_value_obj("tokens_array")  # type: ignore

    if not stopwords:
        outputs.set_value("tokens_array", orig_array)
        return

    # if hasattr(orig_array, "to_pylist"):
    #     token_lists = orig_array.to_pylist()

    tokens_array = orig_array.data.arrow_array

    # TODO: use vaex for this
    result = []
    for token_list in tokens_array:

        cleaned_list = [x for x in token_list.as_py() if x.lower() not in stopwords]
        result.append(cleaned_list)

    outputs.set_value("tokens_array", pa.chunked_array(pa.array(result)))

PreprocessModule

Bases: KiaraModule

Preprocess lists of tokens, incl. lowercasing, remove special characers, etc.

Lowercasing: Lowercase the words. This operation is a double-edged sword. It can be effective at yielding potentially better results in the case of relatively small datasets or datatsets with a high percentage of OCR mistakes. For instance, if lowercasing is not performed, the algorithm will treat USA, Usa, usa, UsA, uSA, etc. as distinct tokens, even though they may all refer to the same entity. On the other hand, if the dataset does not contain such OCR mistakes, then it may become difficult to distinguish between homonyms and make interpreting the topics much harder.

Removing stopwords and words with less than three characters: Remove low information words. These are typically words such as articles, pronouns, prepositions, conjunctions, etc. which are not semantically salient. There are numerous stopword lists available for many, though not all, languages which can be easily adapted to the individual researcher's needs. Removing words with less than three characters may additionally remove many OCR mistakes. Both these operations have the dual advantage of yielding more reliable results while reducing the size of the dataset, thus in turn reducing the required processing power. This step can therefore hardly be considered optional in TM.

Noise removal: Remove elements such as punctuation marks, special characters, numbers, html formatting, etc. This operation is again concerned with removing elements that may not be relevant to the text analysis and in fact interfere with it. Depending on the dataset and research question, this operation can become essential.

Source code in kiara_plugin/language_processing/modules/tokens.py 
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
class PreprocessModule(KiaraModule):
    """Preprocess lists of tokens, incl. lowercasing, remove special characers, etc.

    Lowercasing: Lowercase the words. This operation is a double-edged sword. It can be effective at yielding potentially better results in the case of relatively small datasets or datatsets with a high percentage of OCR mistakes. For instance, if lowercasing is not performed, the algorithm will treat USA, Usa, usa, UsA, uSA, etc. as distinct tokens, even though they may all refer to the same entity. On the other hand, if the dataset does not contain such OCR mistakes, then it may become difficult to distinguish between homonyms and make interpreting the topics much harder.

    Removing stopwords and words with less than three characters: Remove low information words. These are typically words such as articles, pronouns, prepositions, conjunctions, etc. which are not semantically salient. There are numerous stopword lists available for many, though not all, languages which can be easily adapted to the individual researcher's needs. Removing words with less than three characters may additionally remove many OCR mistakes. Both these operations have the dual advantage of yielding more reliable results while reducing the size of the dataset, thus in turn reducing the required processing power. This step can therefore hardly be considered optional in TM.

    Noise removal: Remove elements such as punctuation marks, special characters, numbers, html formatting, etc. This operation is again concerned with removing elements that may not be relevant to the text analysis and in fact interfere with it. Depending on the dataset and research question, this operation can become essential.
    """

    _module_type_name = "preprocess.tokens_array"

    KIARA_METADATA = {
        "tags": ["tokens", "preprocess"],
    }

    def create_inputs_schema(
        self,
    ) -> ValueSetSchema:

        return {
            "tokens_array": {
                "type": "array",
                "doc": "The tokens array to pre-process.",
            },
            "to_lowercase": {
                "type": "boolean",
                "doc": "Apply lowercasing to the text.",
                "default": False,
            },
            "remove_alphanumeric": {
                "type": "boolean",
                "doc": "Remove all tokens that include numbers (e.g. ex1ample).",
                "default": False,
            },
            "remove_non_alpha": {
                "type": "boolean",
                "doc": "Remove all tokens that include punctuation and numbers (e.g. ex1a.mple).",
                "default": False,
            },
            "remove_all_numeric": {
                "type": "boolean",
                "doc": "Remove all tokens that contain numbers only (e.g. 876).",
                "default": False,
            },
            "remove_short_tokens": {
                "type": "integer",
                "doc": "Remove tokens shorter or equal to this value. If value is <= 0, no filtering will be done.",
                "default": 0,
            },
            "remove_stopwords": {
                "type": "list",
                "doc": "Remove stopwords.",
                "optional": True,
            },
        }

    def create_outputs_schema(
        self,
    ) -> ValueSetSchema:

        return {
            "tokens_array": {
                "type": "array",
                "doc": "The pre-processed content, as an array of lists of strings.",
            }
        }

    def process(self, inputs: ValueMap, outputs: ValueMap):

        import polars as pl
        import pyarrow as pa

        tokens_array: KiaraArray = inputs.get_value_data("tokens_array")
        lowercase: bool = inputs.get_value_data("to_lowercase")
        remove_alphanumeric: bool = inputs.get_value_data("remove_alphanumeric")
        remove_non_alpha: bool = inputs.get_value_data("remove_non_alpha")
        remove_all_numeric: bool = inputs.get_value_data("remove_all_numeric")
        remove_short_tokens: int = inputs.get_value_data("remove_short_tokens")

        if remove_short_tokens is None:
            remove_short_tokens = -1

        _remove_stopwords = inputs.get_value_obj("remove_stopwords")
        if _remove_stopwords.is_set:
            stopword_list: Union[Iterable[str], None] = _remove_stopwords.data.list_data
        else:
            stopword_list = None

        # it's better to have one method every token goes through, then do every test seperately for the token list
        # because that way each token only needs to be touched once (which is more effective)
        def check_token(token: str) -> Union[str, None]:

            # remove short tokens first, since we can save ourselves all the other checks (which are more expensive)
            assert isinstance(remove_short_tokens, int)
            if remove_short_tokens > 0:
                if len(token) <= remove_short_tokens:
                    return None

            _token: str = token
            if lowercase:
                _token = _token.lower()

            if remove_non_alpha:
                match = _token if _token.isalpha() else None
                if match is None:
                    return None

            # if remove_non_alpha was set, we don't need to worry about tokens that include numbers, since they are already filtered out
            if remove_alphanumeric and not remove_non_alpha:
                match = _token if _token.isalnum() else None
                if match is None:
                    return None

            # all-number tokens are already filtered out if the remove_non_alpha methods above ran
            if remove_all_numeric and not remove_non_alpha:
                match = None if _token.isdigit() else _token
                if match is None:
                    return None

            if stopword_list and _token and _token.lower() in stopword_list:
                return None

            return _token

        series = pl.Series(name="tokens", values=tokens_array.arrow_array)
        result = series.apply(
            lambda token_list: [
                x for x in (check_token(token) for token in token_list) if x is not None
            ]
        )
        result_array = result.to_arrow()

        # TODO: remove this cast once the array data type can handle non-chunked arrays
        chunked = pa.chunked_array(result_array)
        outputs.set_values(tokens_array=chunked)

Attributes

KIARA_METADATA = {'tags': ['tokens', 'preprocess']} class-attribute

Functions

create_inputs_schema() -> ValueSetSchema
Source code in kiara_plugin/language_processing/modules/tokens.py 
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
def create_inputs_schema(
    self,
) -> ValueSetSchema:

    return {
        "tokens_array": {
            "type": "array",
            "doc": "The tokens array to pre-process.",
        },
        "to_lowercase": {
            "type": "boolean",
            "doc": "Apply lowercasing to the text.",
            "default": False,
        },
        "remove_alphanumeric": {
            "type": "boolean",
            "doc": "Remove all tokens that include numbers (e.g. ex1ample).",
            "default": False,
        },
        "remove_non_alpha": {
            "type": "boolean",
            "doc": "Remove all tokens that include punctuation and numbers (e.g. ex1a.mple).",
            "default": False,
        },
        "remove_all_numeric": {
            "type": "boolean",
            "doc": "Remove all tokens that contain numbers only (e.g. 876).",
            "default": False,
        },
        "remove_short_tokens": {
            "type": "integer",
            "doc": "Remove tokens shorter or equal to this value. If value is <= 0, no filtering will be done.",
            "default": 0,
        },
        "remove_stopwords": {
            "type": "list",
            "doc": "Remove stopwords.",
            "optional": True,
        },
    }
create_outputs_schema() -> ValueSetSchema
Source code in kiara_plugin/language_processing/modules/tokens.py 
371
372
373
374
375
376
377
378
379
380
def create_outputs_schema(
    self,
) -> ValueSetSchema:

    return {
        "tokens_array": {
            "type": "array",
            "doc": "The pre-processed content, as an array of lists of strings.",
        }
    }
process(inputs: ValueMap, outputs: ValueMap)
Source code in kiara_plugin/language_processing/modules/tokens.py 
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
def process(self, inputs: ValueMap, outputs: ValueMap):

    import polars as pl
    import pyarrow as pa

    tokens_array: KiaraArray = inputs.get_value_data("tokens_array")
    lowercase: bool = inputs.get_value_data("to_lowercase")
    remove_alphanumeric: bool = inputs.get_value_data("remove_alphanumeric")
    remove_non_alpha: bool = inputs.get_value_data("remove_non_alpha")
    remove_all_numeric: bool = inputs.get_value_data("remove_all_numeric")
    remove_short_tokens: int = inputs.get_value_data("remove_short_tokens")

    if remove_short_tokens is None:
        remove_short_tokens = -1

    _remove_stopwords = inputs.get_value_obj("remove_stopwords")
    if _remove_stopwords.is_set:
        stopword_list: Union[Iterable[str], None] = _remove_stopwords.data.list_data
    else:
        stopword_list = None

    # it's better to have one method every token goes through, then do every test seperately for the token list
    # because that way each token only needs to be touched once (which is more effective)
    def check_token(token: str) -> Union[str, None]:

        # remove short tokens first, since we can save ourselves all the other checks (which are more expensive)
        assert isinstance(remove_short_tokens, int)
        if remove_short_tokens > 0:
            if len(token) <= remove_short_tokens:
                return None

        _token: str = token
        if lowercase:
            _token = _token.lower()

        if remove_non_alpha:
            match = _token if _token.isalpha() else None
            if match is None:
                return None

        # if remove_non_alpha was set, we don't need to worry about tokens that include numbers, since they are already filtered out
        if remove_alphanumeric and not remove_non_alpha:
            match = _token if _token.isalnum() else None
            if match is None:
                return None

        # all-number tokens are already filtered out if the remove_non_alpha methods above ran
        if remove_all_numeric and not remove_non_alpha:
            match = None if _token.isdigit() else _token
            if match is None:
                return None

        if stopword_list and _token and _token.lower() in stopword_list:
            return None

        return _token

    series = pl.Series(name="tokens", values=tokens_array.arrow_array)
    result = series.apply(
        lambda token_list: [
            x for x in (check_token(token) for token in token_list) if x is not None
        ]
    )
    result_array = result.to_arrow()

    # TODO: remove this cast once the array data type can handle non-chunked arrays
    chunked = pa.chunked_array(result_array)
    outputs.set_values(tokens_array=chunked)

Functions

get_stopwords()

Source code in kiara_plugin/language_processing/modules/tokens.py 
17
18
19
20
21
22
23
24
25
26
27
28
29
def get_stopwords():

    # TODO: make that smarter
    import nltk

    output = io.StringIO()
    nltk.download("punkt", print_error_to=output)
    nltk.download("stopwords", print_error_to=output)

    log.debug("external.message", source="nltk", msg=output.getvalue())
    from nltk.corpus import stopwords

    return stopwords