Index

_

__init__() (core_utils.ctlr.article.article.Article method) (core_utils.ctlr.config_dto.ConfigDTO method) (core_utils.ctlr.pipeline.LibraryWrapper method) (core_utils.ctlr.pipeline.PipelineProtocol method) (core_utils.ctlr.pipeline.TreeNode method) (lab_3_generate_by_ngrams.main.BackOffGenerator method) (lab_3_generate_by_ngrams.main.BeamSearcher method) (lab_3_generate_by_ngrams.main.BeamSearchTextGenerator method) (lab_3_generate_by_ngrams.main.GreedyTextGenerator method) (lab_3_generate_by_ngrams.main.NGramLanguageModel method) (lab_3_generate_by_ngrams.main.NGramLanguageModelReader method) (lab_3_generate_by_ngrams.main.TextProcessor method) __n_gram_size (lab_3_generate_by_ngrams.main.NGramLanguageModel attribute) _abc_impl (core_utils.ctlr.pipeline.LibraryWrapper attribute) (core_utils.ctlr.pipeline.PipelineProtocol attribute) _analyzer (core_utils.ctlr.pipeline.LibraryWrapper attribute) _beam_width (lab_3_generate_by_ngrams.main.BeamSearcher attribute) (lab_3_generate_by_ngrams.main.BeamSearchTextGenerator attribute) _bootstrap() (core_utils.ctlr.pipeline.LibraryWrapper method) _conllu_info (core_utils.ctlr.article.article.Article attribute) _date_to_text() (core_utils.ctlr.article.article.Article method) _decode() (lab_3_generate_by_ngrams.main.TextProcessor method) _encoded_corpus (lab_3_generate_by_ngrams.main.NGramLanguageModel attribute) _end_of_word_token (lab_3_generate_by_ngrams.main.TextProcessor attribute) (lab_3_generate_by_ngrams.main.WordProcessor attribute)

_eow_token (lab_3_generate_by_ngrams.main.NGramLanguageModelReader attribute) _extract_n_grams() (lab_3_generate_by_ngrams.main.NGramLanguageModel method) _get_next_token() (lab_3_generate_by_ngrams.main.BackOffGenerator method) (lab_3_generate_by_ngrams.main.BeamSearchTextGenerator method) _is_protocol (core_utils.ctlr.pipeline.LibraryWrapper attribute) (core_utils.ctlr.pipeline.PipelineProtocol attribute) _json_path (lab_3_generate_by_ngrams.main.NGramLanguageModelReader attribute) _language_model (lab_3_generate_by_ngrams.main.BeamSearchTextGenerator attribute) _language_models (lab_3_generate_by_ngrams.main.BackOffGenerator attribute) _model (lab_3_generate_by_ngrams.main.BeamSearcher attribute) (lab_3_generate_by_ngrams.main.GreedyTextGenerator attribute) _n_gram_frequencies (lab_3_generate_by_ngrams.main.NGramLanguageModel attribute) _postprocess_decoded_text() (lab_3_generate_by_ngrams.main.TextProcessor method) (lab_3_generate_by_ngrams.main.WordProcessor method) _put() (lab_3_generate_by_ngrams.main.TextProcessor method) (lab_3_generate_by_ngrams.main.WordProcessor method) _storage (lab_3_generate_by_ngrams.main.TextProcessor attribute) (lab_3_generate_by_ngrams.main.WordProcessor attribute) _text_processor (lab_3_generate_by_ngrams.main.BackOffGenerator attribute) (lab_3_generate_by_ngrams.main.BeamSearchTextGenerator attribute) (lab_3_generate_by_ngrams.main.GreedyTextGenerator attribute) (lab_3_generate_by_ngrams.main.NGramLanguageModelReader attribute) _tokenize() (lab_3_generate_by_ngrams.main.TextProcessor method) (lab_3_generate_by_ngrams.main.WordProcessor method)

A

analyze() (core_utils.ctlr.pipeline.LibraryWrapper method)

Article (class in core_utils.ctlr.article.article) ArtifactType (class in core_utils.ctlr.article.article)

B

BackOffGenerator (class in lab_3_generate_by_ngrams.main) beam_searcher (lab_3_generate_by_ngrams.main.BeamSearchTextGenerator attribute)

BeamSearcher (class in lab_3_generate_by_ngrams.main) BeamSearchTextGenerator (class in lab_3_generate_by_ngrams.main) build() (lab_3_generate_by_ngrams.main.NGramLanguageModel method)

C

calculate_bleu() (in module lab_2_tokenize_by_bpe.main) calculate_frequencies() (in module lab_1_classify_profile.main) calculate_geo_mean() (in module lab_2_tokenize_by_bpe.main) calculate_mse() (in module lab_1_classify_profile.main) calculate_precision() (in module lab_2_tokenize_by_bpe.main) check_profile() (in module lab_1_classify_profile.main) children (core_utils.ctlr.pipeline.TreeNode attribute) CLEANED (core_utils.ctlr.article.article.ArtifactType attribute) collect_frequencies() (in module lab_2_tokenize_by_bpe.main) collect_ngrams() (in module lab_2_tokenize_by_bpe.main) collect_profiles() (in module lab_1_classify_profile.main) compare_profiles_by_mse() (in module lab_1_classify_profile.main) compare_profiles_by_top_n() (in module lab_1_classify_profile.main)

ConfigDTO (class in core_utils.ctlr.config_dto) continue_sequence() (lab_3_generate_by_ngrams.main.BeamSearcher method) core_utils.ctlr.article.article module core_utils.ctlr.article.io module core_utils.ctlr.config_dto module core_utils.ctlr.pipeline module core_utils.ctlr.visualizer module count_tokens_pairs() (in module lab_2_tokenize_by_bpe.main) create_language_profile() (in module lab_1_classify_profile.main)

D

date (core_utils.ctlr.article.article.Article attribute) date_from_meta() (in module core_utils.ctlr.article.article) decode() (in module lab_2_tokenize_by_bpe.main) (lab_3_generate_by_ngrams.main.TextProcessor method)

detect_language_advanced() (in module lab_1_classify_profile.main) detect_language_by_mse() (in module lab_1_classify_profile.main) detect_language_by_top_n() (in module lab_1_classify_profile.main)

E

encode() (in module lab_2_tokenize_by_bpe.main) (lab_3_generate_by_ngrams.main.TextProcessor method)

encoding (core_utils.ctlr.config_dto.ConfigDTO attribute)

F

fill_from_ngrams() (lab_3_generate_by_ngrams.main.TextProcessor method) FreqDictType (in module lab_1_classify_profile.main)

from_conllu() (core_utils.ctlr.pipeline.LibraryWrapper method) from_meta() (in module core_utils.ctlr.article.io) from_raw() (in module core_utils.ctlr.article.io)

G

generate_next_token() (lab_3_generate_by_ngrams.main.NGramLanguageModel method) get_article_id_from_filepath() (in module core_utils.ctlr.article.article) get_cleaned_text() (core_utils.ctlr.article.article.Article method) get_conllu_info() (core_utils.ctlr.article.article.Article method) get_conllu_text() (core_utils.ctlr.article.article.Article method) get_end_of_word_token() (lab_3_generate_by_ngrams.main.TextProcessor method) get_file_path() (core_utils.ctlr.article.article.Article method) get_id() (lab_3_generate_by_ngrams.main.TextProcessor method) get_meta() (core_utils.ctlr.article.article.Article method) get_meta_file_path() (core_utils.ctlr.article.article.Article method)

get_n_gram_size() (lab_3_generate_by_ngrams.main.NGramLanguageModel method) get_next_token() (lab_3_generate_by_ngrams.main.BeamSearcher method) get_pos_freq() (core_utils.ctlr.article.article.Article method) get_raw_text() (core_utils.ctlr.article.article.Article method) get_raw_text_path() (core_utils.ctlr.article.article.Article method) get_text_processor() (lab_3_generate_by_ngrams.main.NGramLanguageModelReader method) get_token() (lab_3_generate_by_ngrams.main.TextProcessor method) get_top_n_words() (in module lab_1_classify_profile.main) get_vocabulary() (in module lab_2_tokenize_by_bpe.main) GreedyTextGenerator (class in lab_3_generate_by_ngrams.main)

H

headers (core_utils.ctlr.config_dto.ConfigDTO attribute)

headless_mode (core_utils.ctlr.config_dto.ConfigDTO attribute)

L

lab_1_classify_profile.main module lab_2_tokenize_by_bpe.main module lab_3_generate_by_ngrams.main module

LibraryWrapper (class in core_utils.ctlr.pipeline) load() (lab_3_generate_by_ngrams.main.NGramLanguageModelReader method) load_profile() (in module lab_1_classify_profile.main) load_vocabulary() (in module lab_2_tokenize_by_bpe.main)

M

merge_tokens() (in module lab_2_tokenize_by_bpe.main) module core_utils.ctlr.article.article core_utils.ctlr.article.io core_utils.ctlr.config_dto core_utils.ctlr.pipeline core_utils.ctlr.visualizer lab_1_classify_profile.main lab_2_tokenize_by_bpe.main lab_3_generate_by_ngrams.main

N

N-граммы

NGramLanguageModel (class in lab_3_generate_by_ngrams.main) NGramLanguageModelReader (class in lab_3_generate_by_ngrams.main)

P

PipelineProtocol (class in core_utils.ctlr.pipeline) prepare_word() (in module lab_2_tokenize_by_bpe.main)

print_report() (in module lab_1_classify_profile.main) ProfileType (in module lab_1_classify_profile.main) prune_sequence_candidates() (lab_3_generate_by_ngrams.main.BeamSearcher method)

R

remove_stop_words() (in module lab_1_classify_profile.main) run() (core_utils.ctlr.pipeline.PipelineProtocol method) (lab_3_generate_by_ngrams.main.BackOffGenerator method) (lab_3_generate_by_ngrams.main.BeamSearchTextGenerator method) (lab_3_generate_by_ngrams.main.GreedyTextGenerator method)

S

save_profile() (in module lab_1_classify_profile.main) seed_urls (core_utils.ctlr.config_dto.ConfigDTO attribute) set_conllu_info() (core_utils.ctlr.article.article.Article method) set_n_grams() (lab_3_generate_by_ngrams.main.NGramLanguageModel method)

set_patterns_info() (core_utils.ctlr.article.article.Article method) set_pos_info() (core_utils.ctlr.article.article.Article method) should_verify_certificate (core_utils.ctlr.config_dto.ConfigDTO attribute) show_graph() (in module core_utils.ctlr.visualizer) split_by_sentence() (in module core_utils.ctlr.article.article)

T

text (core_utils.ctlr.pipeline.TreeNode attribute) TextProcessor (class in lab_3_generate_by_ngrams.main) timeout (core_utils.ctlr.config_dto.ConfigDTO attribute) to_cleaned() (in module core_utils.ctlr.article.io) to_conllu() (core_utils.ctlr.pipeline.LibraryWrapper method) to_meta() (in module core_utils.ctlr.article.io)

to_raw() (in module core_utils.ctlr.article.io) tokenize() (in module lab_1_classify_profile.main) tokenize_word() (in module lab_2_tokenize_by_bpe.main) total_articles (core_utils.ctlr.config_dto.ConfigDTO attribute) train() (in module lab_2_tokenize_by_bpe.main) TreeNode (class in core_utils.ctlr.pipeline)

U

UDPIPE_CONLLU (core_utils.ctlr.article.article.ArtifactType attribute)

upos (core_utils.ctlr.pipeline.TreeNode attribute)

V

visualize() (in module core_utils.ctlr.visualizer)

W

WordProcessor (class in lab_3_generate_by_ngrams.main)

И

Идентификатор токена

С

Словарь токенов (vocabulary)

Слово

Т

Токен

Ч

Частота слова