The Altair Community is migrating to a new platform to provide a better experience for you. In preparation for the migration, the Altair Community is on read-only mode from October 28 - November 6, 2024. Technical support via cases will continue to work as is. For any urgent requests from Students/Faculty members, please submit the form linked here
LDA results (number of topics and words per topic)
Contributor II
Hi, I am trying to extract topics from files and have set the number of topics to be extracted to 50 and the top words per topic to 20 (defaults are 10 and 5 respectively). But when I run the process I get about 90 topics (that also varies with different runs) and more than 50 words per topic.
I am not sure what I'm doing wrong. Attaching the process here - appreciate the help!
Edit: Now I realized the source of the problem. The parent process "Optimize parameters" has option to set max number of topics and words per topic as the chosen parameters. I was setting their values in the LDA operator and forgetting the ones in the parent operator, which seems to override the LDA values.
Regards,
Aya
I am not sure what I'm doing wrong. Attaching the process here - appreciate the help!
Edit: Now I realized the source of the problem. The parent process "Optimize parameters" has option to set max number of topics and words per topic as the chosen parameters. I was setting their values in the LDA operator and forgetting the ones in the parent operator, which seems to override the LDA values.
Regards,
Aya
<?xml version="1.0" encoding="UTF-8"?><process version="9.10.011">
<context>
<input/>
<output/>
<macros/>
</context>
<operator activated="true" class="process" compatibility="9.10.011" expanded="true" name="Process">
<parameter key="logverbosity" value="init"/>
<parameter key="random_seed" value="2001"/>
<parameter key="send_mail" value="never"/>
<parameter key="notification_email" value=""/>
<parameter key="process_duration_for_mail" value="30"/>
<parameter key="encoding" value="SYSTEM"/>
<process expanded="true">
<operator activated="true" class="concurrency:loop_files" compatibility="9.10.011" expanded="true" height="82" name="Loop Files" width="90" x="112" y="34">
<parameter key="directory" value="/Users/ayari88/Documents/Research/AFA/ROBOT/Transcriptions"/>
<parameter key="filter_type" value="regex"/>
<parameter key="filter_by_regex" value=".*\.docx$"/>
<parameter key="recursive" value="true"/>
<parameter key="skip_inaccessible" value="true"/>
<parameter key="enable_macros" value="false"/>
<parameter key="macro_for_file_name" value="file_name"/>
<parameter key="macro_for_file_type" value="file_type"/>
<parameter key="macro_for_folder_name" value="folder_name"/>
<parameter key="reuse_results" value="false"/>
<parameter key="enable_parallel_execution" value="true"/>
<process expanded="true">
<operator activated="true" class="operator_toolbox:read_word_files" compatibility="2.14.000" expanded="true" height="68" name="Read Office File" width="90" x="380" y="34">
<parameter key="detect_file_type" value="false"/>
<parameter key="file_extension" value="docx"/>
</operator>
<connect from_port="file object" to_op="Read Office File" to_port="file"/>
<connect from_op="Read Office File" from_port="doc" to_port="output 1"/>
<portSpacing port="source_file object" spacing="0"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="sink_output 1" spacing="0"/>
<portSpacing port="sink_output 2" spacing="0"/>
</process>
</operator>
<operator activated="true" class="loop_collection" compatibility="9.10.011" expanded="true" height="82" name="Loop Collection" width="90" x="313" y="34">
<parameter key="set_iteration_macro" value="false"/>
<parameter key="macro_name" value="iteration"/>
<parameter key="macro_start_value" value="1"/>
<parameter key="unfold" value="false"/>
<process expanded="true">
<operator activated="true" class="text:tokenize" compatibility="9.4.000" expanded="true" height="68" name="Tokenize" width="90" x="112" y="34">
<parameter key="mode" value="non letters"/>
<parameter key="characters" value=".:"/>
<parameter key="language" value="English"/>
<parameter key="max_token_length" value="3"/>
</operator>
<operator activated="true" class="text:transform_cases" compatibility="9.4.000" expanded="true" height="68" name="Transform Cases" width="90" x="246" y="34">
<parameter key="transform_to" value="lower case"/>
</operator>
<operator activated="false" class="text:documents_to_data" compatibility="9.4.000" expanded="true" height="68" name="Documents to Data" width="90" x="313" y="442">
<parameter key="add_meta_information" value="true"/>
<parameter key="datamanagement" value="double_sparse_array"/>
<parameter key="data_management" value="auto"/>
<parameter key="use_processed_text" value="false"/>
</operator>
<operator activated="false" class="text:json_to_data" compatibility="9.4.000" expanded="true" height="68" name="JSON To Data" width="90" x="179" y="442">
<parameter key="ignore_arrays" value="false"/>
<parameter key="limit_attributes" value="false"/>
<parameter key="skip_invalid_documents" value="false"/>
<parameter key="guess_data_types" value="true"/>
<parameter key="keep_missing_attributes" value="false"/>
<parameter key="missing_values_aliases" value=", null, NaN, missing"/>
</operator>
<operator activated="false" class="read_csv" compatibility="9.10.011" expanded="true" height="68" name="Read CSV" width="90" x="447" y="442">
<parameter key="column_separators" value=";"/>
<parameter key="trim_lines" value="false"/>
<parameter key="use_quotes" value="true"/>
<parameter key="quotes_character" value="""/>
<parameter key="escape_character" value="\"/>
<parameter key="skip_comments" value="false"/>
<parameter key="comment_characters" value="#"/>
<parameter key="starting_row" value="1"/>
<parameter key="parse_numbers" value="true"/>
<parameter key="decimal_character" value="."/>
<parameter key="grouped_digits" value="false"/>
<parameter key="grouping_character" value=","/>
<parameter key="infinity_representation" value=""/>
<parameter key="date_format" value=""/>
<parameter key="first_row_as_names" value="true"/>
<list key="annotations"/>
<parameter key="time_zone" value="SYSTEM"/>
<parameter key="locale" value="English (United States)"/>
<parameter key="encoding" value="SYSTEM"/>
<parameter key="read_all_values_as_polynominal" value="false"/>
<list key="data_set_meta_data_information"/>
<parameter key="read_not_matching_values_as_missings" value="true"/>
</operator>
<operator activated="false" class="retrieve" compatibility="9.10.011" expanded="true" height="68" name="Retrieve" width="90" x="581" y="442">
<parameter key="repository_entry" value="data/ROBOT/sv_stopwords_114"/>
</operator>
<operator activated="true" class="open_file" compatibility="9.10.011" expanded="true" height="68" name="Open File" width="90" x="246" y="136">
<parameter key="resource_type" value="file"/>
<parameter key="filename" value="/Users/ayari88/Documents/Research/AFA/ROBOT/RapidMiner/sv_stopwords_418.csv"/>
</operator>
<operator activated="true" class="text:filter_stopwords_dictionary" compatibility="9.4.000" expanded="true" height="82" name="Filter Stopwords (Dictionary)" width="90" x="380" y="34">
<parameter key="case_sensitive" value="false"/>
<parameter key="encoding" value="SYSTEM"/>
</operator>
<operator activated="true" class="text:stem_snowball" compatibility="9.4.000" expanded="true" height="68" name="Stem (Snowball)" width="90" x="514" y="34">
<parameter key="language" value="Swedish"/>
</operator>
<connect from_port="single" to_op="Tokenize" to_port="document"/>
<connect from_op="Tokenize" from_port="document" to_op="Transform Cases" to_port="document"/>
<connect from_op="Transform Cases" from_port="document" to_op="Filter Stopwords (Dictionary)" to_port="document"/>
<connect from_op="Open File" from_port="file" to_op="Filter Stopwords (Dictionary)" to_port="file"/>
<connect from_op="Filter Stopwords (Dictionary)" from_port="document" to_op="Stem (Snowball)" to_port="document"/>
<connect from_op="Stem (Snowball)" from_port="document" to_port="output 1"/>
<portSpacing port="source_single" spacing="0"/>
<portSpacing port="sink_output 1" spacing="0"/>
<portSpacing port="sink_output 2" spacing="0"/>
</process>
</operator>
<operator activated="true" class="concurrency:optimize_parameters_grid" compatibility="9.10.011" expanded="true" height="166" name="Optimize Parameters (Grid)" width="90" x="514" y="34">
<list key="parameters">
<parameter key="Extract Topics from Documents (LDA).number_of_topics" value="[1.0;100.0;10;linear]"/>
<parameter key="Extract Topics from Documents (LDA).iterations" value="[1.0;100.0;10;linear]"/>
<parameter key="Extract Topics from Documents (LDA).top_words_per_topic" value="[1.0;100.0;10;linear]"/>
</list>
<parameter key="error_handling" value="fail on error"/>
<parameter key="log_performance" value="true"/>
<parameter key="log_all_criteria" value="false"/>
<parameter key="synchronize" value="false"/>
<parameter key="enable_parallel_execution" value="true"/>
<process expanded="true">
<operator activated="true" class="operator_toolbox:lda" compatibility="2.14.000" expanded="true" height="124" name="Extract Topics from Documents (LDA)" width="90" x="179" y="34">
<parameter key="number_of_topics" value="50"/>
<parameter key="show_optimization_settings" value="true"/>
<parameter key="use_alpha_heuristics" value="true"/>
<parameter key="alpha_sum" value="0.1"/>
<parameter key="use_beta_heuristics" value="true"/>
<parameter key="beta" value="0.01"/>
<parameter key="optimize_hyperparameters" value="true"/>
<parameter key="optimize_interval_for_hyperparameters" value="10"/>
<parameter key="iterations" value="1000"/>
<parameter key="top_words_per_topic" value="20"/>
<parameter key="stopword language" value="swedish"/>
<parameter key="reproducible" value="false"/>
<parameter key="enable_logging" value="false"/>
<parameter key="use_local_random_seed" value="false"/>
<parameter key="local_random_seed" value="1992"/>
<parameter key="include_meta_data" value="true"/>
</operator>
<connect from_port="input 1" to_op="Extract Topics from Documents (LDA)" to_port="col"/>
<connect from_op="Extract Topics from Documents (LDA)" from_port="exa" to_port="output 1"/>
<connect from_op="Extract Topics from Documents (LDA)" from_port="top" to_port="output 2"/>
<connect from_op="Extract Topics from Documents (LDA)" from_port="per" to_port="performance"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="source_input 2" spacing="0"/>
<portSpacing port="sink_performance" spacing="0"/>
<portSpacing port="sink_model" spacing="0"/>
<portSpacing port="sink_output 1" spacing="0"/>
<portSpacing port="sink_output 2" spacing="0"/>
<portSpacing port="sink_output 3" spacing="0"/>
</process>
</operator>
<connect from_op="Loop Files" from_port="output 1" to_op="Loop Collection" to_port="collection"/>
<connect from_op="Loop Collection" from_port="output 1" to_op="Optimize Parameters (Grid)" to_port="input 1"/>
<connect from_op="Optimize Parameters (Grid)" from_port="performance" to_port="result 1"/>
<connect from_op="Optimize Parameters (Grid)" from_port="parameter set" to_port="result 2"/>
<connect from_op="Optimize Parameters (Grid)" from_port="output 1" to_port="result 3"/>
<connect from_op="Optimize Parameters (Grid)" from_port="output 2" to_port="result 4"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="sink_result 1" spacing="0"/>
<portSpacing port="sink_result 2" spacing="0"/>
<portSpacing port="sink_result 3" spacing="0"/>
<portSpacing port="sink_result 4" spacing="0"/>
<portSpacing port="sink_result 5" spacing="0"/>
</process>
</operator>
</process>
Tagged:
0
