Replacing a missing value with the mean average, but wait there's more....

juanV

I was using the "replace missing values" operator to create an average of the missing value but what it does it calculates the average of the entire row. But what I wanted to do is replace the missing value from Rent and Sqft with the average based on area and Unit Type. I don't know if I'm using the operator wrong or is there another one I should be using. I provided the data below not to worry this is data from apartments.com so there is no privacy issue to worry about. I have a background from Biovia Pipeline Pilot, I am learning the rapid miner way now, bare with me.

yyhuang

Hi @juanV,

Do you want to try "impute missing values" operator for the missing values in Rent and Sqft?

You will impute the missing Rent based on the nearest neighbors (k=5) non-missing Rent.

Sample process below:

<?xml version="1.0" encoding="UTF-8"?><process version="9.4.000-BETA2">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="9.4.000-BETA2" expanded="true" name="Process">
    <parameter key="logverbosity" value="init"/>
    <parameter key="random_seed" value="2001"/>
    <parameter key="send_mail" value="never"/>
    <parameter key="notification_email" value="yhuang@rapidminer.com"/>
    <parameter key="process_duration_for_mail" value="1"/>
    <parameter key="encoding" value="SYSTEM"/>
    <process expanded="true">
      <operator activated="true" class="retrieve" compatibility="9.4.000-BETA2" expanded="true" height="68" name="Retrieve example_data" width="90" x="112" y="34">
        <parameter key="repository_entry" value="//demo/AutoModel/example_data"/>
      </operator>
      <operator activated="true" class="replace_missing_values" compatibility="9.4.000-BETA2" expanded="true" height="103" name="Replace Missing Values" width="90" x="313" y="34">
        <parameter key="return_preprocessing_model" value="false"/>
        <parameter key="create_view" value="false"/>
        <parameter key="attribute_filter_type" value="all"/>
        <parameter key="attribute" value=""/>
        <parameter key="attributes" value=""/>
        <parameter key="use_except_expression" value="false"/>
        <parameter key="value_type" value="attribute_value"/>
        <parameter key="use_value_type_exception" value="false"/>
        <parameter key="except_value_type" value="time"/>
        <parameter key="block_type" value="attribute_block"/>
        <parameter key="use_block_type_exception" value="false"/>
        <parameter key="except_block_type" value="value_matrix_row_start"/>
        <parameter key="invert_selection" value="false"/>
        <parameter key="include_special_attributes" value="false"/>
        <parameter key="default" value="average"/>
        <list key="columns"/>
      </operator>
      <operator activated="true" class="multiply" compatibility="9.4.000-BETA2" expanded="true" height="103" name="Multiply" width="90" x="514" y="85"/>
      <operator activated="true" class="declare_missing_value" compatibility="9.4.000-BETA2" expanded="true" height="82" name="Declare Missing Value" width="90" x="648" y="136">
        <parameter key="attribute_filter_type" value="single"/>
        <parameter key="attribute" value="SqFt"/>
        <parameter key="attributes" value=""/>
        <parameter key="use_except_expression" value="false"/>
        <parameter key="value_type" value="attribute_value"/>
        <parameter key="use_value_type_exception" value="false"/>
        <parameter key="except_value_type" value="time"/>
        <parameter key="block_type" value="attribute_block"/>
        <parameter key="use_block_type_exception" value="false"/>
        <parameter key="except_block_type" value="value_matrix_row_start"/>
        <parameter key="invert_selection" value="false"/>
        <parameter key="include_special_attributes" value="false"/>
        <parameter key="mode" value="numeric"/>
        <parameter key="numeric_value" value="1.0"/>
        <parameter key="expression_value" value=""/>
        <description align="center" color="transparent" colored="false" width="126">sqft = 1?</description>
      </operator>
      <operator activated="true" class="declare_missing_value" compatibility="9.4.000-BETA2" expanded="true" height="82" name="Declare Missing Value (2)" width="90" x="782" y="136">
        <parameter key="attribute_filter_type" value="single"/>
        <parameter key="attribute" value="SqFt"/>
        <parameter key="attributes" value=""/>
        <parameter key="use_except_expression" value="false"/>
        <parameter key="value_type" value="attribute_value"/>
        <parameter key="use_value_type_exception" value="false"/>
        <parameter key="except_value_type" value="time"/>
        <parameter key="block_type" value="attribute_block"/>
        <parameter key="use_block_type_exception" value="false"/>
        <parameter key="except_block_type" value="value_matrix_row_start"/>
        <parameter key="invert_selection" value="false"/>
        <parameter key="include_special_attributes" value="false"/>
        <parameter key="mode" value="numeric"/>
        <parameter key="numeric_value" value="10.0"/>
        <parameter key="expression_value" value=""/>
        <description align="center" color="transparent" colored="false" width="126">sqft =10?</description>
      </operator>
      <operator activated="true" class="impute_missing_values" compatibility="9.4.000-BETA2" expanded="true" height="68" name="Impute Missing Values" width="90" x="916" y="136">
        <parameter key="attribute_filter_type" value="subset"/>
        <parameter key="attribute" value=""/>
        <parameter key="attributes" value="Rent|SqFt"/>
        <parameter key="use_except_expression" value="false"/>
        <parameter key="value_type" value="attribute_value"/>
        <parameter key="use_value_type_exception" value="false"/>
        <parameter key="except_value_type" value="time"/>
        <parameter key="block_type" value="attribute_block"/>
        <parameter key="use_block_type_exception" value="false"/>
        <parameter key="except_block_type" value="value_matrix_row_start"/>
        <parameter key="invert_selection" value="false"/>
        <parameter key="include_special_attributes" value="false"/>
        <parameter key="iterate" value="true"/>
        <parameter key="learn_on_complete_cases" value="true"/>
        <parameter key="order" value="chronological"/>
        <parameter key="sort" value="ascending"/>
        <parameter key="use_local_random_seed" value="false"/>
        <parameter key="local_random_seed" value="1992"/>
        <process expanded="true">
          <operator activated="true" class="k_nn" compatibility="9.4.000-BETA2" expanded="true" height="82" name="k-NN" width="90" x="246" y="34">
            <parameter key="k" value="5"/>
            <parameter key="weighted_vote" value="true"/>
            <parameter key="measure_types" value="MixedMeasures"/>
            <parameter key="mixed_measure" value="MixedEuclideanDistance"/>
            <parameter key="nominal_measure" value="NominalDistance"/>
            <parameter key="numerical_measure" value="EuclideanDistance"/>
            <parameter key="divergence" value="GeneralizedIDivergence"/>
            <parameter key="kernel_type" value="radial"/>
            <parameter key="kernel_gamma" value="1.0"/>
            <parameter key="kernel_sigma1" value="1.0"/>
            <parameter key="kernel_sigma2" value="0.0"/>
            <parameter key="kernel_sigma3" value="2.0"/>
            <parameter key="kernel_degree" value="3.0"/>
            <parameter key="kernel_shift" value="1.0"/>
            <parameter key="kernel_a" value="1.0"/>
            <parameter key="kernel_b" value="0.0"/>
          </operator>
          <connect from_port="example set source" to_op="k-NN" to_port="training set"/>
          <connect from_op="k-NN" from_port="model" to_port="model sink"/>
          <portSpacing port="source_example set source" spacing="0"/>
          <portSpacing port="sink_model sink" spacing="0"/>
        </process>
      </operator>
      <connect from_op="Retrieve example_data" from_port="output" to_op="Replace Missing Values" to_port="example set input"/>
      <connect from_op="Replace Missing Values" from_port="example set output" to_port="result 1"/>
      <connect from_op="Replace Missing Values" from_port="original" to_op="Multiply" to_port="input"/>
      <connect from_op="Multiply" from_port="output 1" to_port="result 3"/>
      <connect from_op="Multiply" from_port="output 2" to_op="Declare Missing Value" to_port="example set input"/>
      <connect from_op="Declare Missing Value" from_port="example set output" to_op="Declare Missing Value (2)" to_port="example set input"/>
      <connect from_op="Declare Missing Value (2)" from_port="example set output" to_op="Impute Missing Values" to_port="example set in"/>
      <connect from_op="Impute Missing Values" from_port="example set out" to_port="result 2"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
      <portSpacing port="sink_result 3" spacing="0"/>
      <portSpacing port="sink_result 4" spacing="0"/>
    </process>
  </operator>
</process>

yyhuang

Just noticed in the data, Sqft has very low values, 1 or 10. I added the "declare missing values" before the imputation.