loop example problem

kayman

I am a bit stuck with the loop example operator, so if someone can tell me what I am doing wrong here it would be great.

I have a example set with urls, and each url should be crawled and stored, but I am only able to fetch the last one. Find below simplified process :

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="6.4.000">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="6.4.000" expanded="true" name="Process">
    <process expanded="true">
      <operator activated="true" class="retrieve" compatibility="6.4.000" expanded="true" height="60" name="Retrieve tv" width="90" x="45" y="30">
        <parameter key="repository_entry" value="crawler/links"/>
      </operator>
      <operator activated="true" class="loop_examples" compatibility="6.4.000" expanded="true" height="76" name="Loop Examples (2)" width="90" x="179" y="30">
        <parameter key="iteration_macro" value="e"/>
        <process expanded="true">
          <operator activated="true" class="extract_macro" compatibility="6.4.000" expanded="true" height="60" name="Extract Macro (2)" width="90" x="112" y="30">
            <parameter key="macro" value="href"/>
            <parameter key="macro_type" value="data_value"/>
            <parameter key="attribute_name" value="href"/>
            <parameter key="example_index" value="%{e}"/>
            <list key="additional_macros">
              <parameter key="date" value="date"/>
            </list>
          </operator>
          <operator activated="true" class="generate_attributes" compatibility="6.4.000" expanded="true" height="76" name="Generate Attributes (2)" width="90" x="246" y="30">
            <list key="function_descriptions">
              <parameter key="link" value="macro(&quot;href&quot;)"/>
              <parameter key="index" value="macro(&quot;e&quot;)"/>
            </list>
          </operator>
          <connect from_port="example set" to_op="Extract Macro (2)" to_port="example set"/>
          <connect from_op="Extract Macro (2)" from_port="example set" to_op="Generate Attributes (2)" to_port="example set input"/>
          <connect from_op="Generate Attributes (2)" from_port="example set output" to_port="example set"/>
          <portSpacing port="source_example set" spacing="0"/>
          <portSpacing port="sink_example set" spacing="0"/>
          <portSpacing port="sink_output 1" spacing="0"/>
        </process>
      </operator>
      <connect from_op="Retrieve tv" from_port="output" to_op="Loop Examples (2)" to_port="example set"/>
      <connect from_op="Loop Examples (2)" from_port="example set" to_port="result 1"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
    </process>
  </operator>
</process>

running this will give me my original url, but for every example I get the last value in the example set, where I was expecting the value for the given index (macro value e in this example)

What am I overlooking ? How can I get the actual attribute value of the current itteration ?

MartinLiebig

Hi,

this one should work better:

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="7.1.000">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="7.1.000" expanded="true" name="Process">
    <process expanded="true">
      <operator activated="false" class="retrieve" compatibility="7.1.000" expanded="true" height="68" name="Retrieve tv" width="90" x="45" y="30">
        <parameter key="repository_entry" value="crawler/links"/>
      </operator>
      <operator activated="true" class="retrieve" compatibility="7.1.000" expanded="true" height="68" name="Retrieve Golf" width="90" x="112" y="85">
        <parameter key="repository_entry" value="//Samples/data/Golf"/>
      </operator>
      <operator activated="true" class="loop_examples" compatibility="7.1.000" expanded="true" height="103" name="Loop Examples (2)" width="90" x="313" y="34">
        <parameter key="iteration_macro" value="e"/>
        <process expanded="true">
          <operator activated="true" class="filter_example_range" compatibility="7.1.000" expanded="true" height="82" name="Filter Example Range" width="90" x="45" y="136">
            <parameter key="first_example" value="%{e}"/>
            <parameter key="last_example" value="%{e}"/>
          </operator>
          <operator activated="true" class="extract_macro" compatibility="7.1.000" expanded="true" height="68" name="Extract Macro (2)" width="90" x="179" y="187">
            <parameter key="macro" value="href"/>
            <parameter key="macro_type" value="data_value"/>
            <parameter key="attribute_name" value="Play"/>
            <parameter key="example_index" value="1"/>
            <list key="additional_macros">
              <parameter key="date" value="Humidity"/>
            </list>
          </operator>
          <operator activated="true" class="generate_attributes" compatibility="6.4.000" expanded="true" height="82" name="Generate Attributes (2)" width="90" x="380" y="238">
            <list key="function_descriptions">
              <parameter key="link" value="macro(&quot;href&quot;)"/>
              <parameter key="index" value="macro(&quot;e&quot;)"/>
            </list>
          </operator>
          <connect from_port="example set" to_op="Filter Example Range" to_port="example set input"/>
          <connect from_op="Filter Example Range" from_port="example set output" to_op="Extract Macro (2)" to_port="example set"/>
          <connect from_op="Filter Example Range" from_port="original" to_port="example set"/>
          <connect from_op="Extract Macro (2)" from_port="example set" to_op="Generate Attributes (2)" to_port="example set input"/>
          <connect from_op="Generate Attributes (2)" from_port="example set output" to_port="output 1"/>
          <portSpacing port="source_example set" spacing="0"/>
          <portSpacing port="sink_example set" spacing="0"/>
          <portSpacing port="sink_output 1" spacing="0"/>
          <portSpacing port="sink_output 2" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="append" compatibility="7.1.000" expanded="true" height="82" name="Append" width="90" x="514" y="34"/>
      <connect from_op="Retrieve Golf" from_port="output" to_op="Loop Examples (2)" to_port="example set"/>
      <connect from_op="Loop Examples (2)" from_port="output 1" to_op="Append" to_port="example set 1"/>
      <connect from_op="Append" from_port="merged set" to_port="result 1"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
    </process>
  </operator>
</process>

Loop Examples is giving you the full example set each time. The upper most port is given to you in the next iteration. The lower ports are the output ports.

~Martin