<?xmlversion="1.0" encoding="US-ASCII"?>version='1.0' encoding='utf-8'?> <!DOCTYPE rfc SYSTEM"rfc2629.dtd" [ <!ENTITY rfc2119 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.2119.xml"> <!ENTITY rfc3550 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.3550.xml"> <!ENTITY rfc3551 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.3551.xml"> <!ENTITY rfc3611 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.3611.xml"> <!ENTITY rfc4585 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.4585.xml"> <!ENTITY rfc5506 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.5506.xml"> <!ENTITY rfc5166 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.5166.xml"> <!ENTITY rfc5033 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.5033.xml"> <!ENTITY rfc5681 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.5681.xml"> <!ENTITY I-D.ietf-rmcat-cc-requirements PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-rmcat-cc-requirements.xml"> <!ENTITY I-D.ietf-avtcore-rtp-circuit-breakers PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-avtcore-rtp-circuit-breakers.xml"> <!ENTITY I-D.ietf-rmcat-eval-criteria PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-rmcat-eval-criteria.xml"> <!ENTITY I-D.ietf-rtcweb-use-cases-and-requirements PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-rtcweb-use-cases-and-requirements.xml"> <!ENTITY I-D.ietf-rmcat-wireless-tests PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-rmcat-wireless-tests.xml"> <!ENTITY I-D.ietf-rmcat-video-traffic-model PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-rmcat-video-traffic-model.xml"> ]> <?xml-stylesheet type='text/xsl' href='rfc2629.xslt' ?> <?rfc toc="yes" ?> <?rfc compact="yes" ?> <?rfc symrefs="yes" ?>"rfc2629-xhtml.ent"> <rfc xmlns:xi="http://www.w3.org/2001/XInclude" submissionType="IETF" category="info" consensus="true" docName="draft-ietf-rmcat-eval-test-10"ipr="trust200902"> <!---->number="8867" ipr="trust200902" obsoletes="" updates="" xml:lang="en" tocInclude="true" symRefs="true" sortRefs="true" version="3"> <!-- xml2rfc v2v3 conversion 2.43.0 --> <front> <title abbrev="Test Scenarios for RMCAT">Test Cases for EvaluatingRMCAT Proposals</title>Congestion Control for Interactive Real-Time Media</title> <seriesInfo name="RFC" value="8867"/> <author fullname="Zaheduzzaman Sarker" initials="Z" surname="Sarker"> <organization>Ericsson AB</organization> <address> <postal> <street>Torshamnsgatan 23</street> <city>Stockholm</city><region>SE</region><code>164 83</code> <country>Sweden</country> </postal> <phone>+46 10 717 37 43</phone> <email>zaheduzzaman.sarker@ericsson.com</email> </address> </author> <author fullname="Varun Singh" initials="V" surname="Singh"> <organizationabbrev="callstats.io">Nemu Dialogue Systemsabbrev="callstats.io">CALLSTATS I/O Oy</organization> <address> <postal><street>Runeberginkatu 4c A 4</street><street>Rauhankatu 11 C</street> <city>Helsinki</city> <code>00100</code> <country>Finland</country> </postal> <email>varun.singh@iki.fi</email> <uri>http://www.callstats.io/</uri> </address> </author> <author fullname="Xiaoqing Zhu" initials="X" surname="Zhu"> <organization>Cisco Systems</organization> <address> <postal> <street>12515 Research Blvd</street><city>Austing</city><city>Austin</city> <region>TX</region> <code>78759</code><country>USA</country><country>United States of America</country> </postal> <email>xiaoqzhu@cisco.com</email> </address> </author> <author fullname="Michael A. Ramalho"initials="M. A."initials="M." surname="Ramalho"> <organizationabbrev="Cisco Systems">Cisco Systems, Inc.</organization>abbrev="AcousticComms">AcousticComms Consulting</organization> <address> <postal> <street>6310 Watercrest Way Unit 203</street> <city>Lakewood Ranch</city> <region>FL</region> <code>34202-5211</code> <country>USA</country> </postal> <phone>+1919 476 2038</phone> <email>mramalho@cisco.com</email>732 832 9723</phone> <email>mar42@cornell.edu</email> <uri>http://ramalho.webhop.info/</uri> </address> </author> <dateday="23" month="May" year="2019"/>month="July" year="2020"/> <area>TSV</area> <keyword>Multimedia</keyword> <keyword>Test cases</keyword> <keyword>Congestion Control</keyword> <abstract> <t>The Real-time Transport Protocol (RTP) is used to transmit media in multimedia telephony applications. These applications are typically required to implement congestion control. This document describes the test cases to be used in the performance evaluation of such congestion control algorithms in a controlled environment.</t> </abstract> </front> <middle> <sectiontitle="Introduction">numbered="true" toc="default"> <name>Introduction</name> <t>This memo describes a set of test cases for evaluating congestion control algorithm proposals in controlled environments for real-time interactive media. It is based on the guidelines enumerated in <xreftarget="I-D.ietf-rmcat-eval-criteria"/>target="RFC8868" format="default"/> and the requirements discussed in <xreftarget="I-D.ietf-rmcat-cc-requirements"/>.target="RFC8836" format="default"/>. The test cases cover basic usage scenarios and are described using a common structure, which allows for additional test cases to be added to those described herein to accommodate other topologies and/or themodellingmodeling of different path characteristics. The described test cases in this memo should be used to evaluate any proposed congestion control algorithm for real-time interactive media.</t> </section> <section anchor="sec-terminology"title="Terminology"> <!--The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in BCP 14, <xref target="RFC2119" /> and indicate requirement levels for compliant implementations.-->numbered="true" toc="default"> <name>Terminology</name> <t>The terminology defined in <xreftarget="RFC3550">RTP</xref>,target="RFC3550" format="default">RTP</xref>, <xreftarget="RFC3551">RTPtarget="RFC3551" format="default">RTP Profile for Audio and Video Conferences with Minimal Control</xref>, <xreftarget="RFC3611">RTCPtarget="RFC3611" format="default">RTCP Extended Report (XR)</xref>, <xreftarget="RFC4585">Extendedtarget="RFC4585" format="default">Extended RTP Profile for RTCP-based Feedback (RTP/AVPF)</xref>, and <xreftarget="RFC5506">Supporttarget="RFC5506" format="default">Support for Reduced-Size RTCP</xref>apply.</t>applies.</t> </section> <section anchor="TS"title="Structurenumbered="true" toc="default"> <name>Structure of Testcases">Cases</name> <t>All the test cases in this document follow a basic structure allowing implementers to describe a new test scenario without repeatedly explaining common attributes. The structure includes a general description section that describes the test case and its motivation. Additionally the test case defines a set of attributes that characterize the testbed, for example, the network path between communicating peers and the diverse traffic sources.</t><t><list style="symbols"> <t>Define<dl spacing="normal"> <dt>Define the testcase: <list style="symbols"> <t>General description: describescase:</dt><dd> <t><br/></t> <dl spacing="normal"> <dt>General description:</dt><dd>describes the motivation and the goals of the testcase.</t> <t>Expected behavior: describescase.</dd> <dt>Expected behavior:</dt><dd>describes the desired rate adaptationbehavior.</t> <t>Define a listbehavior.</dd> <dt>List of metrics to evaluate the desiredbehavior:behavior:</dt><dd> this indicates the minimum set of metrics (e.g., link utilization, media sending rate) that a proposed algorithm needs to measure to validate the expected rate adaptation behavior. It should also indicate the time granularity (e.g., averaged over10ms, 100ms,10 ms, 100 ms, or1s)1 s) for measuring certain metrics. Typical measurement interval is200ms.</t> <!-- we agreed this should be 200ms or was it based on some IPPM doc? --> </list></t> <t>Define200 ms.</dd> </dl> </dd> <dt>Define testbedtopology: everytopology:</dt><dd> <t><br/></t> <t>Every test case needs to define an evaluation testbed topology. <xreftarget="fig-eval-topo"/>target="fig-eval-topo" format="default"/> shows such an evaluation topology. In this evaluation topology, S1..Sn are traffic sources. These sources generate media traffic and use the congestion control algorithm(s) under investigation. R1..Rn are the corresponding receivers. A test case can have one or more such traffic sources (S) and their corresponding receivers (R). The path from the source to destination is denoted as"forward""forward", and the path from a destination to a source is denoted as "backward". The following basic structure of the test case has been described from the perspective ofmedia generatingmedia-generating endpoints attached on the left-hand side of <xreftarget="fig-eval-topo"/>.target="fig-eval-topo" format="default"/>. In this setup, the media flows are transported in the forwarddirectiondirection, and the corresponding feedback/control messages are transported in the backward direction. However, it is also possible to set up the test with media in both forward and backward directions. In that case, unless otherwise specified by the test case, it is expected that the backward path does not introduce anycongestion relatedcongestion-related impairments and has enough capacity to accommodate both media and feedback/control messages. It should be notedthatthat, depending on the testcasescases, it is possible to have different path characteristics in either of thedirections.<figure anchor="fig-eval-topo" title="Exampledirections.</t> <figure anchor="fig-eval-topo"> <name>Example ofAa TestbedTopology"> <artwork><![CDATA[Topology</name> <artwork name="" type="" align="left" alt=""><![CDATA[ +---+ +---+ |S1 |====== \ Forward --> / =======|R1 | +---+ \\ // +---+ \\ // +---+ +-----+ +-----+ +---+ |S2 |=======| A|------------------------------>||--------------------------->| B |=======|R2 | +---+ ||<------------------------------||<---------------------------| | +---+ +-----+ +-----+ (...) // \\ (...) // <-- Backward \\ +---+ // \\ +---+ |Sn |====== / \ ======|Rn | +---+ +---+ ]]></artwork></figure>In</figure> <t>In a testbed environment with realequipments,equipment, there may exist a significant amount of unwanted traffic on the portions of the network path between the endpoints. Some of this traffic may be generated by other processes on the endpoints themselves (e.g., discovery protocols) or by other endpoints not presently under test. Such unwanted traffic should be removed or avoided to the greatest extent possible.</t><t>Define</dd> <dt>Define testbedattributes: <list style="symbols"> <t>Duration: definesattributes:</dt> <dd> <t><br/></t> <dl spacing="normal"> <dt>Duration:</dt><dd>defines the duration of the test inseconds.</t> <t>Path characteristics: definesseconds.</dd> <dt>Path characteristics:</dt><dd> <t>defines the end-to-end transport level path characteristics of the testbed for a particular test case. Two sets of attributes describe the path characteristics, one for the forward path and the other for the backward path. The path characteristics for a particular path directionisare applicable to all theSourcessources "S" sending traffic on that path. If only one attribute is specified, it is used for both pathdirections,directions; however, unless specified the reverse path has no capacity restrictions and no pathloss.<list style="symbols"> <t>Path direction: forwardloss.</t> <dl spacing="normal"> <dt>Path direction:</dt><dd>forward orbackward.</t> <t>Minimumbackward.</dd> <dt>Minimum bottleneck-linkcapacity: definescapacity:</dt><dd>defines the minimum capacity of the end-to-endpath</t> <t>Referencepath.</dd> <dt>Reference bottleneckcapacity: definescapacity:</dt><dd>defines a reference value for the bottleneck capacity for test cases with time-varying bottleneck capacities. All bottleneck capacities will be specified as a ratio with respect to the reference capacityvalue.</t> <t>One-wayvalue.</dd> <dt>One-way propagationdelay: describesdelay:</dt><dd>describes the end-to-end latency along the path when network queues are empty, i.e., the time it takes for a packet to go from the sender to the receiver without encountering any queuingdelay.</t> <t>Maximumdelay.</dd> <dt>Maximum end-to-endjitter: definesjitter:</dt><dd>defines the maximum jitter that can be observed along thepath.</t> <t>Bottleneckpath.</dd> <dt>Bottleneck queuetype: fortype:</dt><dd>for example, "tail drop" <xreftarget="RFC7567"/>,target="RFC7567" format="default"/>, Flow Queue-CoDel (FQ-CoDel)<xref target="RFC8290"/>,Controlled Delay (FQ-CoDel) <xref target="RFC8290" format="default"/>, or Proportional Integral controller Enhanced(PIE)<xref target="RFC8033"/>.</t> <t>Bottleneck(PIE) <xref target="RFC8033" format="default"/>.</dd> <dt>Bottleneck queuesize: definessize:</dt><dd>defines the size of queue in terms of queuing time when the queue is full (inmilliseconds).</t> <t>Pathmilliseconds).</dd> <dt>Path lossratio: characterizesratio:</dt><dd>characterizes the non-congested,additive,additive losses to be generated on the end-to-end path. This must describe the loss pattern or loss model used to generate thelosses.</t> </list></t> <t>Application-related: defineslosses.</dd> </dl> </dd> <dt>Application-related:</dt><dd> <t>defines the traffic source behavior for implementing the testcase <list style="symbols"> <t>Mediacase:</t> <dl spacing="normal"> <dt>Media trafficSource: definessource:</dt><dd> <t>defines the characteristics of the media sources. When using more than one media source, the different attributes are enumerated separately for each different media source.<list style="symbols"> <t>Media type: Video/Voice</t> <t>Media</t> <dl spacing="normal"> <dt>Media type:</dt><dd>Video/Voice.</dd> <dt>Media flowdirection: forward, backwarddirection:</dt><dd>forward, backward, orboth.</t> <t>Numberboth.</dd> <dt>Number of mediasources: definessources:</dt><dd>defines the total number of mediasources</t> <t>Media codec: Constantsources.</dd> <dt>Media codec:</dt><dd>Constant Bit Rate (CBR) or Variable Bit Rate(VBR)</t> <t>Media(VBR).</dd> <dt>Media sourcebehavior: describesbehavior:</dt><dd> <t>describes the media encoder behavior. It defines the main parameters that affect the adaptation behavior. This may include but is not limitedto: <list style="symbols"> <t>Adaptability: describesto the following: </t> <dl spacing="normal"> <dt>Adaptability:</dt><dd>describes the adaptation options. For example, in the case ofvideovideo, it defines the following ranges of adaptation: bit rate, frame rate, and video resolution. Similarly, in the case of voice, it defines the range of bit rate adaptation, the sampling rate variation, and the variation in packetizationinterval.</t> <t>Output variation : forinterval.</dd> <dt>Output variation:</dt><dd>for a VBRencoderencoder, it defines the encoder output variation from the average target rate over a particular measurement interval. For example, on average the encoder output may vary between 5% to 15% above or below the average target bit rate when measured over a 100 ms time window. The time interval over which the variation is specified must beprovided.</t> <!-- --> <t>Responsivenessprovided.</dd> <dt>Responsiveness to a new bit raterequest: therequest:</dt><dd>the lag in time between a new bit rate request from the congestion control algorithm and actual rate changes in encoder output. Depending on the encoder, this value may be specified in absolute time(e.g. 10ms(e.g., 10 ms to1000ms)1000 ms) or other appropriate metric(e.g.(e.g., next frame intervaltime).</t> </list>time).</dd> </dl> <t> More detailed discussions on expected media source behavior, including those from synthetic video traffic sources,is atcan be found in <xreftarget="I-D.ietf-rmcat-video-traffic-model"/>.</t> <t>Media content: describestarget="RFC8593" format="default"/>.</t> </dd> <dt>Media content:</dt><dd>describes the chosen video scenario. For example, video test sequences are availableat:at <xreftarget="xiph-seq"/>target="xiph-seq" format="default"/> and <xreftarget="HEVC-seq"/>.target="HEVC-seq" format="default"/>. Different video scenarios give differentdistributiondistributions of video frames produced by the video encoder. Hence, it is important to specify the media content used in a particular test. If a synthetic video trafficsoucesource <xreftarget="I-D.ietf-rmcat-video-traffic-model"/>target="RFC8593" format="default"/> is used, then the synthetic video traffic source needs toconfigurebe configured according to the characteristics of the media contentspecified.</t> <t>Media timeline: describesspecified.</dd> <dt>Media timeline:</dt><dd>describes the point when the media source is introduced and removed from the testbed. For example, the media source may start transmitting immediately when the test case begins, or after a fewseconds.</t> <t>Startup behavior: theseconds.</dd> <dt>Startup behavior:</dt><dd>the media starts at a defined bit rate, which may be the minimum, maximum bit rate, or a value in between (inKbps).</t> </list></t> <t>CompetingKbps).</dd> </dl> </dd> <dt>Competing trafficsource: describessource:</dt><dd> <t>describes the characteristics of the competing traffic source, the different types of competing flows are enumerated in <xreftarget="I-D.ietf-rmcat-eval-criteria"/>. <list style="symbols"> <t>Traffic direction: forward, backwardtarget="RFC8868" format="default"/>. </t> <dl spacing="normal"> <dt>Traffic direction:</dt><dd>forward, backward, orboth.</t> <t>Typeboth.</dd> <dt>Type ofsources: definessources:</dt><dd>defines the types of competing traffic sources. Types of competing traffic flows are listed in <xreftarget="I-D.ietf-rmcat-eval-criteria"/>.target="RFC8868" format="default"/>. For example, the number of TCP flows connected to a web browser, the mean size and distribution of the contentdownloaded.</t> <t>Numberdownloaded.</dd> <dt>Number ofsources: definessources:</dt><dd>defines the total number of competing sources of each media type per trafficdirection.</t> <t>Congestion control: enumeratesdirection.</dd> <dt>Congestion control:</dt><dd>enumerates the congestion control used by each type of competingtraffic.</t> <t>Traffic timeline: describestraffic.</dd> <dt>Traffic timeline:</dt><dd>describes when the competing traffic starts and ends in the testcase.</t> </list></t> </list></t> <t>Additional attributes: describescase.</dd> </dl> </dd> </dl> </dd> <dt>Additional attributes:</dt><dd>describes attributes essential for implementing a test casewhichthat are not included in the above structure. These attributes must be well defined, so that the other implementers of that particular test case are able to implement iteasily.</t> </list></t> </list></t>easily.</dd> </dl> </dd> </dl> <t>Any attribute can have a set of values (enclosed within "[]"). Each member value of such a set must be treated as different value for the same attribute. It is desired to run separate tests for each such attribute value.</t> <t>The test cases described in this document follow the above structure.</t> </section> <section anchor="sec-rec-eval"title="Recommendednumbered="true" toc="default"> <name>Recommended EvaluationSettings">Settings</name> <t>This section describes recommended test case settings and could be overwritten by the respective test cases.</t> <section anchor="EM"title="Evaluation metrics">numbered="true" toc="default"> <name>Evaluation Metrics</name> <t>To evaluate the performance of the candidatealgorithmsalgorithms, the implementers must log enough information to visualize the following metrics at a fine enough time granularity:<list style="numbers"></t> <ol spacing="normal" type="1"> <li> <t>Flow level:<list style="letters"> <t>End-to-end</t> <ol spacing="normal" type="A"> <li>End-to-end delay for thecongestion controlledcongestion-controlled media flow(s). Forexample -example, end-to-end delay observed on the IP packetlevel,level and the video framelevel.</t> <t>Variationlevel.</li> <li>Variation in sending bit rate and throughput. Mainly observing the frequency and magnitude ofoscillations.</t> <t>Packetoscillations.</li> <li>Packet losses observed at the receivingendpoint.</t> <t>Feedbackendpoint.</li> <li>Feedback messageoverhead.</t> <t>Convergence time - timeoverhead.</li> <li>Convergence time. Time to reach steady state for thecongestion controlledcongestion-controlled media flow(s). Each occurrence of convergence during the test periodneedneeds to bepresented.</t> </list></t>presented.</li> </ol> </li> <li> <t>Transport level:<list style="letters"> <t>Bandwidth utilization.</t> <t>Queue</t> <ol spacing="normal" type="A"> <li>Bandwidth utilization.</li> <li>Queue length (milliseconds at specified pathcapacity).</t> </list></t> </list></t>capacity).</li> </ol> </li> </ol> </section> <section anchor="PC"title="Path characteristics">numbered="true" toc="default"> <name>Path Characteristics</name> <t>Each path between a sender and receiver as described in <xreftarget="fig-eval-topo"/> havetarget="fig-eval-topo" format="default"/> has the following characteristics unless otherwise specified in the testcase. <list style="symbols"> <t>Pathcase: </t> <dl newline="false" spacing="normal"> <dt>Path direction:forward</dt><dd>forward andbackward.</t> <t>Referencebackward.</dd> <dt>Reference bottleneck capacity:1Mbps.</t> <t>One-Way</dt><dd>1 Mbps.</dd> <dt>One-way propagation delay:50ms.</dt><dd>50 ms. Implementers are encouraged to run the experiment with additional propagation delays mentioned in <xreftarget="I-D.ietf-rmcat-eval-criteria"/></t> <t>Maximumtarget="RFC8868" format="default"/>.</dd> <dt>Maximum end-to-end jitter:30ms.</dt><dd>30 ms. Jitter models are described in <xreftarget="I-D.ietf-rmcat-eval-criteria"/></t> <t>Bottlenecktarget="RFC8868" format="default"/>.</dd> <dt>Bottleneck queue type:"tail</dt><dd>"tail drop". Implementers are encouraged to run the experiment with otherAQMActive Queue Management (AQM) schemes, such as FQ-CoDel andPIE.</t> <t>BottleneckPIE.</dd> <dt>Bottleneck queue size:300ms.</t> <t>Path</dt><dd>300 ms.</dd> <dt>Path loss ratio:0%.</t> </list></t></dt><dd>0%.</dd> </dl> <t>Examples of additional network parameters are discussed in <xreftarget="I-D.ietf-rmcat-eval-criteria"/>.</t>target="RFC8868" format="default"/>.</t> <t>For test cases involving time-varying bottleneck capacity, all capacity values are specified as a ratio with respect to a reference capacity value, so as to allow flexible scaling of capacity values along with media source rate range. There exist two different mechanisms for inducing path capacity variation: a) by explicitly modifying the value of physical linkcapacity;capacity, or b) by introducing background non-adaptive UDP traffic with time-varying traffic rate. Implementers are encouraged to run the experiments with both mechanisms for test cases specified in <xreftarget="VACS"/>,target="VACS" format="default"/>, <xreftarget="VACM"/>,target="VACM" format="default"/>, and <xreftarget="CFL"/>.</t>target="CFL" format="default"/>.</t> </section> <section anchor="MS"title="Media source">numbered="true" toc="default"> <name>Media Source</name> <t>Unless otherwise specified, each test case will include one or more media sources as describedbelow. <list style="symbols"> <t>Media type: Video <list style="symbols"> <t>Media codec: VBR</t> <t>Mediabelow: </t> <dl newline="false" spacing="normal"> <dt>Media type:</dt> <dd> <t>Video</t> <dl newline="false" spacing="normal"> <dt>Media codec:</dt> <dd>VBR</dd> <dt>Media sourcebehavior: <list style="symbols"> <t>Adaptability: <list style="symbols"> <t>Bitbehavior:</dt> <dd> <t><br/></t> <dl newline="false" spacing="normal"> <dt>Adaptability:</dt> <dd> <t><br/></t> <dl newline="false" spacing="normal"> <dt>Bit raterange:range:</dt> <dd> 150 Kbps - 1.5 Mbps. In real-lifeapplicationsapplications, the bit rate range can vary a lot depending on the providedservice,service; for example, the maximum bit rate can be up to4Mbps.4 Mbps. However, for running tests to evaluate the congestion controlalgorithmsalgorithms, it is more important to have a look at how theyare reactingreact to a certain amount of bandwidth change. Also it is possible that the media traffic generator used in a particular simulator or testbed is not capable of generating a higher bit rate.HenceHence, we have selected a suitable bit rate range typical of consumer-grade video conferencing applications in designing the test case. If a different bit rate range is used in the test cases, then the end-to-end path capacity values will also need to be scaledaccordingly.</t> <t>Frameaccordingly.</dd> <dt>Frame resolution:144p</dt> <dd>144p - 720p (or 1080p). This resolution range is selected based on the bit rate range. If a different bit rate range is used in the testcasescases, thenthea suitable frame resolution range alsoneedneeds to beselected suitably.</t> <t>Frameselected.</dd> <dt>Frame rate:10fps</dt> <dd>10 fps -30fps.30 fps. This frame rate range is selected based on the bit rate range. If a different bit rate range is used in the testcasescases, then the frame rate range alsoneedneeds to beadjusted suitably.</t> </list></t> <t>Variationsuitably adjusted.</dd> </dl> </dd> <dt>Variation from target bit rate:+/-5%.</dt> <dd>+/-5%. Unless otherwise specified in the test case(s), bit rate variation should be calculated over a one (1) second period oftime.</t> <t>Responsivenesstime.</dd> <dt>Responsiveness to new bit rate request:100ms</t> </list></t> <t>Media content: The</dt> <dd>100 ms</dd> </dl> </dd> <dt>Media content:</dt> <dd>The media content should represent a typical video conversational scenario with head and shoulder movement. We recommendto useusing the Foreman videosequence<xref target="xiph-seq"/>.</t> <t>Mediasequence <xref target="xiph-seq" format="default"/>.</dd> <dt>Media startupbehavior: 150Kbps.behavior:</dt> <dd>150 Kbps. It should be noted that applications can use smart ways to select an optimal startup bit rate value for a certain network condition. In suchcasescases, the candidate proposals may show the effectiveness of such a smart approach asanadditional information for the evaluationprocess.</t> </list></t> </list><list style="symbols"> <t>Media type: Audio<list style="symbols"> <t>Mediaprocess.</dd> </dl> </dd> <dt>Media type:</dt> <dd> <t>Audio</t> <dl spacing="normal"> <dt>Media codec:CBR</t> <t>Media</dt><dd>CBR</dd> <dt>Media bitrate: 20Kbps</t> </list></t> </list></t>rate:</dt><dd>20 Kbps</dd> </dl> </dd> </dl> </section> </section> <section anchor="TC"title="Basicnumbered="true" toc="default"> <name>Basic TestCases">Cases</name> <section anchor="VACS"title="Variablenumbered="true" toc="default"> <name>Variable Available Capacity with a SingleFlow">Flow</name> <t>In this testcasecase, the minimum bottleneck-link capacity between the two endpoints varies over time. This test is designed to measure the responsiveness of the candidate algorithm. This test tries to address the requirements in <xreftarget="I-D.ietf-rmcat-cc-requirements"/>,target="RFC8836" format="default"/>, which requires the algorithm to adapt the flow(s) and provide lower end-to-end latency when there exists:<list style="symbols"> <t>an</t> <ul spacing="normal"> <li>an intermediatebottleneck</t> <t>changebottleneck</li> <li>change in available capacity (e.g., due to interface change, routing change, abrupt arrival/departure of background non-adaptivetraffic).</t> <t>maximumtraffic).</li> <li>maximum media bit rate is greater than link capacity. In this case, when the application tries to ramp up to its maximum bit rate, since the link capacity is limited to avalue lower,lower value, the congestion control scheme is expected to stabilize the sending bit rate close to the available bottleneckcapacity.</t> <!-- --> </list>Itcapacity.</li> </ul> <t>It should be noted that the exact variation in available capacity due to any of the above depends on the underlying technologies. Hence, we describe a set of known factors, which may be extended to devise a more specific test case targeting certain behaviors in a certain network environment.</t><t>Expected behavior: the<dl spacing="normal"> <dt>Expected behavior:</dt><dd>The candidate algorithm is expected to detect the path capacity constraint, converge to the bottleneck link'scapacitycapacity, and adapt the flow to avoid unwanted media rate oscillation when the sending bit rate is approaching the bottleneck link's capacity. Such oscillations might occur when the media flow(s) attempts to reach its maximum bit rate but overshoots the usage of the available bottleneckcapacitycapacity, then to rectify, it reduces the bit rate and starts to ramp upagain.</t> <t>Evaluation metrics : asagain.</dd> <dt>Evaluation metrics:</dt><dd>As described in <xreftarget="EM"/>.</t> <t>Testbed topology: Onetarget="EM" format="default"/>.</dd> <dt>Testbed topology:</dt><dd>One media source S1 is connected to the corresponding R1. The media traffic is transported over the forward path and corresponding feedback/control traffic is transported over the backward path. </dd> </dl> <figureanchor="fig-eval-topo-4-2" title="Testbedanchor="fig-eval-topo-4-2"> <name>Testbed Topology for Limited LinkCapacity"> <artwork><![CDATA[Capacity</name> <artwork name="" type="" align="left" alt=""><![CDATA[ Forward --> +---+ +-----+ +-----+ +---+ |S1 |=======| A |------------------------------>| B |=======|R1 | +---+ | |<------------------------------| | +---+ +-----+ +-----+ <-- Backward ]]></artwork></figure></t> <t>Testbed attributes:</t> <t><list style="symbols"> <t>Test duration: 100s</t> <t>Path characteristics: as</figure> <dl spacing="normal"> <dt>Testbed attributes:</dt> <dd> <t><br/></t> <dl spacing="normal"> <dt>Test duration:</dt><dd>100 s</dd> <dt>Path characteristics:</dt><dd>as described in <xreftarget="PC"/></t> <t>Application-related: <list style="symbols"> <t>Media Traffic:<list style="symbols"> <t>Media type: Video<list style="symbols"> <t>Media direction: forward.</t> <t>Number of media sources: one (1)</t> <t>Media timeline: <list style="symbols"> <t>Start time: 0s.</t> <t>End time: 99s.</t> </list></t> </list></t> <t>Media type: Audio<list style="symbols"> <t>Media direction: forward.</t> <t>Number of media sources: one (1)</t> <t>Media timeline: <list style="symbols"> <t>Start time: 0s.</t> <t>End time: 99s.</t> </list></t> </list></t> </list></t> <t>Competing traffic: <list style="symbols"> <t>Number of sources : zero (0)</t> </list></t> </list></t> <t>Test Specific Information: <list style="symbols"> <t>One-waytarget="PC" format="default"/></dd> <dt>Application-related:</dt> <dd> <t><br/></t> <dl spacing="normal"> <dt>Media Traffic:</dt> <dd> <t><br/></t> <dl spacing="normal"> <dt>Media type:</dt> <dd> <t>Video</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward</dd> <dt>Number of media sources:</dt><dd>one (1)</dd> <dt>Media timeline:</dt> <dd> <t><br/></t> <dl spacing="normal"> <dt>Start time:</dt><dd>0 s</dd> <dt>End time:</dt><dd>99 s</dd> </dl> </dd> </dl> </dd> <dt>Media type:</dt> <dd> <t>Audio</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward</dd> <dt>Number of media sources:</dt><dd>one (1)</dd> <dt>Media timeline:</dt> <dd> <t><br/></t> <dl spacing="normal"> <dt>Start time:</dt><dd>0 s</dd> <dt>End time:</dt><dd>99 s</dd> </dl> </dd> </dl> </dd> </dl> </dd> <dt>Competing traffic:</dt> <dd> <t><br/></t> <dl spacing="normal"> <dt>Number of sources:</dt><dd>zero (0)</dd> </dl> </dd> </dl> </dd> </dl> </dd> <dt>Test-specific information:</dt> <dd> <t><br/></t> <dl spacing="normal"> <dt>One-way propagationdelay: [ 50delay:</dt> <dd>[50 ms, 100 ms].onOn the forward pathdirection</t>direction.</dd> </dl> <t>This test uses bottleneck path capacity variation as listed in <xreftarget="VACS_US"/></t>target="VACS_US" format="default"/>.</t> <t>When using background non-adaptive UDP traffic to induce a time-varyingbottleneck ,bottleneck, the physical path capacity remains at4Mbps4 Mbps, and the UDP traffic source rate changes over time as (4 - (Y x r)), where r is the Reference bottleneck capacity inMbpsMbps, and Y is the path capacity ratio specified in <xreftarget="VACS_US"/></t> </list></t> </list></t> <texttabletarget="VACS_US" format="default"/>.</t> </dd> </dl> <table anchor="VACS_US"title="Path capacity variation patternalign="center"> <name>Path Capacity Variation Pattern forforward direction"> <ttcol>Variation pattern index</ttcol> <ttcol>Path direction</ttcol> <ttcol>Start time</ttcol> <ttcol>Path capacity ratio</ttcol> <c>One</c> <c>Forward</c> <c>0s</c> <c>1.0</c> <c>Two</c> <c>Forward</c> <c>40s</c> <c>2.5</c> <c>Three</c> <c>Forward</c> <c>60s</c> <c>0.6</c> <c>Four</c> <c>Forward</c> <c>80s</c> <c>1.0</c> <!-- <postamble>Table 1: Path capacity variationthe Forward Direction</name> <thead> <tr> <th align="left">Variation patternfor forward direction</postamble> --> </texttable>index</th> <th align="left">Path direction</th> <th align="left">Start time</th> <th align="left">Path capacity ratio</th> </tr> </thead> <tbody> <tr> <td align="left">One</td> <td align="left">Forward</td> <td align="left">0 s</td> <td align="left">1.0</td> </tr> <tr> <td align="left">Two</td> <td align="left">Forward</td> <td align="left">40 s</td> <td align="left">2.5</td> </tr> <tr> <td align="left">Three</td> <td align="left">Forward</td> <td align="left">60 s</td> <td align="left">0.6</td> </tr> <tr> <td align="left">Four</td> <td align="left">Forward</td> <td align="left">80 s</td> <td align="left">1.0</td> </tr> </tbody> </table> <t/> </section> <section anchor="VACM"title="Variablenumbered="true" toc="default"> <name>Variable Available Capacity with MultipleFlows">Flows</name> <t>This test case is similar to <xreftarget="VACS"/>. However in additiontarget="VACS" format="default"/>. However, this test will also consider persistent network load due to competing traffic.</t><t>Expected behavior: the<dl spacing="normal"> <dt>Expected behavior:</dt><dd>The candidate algorithm is expected to detect the variation in available capacity and adapt the media stream(s) accordingly. The flows stabilize around their maximum bit rate as the maximum link capacity is large enough to accommodate the flows. When the available capacity drops, the flows adapt by decreasing their sending bit rate, and when congestion disappears, the flows are again expected to rampup.</t> <t>Evaluation metrics : asup.</dd> <dt>Evaluation metrics:</dt><dd>As described in <xreftarget="EM"/>.</t> <t>Testbed Topology: Twotarget="EM" format="default"/>.</dd> <dt>Testbed topology:</dt><dd>Two (2) media sources S1 and S2 are connected to their corresponding destinations R1 and R2. The media traffic is transported over the forward path and corresponding feedback/control traffic is transported over the backward path. </dd> </dl> <figureanchor="fig-eval-topo-4-1" title="Testbedanchor="fig-eval-topo-4-1"> <name>Testbed Topology for Variable AvailableCapacity"> <artwork><![CDATA[Capacity</name> <artwork name="" type="" align="left" alt=""><![CDATA[ +---+ +---+ |S1 |===== \ / =======|R1 | +---+ \\ Forward --> // +---+ \\ // +-----+ +-----+ | A |------------------------------>| B | | |<------------------------------| | +-----+ +-----+ // \\ // <-- Backward \\ +---+ // \\ +---+ |S2 |====== / \ ======|R2 | +---+ +---+ ]]></artwork></figure></t> <t>Testbed attributes:</t> <t>Testbed</figure> <dl spacing="normal"> <dt>Testbed attributes:</dt> <dd>Testbed attributes are similarasto those described in <xreftarget="VACS"/>target="VACS" format="default"/>, except for thetest specifictest-specific capacity variationsetup.</t> <t>Test Specific Information: Thissetup.</dd> <dt>Test-specific information:</dt> <dd>This test uses path capacity variation as listed in <xreftarget="VACM_US"/>target="VACM_US" format="default"/> with a corresponding end time of 125 seconds. The reference bottleneck capacity is2Mbps.2 Mbps. When using background non-adaptive UDP traffic to induce time-varying bottleneck forcongestion controlledcongestion-controlled media flows, the physical path capacity is4Mbps4 Mbps, and the UDP traffic source rate changes over time as (4 - (Y x r)), where r is the Reference bottleneck capacity inMbpsMbps, and Y is the path capacity ratio specified in <xreftarget="VACM_US"/>.</t> <texttabletarget="VACM_US" format="default"/>.</dd> </dl> <table anchor="VACM_US"title="Path capacity variation patternalign="center"> <name>Path Capacity Variation Pattern forforward direction"> <ttcol>Variation pattern index</ttcol> <ttcol>Path direction</ttcol> <ttcol>Start time</ttcol> <ttcol>Path capacity ratio</ttcol> <c>One</c> <c>Forward</c> <c>0s</c> <c>2.0</c> <c>Two</c> <c>Forward</c> <c>25s</c> <c>1.0</c> <c>Three</c> <c>Forward</c> <c>50s</c> <c>1.75</c> <c>Four</c> <c>Forward</c> <c>75s</c> <c>0.5</c> <c>Five</c> <c>Forward</c> <c>100s</c> <c>1.0</c> <!-- <postamble>Table 2: Path capacity variationthe Forward Direction</name> <thead> <tr> <th align="left">Variation patternfor forward direction</postamble> --> </texttable>index</th> <th align="left">Path direction</th> <th align="left">Start time</th> <th align="left">Path capacity ratio</th> </tr> </thead> <tbody> <tr> <td align="left">One</td> <td align="left">Forward</td> <td align="left">0 s</td> <td align="left">2.0</td> </tr> <tr> <td align="left">Two</td> <td align="left">Forward</td> <td align="left">25 s</td> <td align="left">1.0</td> </tr> <tr> <td align="left">Three</td> <td align="left">Forward</td> <td align="left">50 s</td> <td align="left">1.75</td> </tr> <tr> <td align="left">Four</td> <td align="left">Forward</td> <td align="left">75 s</td> <td align="left">0.5</td> </tr> <tr> <td align="left">Five</td> <td align="left">Forward</td> <td align="left">100 s</td> <td align="left">1.0</td> </tr> </tbody> </table> </section> <section anchor="CFL"title="Congestednumbered="true" toc="default"> <name>Congested Feedback Link with Bi-directional MediaFlows">Flows</name> <t>Real-time interactive media usesRTPRTP; hence it is assumed that RTCP, RTP headerextensionextension, or such would be used by the congestion control algorithm in thebackchannel.back channel. Due to the asymmetric nature of the link between communicatingpeerspeers, it is possible for a participating peer to not receive such feedback information due to an impaired or congestedbackchannelback channel (even when the forward channel might not be impaired). This test case is designed to observe the candidate congestion control behavior in such an event.</t><t>Expected behavior: It<dl spacing="normal"> <dt>Expected behavior:</dt> <dd> <t>It is expected that the candidate algorithms are able to cope with the lack of feedback information and to adapt to minimize the performance degradation of media flows in the forward channel.</t> <t>It should be noted that for this testcase:case, logs are compared with the reference case,i.e,i.e., when the backward channel has no impairments.</t><t>Evaluation metrics : as</dd> <dt>Evaluation metrics:</dt><dd>As described in <xreftarget="EM"/>.</t> <t>Testbed topology: Onetarget="EM" format="default"/>.</dd> <dt>Testbed topology:</dt><dd>One (1) media source S1 is connected to corresponding R1, but both endpoints are additionally receiving and sending data, respectively. The media traffic (S1->R1) is transported over the forwardpathpath, and the corresponding feedback/control traffic is transported over the backward path.LikewiseLikewise, media traffic (S2->R2) is transported over the backwardpathpath, and the corresponding feedback/control traffic is transported over the forwardpath.</t> <t><figure anchor="fig-eval-topo-4-6" title="Testbedpath.</dd> </dl> <figure anchor="fig-eval-topo-4-6"> <name>Testbed Topology for Congested FeedbackLink"> <artwork><![CDATA[Link</name> <artwork name="" type="" align="left" alt=""><![CDATA[ +---+ +---+ |S1|=====|====== \ Forward --> / =======|R1 | +---+ \\ // +---+ \\ // +-----+ +-----+ | A |------------------------------>| B | | |<------------------------------| | +-----+ +-----+ // \\ // <-- Backward \\ +---+ // \\ +---+ |R2 |===== / \ ======|S2 | +---+ +---+ ]]></artwork></figure></t> <t>Testbed attributes:</t> <t><list style="symbols"> <t>Test duration: 100s</t> <t>Path characteristics: <list style="symbols"> <t>Reference</figure> <dl spacing="normal"> <dt>Testbed attributes:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Test duration:</dt><dd>100 s</dd> <dt>Path characteristics:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Reference bottleneckcapacity: 1Mbps.</t> </list></t> <t>Application-related: <list style="symbols"> <t>Media Source: <list style="symbols"> <t>Media type: Video<list style="symbols"> <t>Media direction: forward and backward</t> <t>Number of media sources: two (2)</t> <t>Media timeline: <list style="symbols"> <t>Start time: 0s.</t> <t>End time: 99s.</t> </list></t> </list></t> <t>Media type: Audio<list style="symbols"> <t>Media direction: forward and backward</t> <t>Number of media sources: two (2)</t> <t>Mediacapacity:</dt><dd>1 Mbps</dd> </dl> </dd> <dt>Application-related:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media source:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media type:</dt> <dd> <t>Video</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward and backward</dd> <dt>Number of media sources:</dt><dd>two (2)</dd> <dt>Media timeline:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Start time:</dt><dd>0 s</dd> <dt>End time:</dt><dd>99 s</dd> </dl> </dd> </dl> </dd> <dt>Media type:</dt> <dd> <t>Audio</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward and backward</dd> <dt>Number of media sources:</dt><dd>two (2)</dd> <dt>Media timeline:<list style="symbols"> <t>Start time: 0s.</t> <t>End time: 99s.</t> </list></t> </list></t> </list></t> <t>Competing</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Start time:</dt><dd>0 s</dd> <dt>End time:</dt><dd>99 s</dd> </dl> </dd> </dl> </dd> </dl> </dd> <dt>Competing traffic:<list style="symbols"> <t>Number of sources : zero (0)</t> </list></t> </list></t> <t>Test Specific Information: this</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Number of sources:</dt><dd>zero (0)</dd> </dl> </dd> </dl> </dd> </dl> </dd> <dt>Test-specific information:</dt> <dd>This test uses path capacity variations to create a congested feedback link. <xreftarget="CFL_US"> </xref>target="CFL_US" format="default"/> lists the variation patterns applied to the forwardpathpath, and <xreftarget="CFL_DS"/>target="CFL_DS" format="default"/> lists the variation patterns applied to the backward path. When using background non-adaptive UDP traffic to induce a time-varying bottleneck forcongestion controlledcongestion-controlled media flows, the physical path capacity is4Mbps4 Mbps for bothdirectionsdirections, and the UDP traffic source rate changes over time as(4-x)Mbps(4-x) Mbps in each direction, where x is the bottleneck capacity specified in <xreftarget="CFL_DS"/>.</t> </list></t> <texttabletarget="CFL_DS" format="default"/>.</dd> </dl> <table anchor="CFL_US"title="Path capacity variation patternalign="center"> <name>Path Capacity Variation Pattern forforward direction"> <ttcol>Variation pattern index</ttcol> <ttcol>Path direction</ttcol> <ttcol>Start time</ttcol> <ttcol>Path capacity ratio</ttcol> <c>One</c> <c>Forward</c> <c>0s</c> <c>2.0</c> <c>Two</c> <c>Forward</c> <c>20s</c> <c>1.0</c> <c>Three</c> <c>Forward</c> <c>40s</c> <c>0.5</c> <c>Four</c> <c>Forward</c> <c>60s</c> <c>2.0</c> <!-- <postamble>Table 3: Path capacity variationthe Forward Direction</name> <thead> <tr> <th align="left">Variation patternfor forward direction</postamble> --> </texttable>index</th> <th align="left">Path direction</th> <th align="left">Start time</th> <th align="left">Path capacity ratio</th> </tr> </thead> <tbody> <tr> <td align="left">One</td> <td align="left">Forward</td> <td align="left">0 s</td> <td align="left">2.0</td> </tr> <tr> <td align="left">Two</td> <td align="left">Forward</td> <td align="left">20 s</td> <td align="left">1.0</td> </tr> <tr> <td align="left">Three</td> <td align="left">Forward</td> <td align="left">40 s</td> <td align="left">0.5</td> </tr> <tr> <td align="left">Four</td> <td align="left">Forward</td> <td align="left">60 s</td> <td align="left">2.0</td> </tr> </tbody> </table> <t/><texttable<table anchor="CFL_DS"title="Path capacity variation patternalign="center"> <name>Path Capacity Variation Pattern forbackward direction"> <ttcol>Variation pattern index</ttcol> <ttcol>Path direction</ttcol> <ttcol>Start time</ttcol> <ttcol>Path capacity ratio</ttcol> <c>One</c> <c>Backward</c> <c>0s</c> <c>2.0</c> <c>Two</c> <c>Backward</c> <c>35s</c> <c>0.8</c> <c>Three</c> <c>Backward</c> <c>70s</c> <c>2.0</c> <!-- <postamble>Table 4: Path capacity variationthe Backward Direction</name> <thead> <tr> <th align="left">Variation patternfor backward direction</postamble> --> </texttable>index</th> <th align="left">Path direction</th> <th align="left">Start time</th> <th align="left">Path capacity ratio</th> </tr> </thead> <tbody> <tr> <td align="left">One</td> <td align="left">Backward</td> <td align="left">0 s</td> <td align="left">2.0</td> </tr> <tr> <td align="left">Two</td> <td align="left">Backward</td> <td align="left">35 s</td> <td align="left">0.8</td> </tr> <tr> <td align="left">Three</td> <td align="left">Backward</td> <td align="left">70 s</td> <td align="left">2.0</td> </tr> </tbody> </table> </section> <section anchor="competing-rmcat-flow"title="Competingnumbered="true" toc="default"> <name>Competing Media Flows withsamethe Same Congestion ControlAlgorithm">Algorithm</name> <t>In this test case, more than one media flow share the bottlenecklinklink, and each of them uses the same congestion control algorithm. This is a typical scenario where a real-time interactive application sends more than one media flow to the samedestinationdestination, and these flows are multiplexed over the same port. In such ascenarioscenario, it is likely that the flows will be routed via the same path and need to share the available bandwidth amongst themselves. For the sake ofsimplicitysimplicity, it is assumed that there are no other competing traffic sources in the bottleneck link and that there is sufficient capacity to accommodate all the flows individually. While this appears to be a variant of the test case defined in <xreftarget="VACM"/>,target="VACM" format="default"/>, it focuses on thecapacity sharingcapacity-sharing aspect of the candidate algorithm. The previous test case, on the other hand, measures adaptability, stability, and responsiveness of the candidate algorithm.</t><t>Expected behavior: It<dl spacing="normal"> <dt>Expected behavior:</dt><dd>It is expected that the competing flows will converge to an optimum bit rate to accommodate all the flows with minimum possible latency and loss. Specifically, the test introduces three media flows at different timeinstances, wheninstances. When the second flowappearsappears, there should still be room to accommodate another flow on the bottleneck link. Lastly, when the third flowappearsappears, the bottleneck link should besaturated.</t> <t>Evaluation metrics : assaturated.</dd> <dt>Evaluation metrics:</dt><dd>As described in <xreftarget="EM"/>.</t> <t>Testbedtarget="EM" format="default"/>.</dd> <dt>Testbed topology:Three</dt><dd>Three media sources S1, S2, and S3 are connected to R1, R2,R3and R3, respectively. The media traffic is transported over the forwardpathpath, and the corresponding feedback/control traffic is transported over the backward path. </dd> </dl> <figureanchor="fig-eval-topo-4-3" title="Testbedanchor="fig-eval-topo-4-3"> <name>Testbed Topology for Multiplecongestion controlled media Flows"> <artwork><![CDATA[Congestion-Controlled Media Flows</name> <artwork name="" type="" align="left" alt=""><![CDATA[ +---+ +---+ |S1 |===== \ Forward --> / =======|R1 | +---+ \\ // +---+ \\ // +---+ +-----+ +-----+ +---+ |S2 |=======| A |------------------------------>| B |=======|R2 | +---+ | |<------------------------------| | +---+ +-----+ +-----+ // <-- Backward \\ +---+ // \\ +---+ |S3 |===== / \ ======|R3 | +---+ +---+ ]]></artwork></figure></t> <t>Testbed attributes:</t> <t><list style="symbols"> <t>Test duration: 120s</t> <t>Path characteristics: <list style="symbols"> <t>Reference</figure> <dl spacing="normal"> <dt>Testbed attributes:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Test duration:</dt><dd>120 s</dd> <dt>Path characteristics:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Reference bottleneckcapacity: 3.5Mbps</t> <t>Path capacity ratio: 1.0</t> <!-- <t>One-Way propagation delay: [10ms, 50ms]</t> --> </list></t> <t>Application-related: <list style="symbols"> <t>Mediacapacity:</dt><dd>3.5 Mbps</dd> <dt>Path capacity ratio:</dt><dd>1.0</dd> </dl> </dd> <dt>Application-related: </dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media Source:<list style="symbols"> <t>Media type: Video<list style="symbols"> <t>Media direction: forward.</t> <t>Number of media sources: three (3)</t> <t>Media timeline: new</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media type:</dt> <dd> <t>Video</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward</dd> <dt>Number of media sources:</dt><dd>three (3)</dd> <dt>Media timeline:</dt><dd>new media flows are added sequentially, at short time intervals. Seetest specificthe test-specific setupbelow.</t> </list></t> <t>Media type: Audio<list style="symbols"> <t>Media direction: forward.</t> <t>Number of media sources: three (3)</t> <t>Media timeline: newbelow.</dd> </dl> </dd> <dt>Media type:</dt> <dd> <t>Audio</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward</dd> <dt>Number of media sources:</dt><dd>three (3)</dd> <dt>Media timeline:</dt><dd>new media flows are added sequentially, at short time intervals. Seetest specificthe test-specific setupbelow.</t> </list></t> </list></t> <t>Competingbelow.</dd> </dl> </dd> </dl> </dd> <dt>Competing traffic:<list style="symbols"> <t>Number of sources : zero (0)</t> </list></t> </list></t> <t>Test Specific Information:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Number of sources:</dt><dd>zero (0)</dd> </dl> </dd> </dl> </dd> </dl> </dd> <dt>Test-specific information:</dt><dd> <xreftarget="MTL_CF"/>target="MTL_CF" format="default"/> defines the media timeline for both mediatype.</t> </list></t> <texttabletypes.</dd> </dl> <table anchor="MTL_CF"title="Media Timelinealign="center"> <name>Media Timelines for Video and Audiomedia sources"> <ttcol>Flow ID</ttcol> <ttcol>Media type</ttcol> <ttcol>Start time</ttcol> <ttcol>End time</ttcol> <c>1</c> <c>Video</c> <c>0s</c> <c>119s</c> <c>2</c> <c>Video</c> <c>20s</c> <c>119s</c> <c>3</c> <c>Video</c> <c>40s</c> <c>119s</c> <c>4</c> <c>Audio</c> <c>0s</c> <c>119s</c> <c>5</c> <c>Audio</c> <c>20s</c> <c>119s</c> <c>6</c> <c>Audio</c> <c>40s</c> <c>119s</c> </texttable>Media Sources</name> <thead> <tr> <th align="left">Flow ID</th> <th align="left">Media type</th> <th align="left">Start time</th> <th align="left">End time</th> </tr> </thead> <tbody> <tr> <td align="left">1</td> <td align="left">Video</td> <td align="left">0 s</td> <td align="left">119 s</td> </tr> <tr> <td align="left">2</td> <td align="left">Video</td> <td align="left">20 s</td> <td align="left">119 s</td> </tr> <tr> <td align="left">3</td> <td align="left">Video</td> <td align="left">40 s</td> <td align="left">119 s</td> </tr> <tr> <td align="left">4</td> <td align="left">Audio</td> <td align="left">0 s</td> <td align="left">119 s</td> </tr> <tr> <td align="left">5</td> <td align="left">Audio</td> <td align="left">20 s</td> <td align="left">119 s</td> </tr> <tr> <td align="left">6</td> <td align="left">Audio</td> <td align="left">40 s</td> <td align="left">119 s</td> </tr> </tbody> </table> </section> <sectiontitle="Roundnumbered="true" toc="default"> <name>Round Trip TimeFairness">Fairness</name> <t>In this test case, multiple media flows share the bottleneck link, but the one-way propagation delay for each flow is different. For the sake ofsimplicitysimplicity, it is assumed that there are no other competing traffic sources in the bottleneck link and that there is sufficient capacity to accommodate all the flows. While this appears to be a variant of test case5.2,5.2 (<xref target="VACM" format="default"/>), it focuses on thecapacity sharingcapacity-sharing aspect of the candidate algorithm under different RTTs.</t><t>Expected behavior: It<dl spacing="normal"> <dt>Expected behavior:</dt><dd>It is expected that the competing flows will converge to bit rates to accommodate all the flows with minimum possible latency and loss. The effectiveness of the algorithm depends on how fast and fairly the competing flows converge to their steady states irrespective of the RTTobserved.</t> <t>Evaluation metrics : asobserved.</dd> <dt>Evaluation metrics:</dt><dd>As described in <xreftarget="EM"/>.</t> <t>Testbed Topology: Fivetarget="EM" format="default"/>.</dd> <dt>Testbed topology: </dt><dd>Five (5) media sourcesS1,S2,..,S5S1..S5 are connected to their corresponding media sinksR1,R2,..,R5.R1..R5. The media traffic is transported over the forwardpathpath, and the corresponding feedback/control traffic is transported over the backward path. The topology is the same as in <xreftarget="competing-rmcat-flow"/>.</t> <t>Testbed attributes:</t> <t><list style="symbols"> <t>Test duration: 300s</t> <t>Pathtarget="competing-rmcat-flow" format="default"/>.</dd> <dt>Testbed attributes: </dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Test duration:</dt><dd>300 s</dd> <dt>Path characteristics:<list style="symbols"> <t>Reference</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Reference bottleneckcapacity: 4Mbps</t> <t>Pathcapacity:</dt><dd>4 Mbps</dd> <dt>Path capacityratio: 1.0</t> <t>One-Wayratio:</dt><dd>1.0</dd> <dt>One-way propagation delay for eachflow: 10msflow:</dt><dd>10 ms for S1-R1,25ms25 ms for S2-R2,50ms50 ms for S3-R3,100ms100 ms for S4-R4, and150ms S5-R5.</t> </list></t> <t>Application-related: <list style="symbols"> <t>Media Source: <list style="symbols"> <t>Media type: Video<list style="symbols"> <t>Media direction: forward</t> <t>Number of media sources: five (5)</t> <t>Media timeline: new150 ms S5-R5.</dd> </dl> </dd> <dt>Application-related: </dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media source: </dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media type:</dt> <dd> <t>Video</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward</dd> <dt>Number of media sources:</dt><dd>five (5)</dd> <dt>Media timeline:</dt><dd>new media flows are added sequentially, at short time intervals. Seetest specificthe test-specific setupbelow.</t> </list></t> <t>Media type: Audio<list style="symbols"> <t>Media direction: forward.</t> <t>Numberbelow.</dd> </dl> </dd> <dt>Media type:</dt> <dd> <t>Audio</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward</dd> <dt>Number of mediasources:sources:</dt><dd> five(5)</t> <t>Media timeline: new(5)</dd> <dt>Media timeline:</dt><dd>New media flows are added sequentially, at short time intervals. Seetest specificthe test-specific setupbelow.</t> </list></t> </list></t> <t>Competingbelow.</dd> </dl> </dd> </dl> </dd> <dt>Competing traffic:<list style="symbols"> <t>Number of sources : zero (0)</t> </list></t> </list></t> <t>Test Specific Information: <xref target="MTL_RTT"/></dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Number of sources:</dt><dd>zero (0)</dd> </dl> </dd> </dl> </dd> </dl> </dd> <dt>Test-specific information: </dt><dd><xref target="MTL_RTT" format="default"/> defines the media timeline for both mediatype.</t> </list></t> <texttabletypes.</dd> </dl> <table anchor="MTL_RTT"title="Mediaalign="center"> <name>Media Timeline for Video and Audiomedia sources"> <ttcol>Flow IF</ttcol> <ttcol>Media type</ttcol> <ttcol>Start time</ttcol> <ttcol>End time</ttcol> <c>1</c> <c>Video</c> <c>0s</c> <c>299s</c> <c>2</c> <c>Video</c> <c>10s</c> <c>299s</c> <c>3</c> <c>Video</c> <c>20s</c> <c>299s</c> <c>4</c> <c>Video</c> <c>30s</c> <c>299s</c> <c>5</c> <c>Video</c> <c>40s</c> <c>299s</c> <c>6</c> <c>Audio</c> <c>0</c> <c>299s</c> <c>7</c> <c>Audio</c> <c>10s</c> <c>299s</c> <c>8</c> <c>Audio</c> <c>20s</c> <c>299s</c> <c>9</c> <c>Audio</c> <c>30s</c> <c>299s</c> <c>10</c> <c>Audio</c> <c>40s</c> <c>299s</c> </texttable>Media Sources</name> <thead> <tr> <th align="left">Flow ID</th> <th align="left">Media type</th> <th align="left">Start time</th> <th align="left">End time</th> </tr> </thead> <tbody> <tr> <td align="left">1</td> <td align="left">Video</td> <td align="left">0 s</td> <td align="left">299 s</td> </tr> <tr> <td align="left">2</td> <td align="left">Video</td> <td align="left">10 s</td> <td align="left">299 s</td> </tr> <tr> <td align="left">3</td> <td align="left">Video</td> <td align="left">20 s</td> <td align="left">299 s</td> </tr> <tr> <td align="left">4</td> <td align="left">Video</td> <td align="left">30 s</td> <td align="left">299 s</td> </tr> <tr> <td align="left">5</td> <td align="left">Video</td> <td align="left">40 s</td> <td align="left">299 s</td> </tr> <tr> <td align="left">6</td> <td align="left">Audio</td> <td align="left">0 s</td> <td align="left">299 s</td> </tr> <tr> <td align="left">7</td> <td align="left">Audio</td> <td align="left">10 s</td> <td align="left">299 s</td> </tr> <tr> <td align="left">8</td> <td align="left">Audio</td> <td align="left">20 s</td> <td align="left">299 s</td> </tr> <tr> <td align="left">9</td> <td align="left">Audio</td> <td align="left">30 s</td> <td align="left">299 s</td> </tr> <tr> <td align="left">10</td> <td align="left">Audio</td> <td align="left">40 s</td> <td align="left">299 s</td> </tr> </tbody> </table> </section> <sectiontitle="Medianumbered="true" toc="default"> <name>Media Flow Competing with a Long TCPFlow">Flow</name> <t>In this test case, one or more media flows share the bottleneck link with at least onelong livedlong-lived TCP flow.Long livedLong-lived TCP flows download data throughout the session and are expected to have infinite amount of data to send and receive. This is a scenario where a multimedia applicationco-existscoexists with a large file download. The test case measures the adaptivity of the candidate algorithm to competing traffic. It addressestherequirement 3 in <xreftarget="I-D.ietf-rmcat-cc-requirements"/>.</t> <t>Expected behavior: dependingtarget="RFC8836" section="2" sectionFormat="of" format="default"/>.</t> <dl spacing="normal"> <dt>Expected behavior:</dt><dd>Depending on the convergence observed in testcasecases 5.1 and 5.2, the candidate algorithm may be able to avoid congestion collapse. In the worst case, the media stream will fall to the minimum media bitrate.</t> <t>Evaluation metrics :rate.</dd> <dt>Evaluation metrics:</dt> <dd><t>Includes the following metrics in addition toasthose described in <xreftarget="EM"/>. <list style="numbers">target="EM" format="default"/>: </t> <ol spacing="normal" type="1"> <li> <t>Flow level:<list style="letters"> <t>TCP throughput.</t> <t>Loss</t> <ol spacing="normal" type="a"> <li>TCP throughput</li> <li>Loss for the TCPflow</t> </list></t> </list></t> <t>Testbed topology: Oneflow</li> </ol> </li> </ol> </dd> <dt>Testbed topology:</dt><dd>One (1) media source S1 is connected to the corresponding media sink, R1. In addition, there is along-livelong-lived TCP flow sharing the same bottleneck link. The media traffic is transported over the forwardpathpath, and the corresponding feedback/control traffic is transported over the backward path. The TCP traffic goes over the forward pathfrom,from S_tcp with acknowledgment packetsgogoing over the backward pathfrom, R_tcp.</t> <t><figure anchor="fig-eval-topo-4-4" title="Testbedfrom R_tcp.</dd> </dl> <figure anchor="fig-eval-topo-4-4"> <name>Testbed Topology for TCP vscongestion controlled media Flows"> <artwork><![CDATA[Congestion-Controlled Media Flows</name> <artwork name="" type="" align="left" alt=""><![CDATA[ +--+ +--+ |S1|===== \ Forward --> / =====|R1| +--+ \\ // +--+ \\ // +-----+ +-----+ | A |---------------------------->| B | | |<----------------------------| | +-----+ +-----+ // <-- Backward \\ +-----+ // \\ +-----+ |S_tcp|=== / \ ===|R_tcp| +-----+ +-----+ ]]></artwork></figure></t> <t>Testbed attributes:</t> <t><list style="symbols"> <t>Test duration: 120s</t> <t>Path characteristics: <list style="symbols"> <t>Reference</figure> <dl spacing="normal"> <dt>Testbed attributes:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Test duration:</dt><dd>120 s</dd> <dt>Path characteristics:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Reference bottleneckcapacity: 2Mbps</t> <t>Pathcapacity:</dt><dd>2 Mbps</dd> <dt>Path capacityratio: 1.0</t> <!-- <t>One-Way propagation delay: [10ms, 150ms]</t> --> <t>Bottleneckratio:</dt><dd>1.0</dd> <dt>Bottleneck queuesize: [300ms, 1000ms]</t> </list></t> <t>Application-related: <list style="symbols"> <t>Media Source: <list style="symbols"> <t>Media type: Video<list style="symbols"> <t>Media direction: forward</t> <t>Number of media sources: one (1)</t> <t>Media timeline: <list style="symbols"> <t>Start time: 5s.</t> <t>End time: 119s.</t> </list></t> </list></t> <t>Media type: Audio<list style="symbols"> <t>Media direction: forward</t> <t>Number of media sources: one (1)</t> <t>Media timeline: <list style="symbols"> <t>Start time: 5s.</t> <t>End time: 119s.</t> </list></t> </list></t> </list></t>size:</dt><dd>[300 ms, 1000 ms]</dd> </dl> </dd> <dt>Application-related:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media source:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media type:</dt> <dd> <t>Video</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward</dd> <dt>Number of media sources:</dt><dd>one (1)</dd> <dt>Media timeline:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Start time:</dt><dd>5 s</dd> <dt>End time:</dt><dd>119 s</dd> </dl> </dd> </dl> </dd> <dt>Media type:</dt> <dd> <t>Audio</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward</dd> <dt>Number of media sources:</dt><dd>one (1)</dd> <dt>Media timeline:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Start time:</dt><dd>5 s</dd> <dt>End time:</dt><dd>119 s</dd> </dl> </dd> </dl> </dd> </dl> <t>Additionally, implementers are encouraged to run the experiment with multiple media sources.</t><t>Competing traffic: <list style="symbols"> <t>Number</dd> <dt>Competing traffic:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Number andTypestypes ofsources : onesources:</dt><dd>one (1) and long-livedTCP</t> <t>Traffic direction : forward</t> <t>Congestion control: defaultTCP</dd> <dt>Traffic direction:</dt><dd>forward</dd> <dt>Congestion control:</dt><dd>default TCP congestioncontrol<xref target="RFC5681"/>.control <xref target="RFC5681" format="default"/>. Implementers are also encouraged to run the experiment with alternative TCP congestion controlalgorithm.<!----></t> <t>Trafficalgorithms.</dd> <dt>Traffic timeline:<list style="symbols"> <t>Start time: 0s.</t> <!----> <t>End time: 119s.</t> </list></t> </list></t> </list></t> <t>Test Specific Information: none</t> </list></t></dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Start time:</dt><dd>0 s</dd> <dt>End time:</dt><dd>119 s</dd> </dl> </dd> </dl> </dd> </dl> </dd> </dl> </dd> <dt>Test-specific information:</dt><dd>none</dd> </dl> </section> <sectiontitle="Medianumbered="true" toc="default"> <name>Media Flow Competing with Short TCPFlows">Flows</name> <t>In this test case, one or morecongestion controlledcongestion-controlled mediaflow sharesflows share the bottleneck link with multiple short-lived TCP flows. Short-lived TCP flows resemble the on/off pattern observed intheweb traffic, wherein clients (for example, browsers) connect to a server and download a resource (typically a web page, few images, text files, etc.) using several TCP connections. This scenario shows the performance of a multimedia application when several browser windows are active. The test case measures the adaptivity of the candidate algorithm to competing web traffic, and it addressesthe requirementsrequirement 1.E in <xreftarget="I-D.ietf-rmcat-cc-requirements"/>.</t>target="RFC8836" section="2" sectionFormat="of" format="default"/>.</t> <t>Depending on the number of short TCP flows, thecross-trafficcross traffic either appears as a short burst flow or resembles alonglong-lived TCP flow. The intention of this test is to observe the impact of a short-term burst on the behavior of the candidate algorithm.</t><t>Expected behavior:<dl spacing="normal"> <dt>Expected behavior:</dt><dd> The candidate algorithm is expected to avoid flow starvation during the presence of short and bursty competing TCP flows, streaming at least at the minimum media bit rate. After competing TCP flows terminate, the media streams are expected to be robust enough to eventually recover to previous steady state behavior, and at the very least, avoid persistentstarvation.</t> <t>Evaluation metrics :starvation.</dd> <dt>Evaluation metrics:</dt> <dd> <t>Includes the following metrics in addition toasthose described in <xreftarget="EM"/>.<list style="numbers"> <t>Flowtarget="EM" format="default"/>:</t> <ol spacing="normal" type="1"> <li><t>Flow level:<list style="letters"> <t>Variation</t> <ol spacing="normal" type="A"> <li>Variation in the sending rate of the TCPflow.</t> <t>TCP throughput.</t> </list></t> </list></t> <t>Testbed topology: Theflow</li> <li>TCP throughput</li> </ol> </li> </ol> </dd> <dt>Testbed topology:</dt><dd>The topology described here is the same as the one described in <xreftarget="fig-eval-topo-4-4"/>.</t> <t>Testbed attributes:</t> <t><list style="symbols"> <t>Test duration: 300s</t> <t>Path characteristics: <list style="symbols"> <t>Referencetarget="fig-eval-topo-4-4" format="default"/>.</dd> <dt>Testbed attributes:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Test duration:</dt><dd>300 s</dd> <dt>Path characteristics:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Reference bottleneckcapacity: 2.0Mbps</t> <t>Path capacity ratio: 1.0</t> <!-- <t>One-Way propagation delay: [10ms, 150ms]</t> --> </list></t> <t>Application-related: <list style="symbols"> <t>Mediacapacity:</dt><dd>2.0 Mbps</dd> <dt>Path capacity ratio:</dt><dd>1.0</dd> </dl> </dd> <dt>Application-related: </dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media source:<list style="symbols"> <t>Media type: Video<list style="symbols"> <t>Media direction: forward</t> <t>Number of media sources: two (2)</t> <t>Media</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media type:</dt> <dd> <t>Video</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward</dd> <dt>Number of media sources:</dt><dd>two (2)</dd> <dt>Media timeline:<list style="symbols"> <t>Start time: 5s.</t> <t>End time: 299s.</t> </list></t> </list></t> <t>Media type: Audio<list style="symbols"> <t>Media direction: forward</t> <t>Number</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Start time:</dt><dd>5 s</dd> <dt>End time:</dt><dd>299 s</dd> </dl> </dd> </dl> </dd> <dt>Media type:</dt> <dd> <t>Audio</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>forward</dd> <dt>Number of mediasources:sources:</dt><dd> two(2)</t> <t>Media(2)</dd> <dt>Media timeline:<list style="symbols"> <t>Start time: 5s.</t> <t>End</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Start time:</dt><dd> 5 s</dd> <dt>End time:299s.</t> </list></t> </list></t> </list></t> <t>Competing</dt><dd>299 s</dd> </dl> </dd> </dl> </dd> </dl> </dd> <dt>Competing traffic:<list style="symbols"> <t>Number</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Number andTypestypes ofsources :sources:</dt><dd> ten (10), short-lived TCPflows.</t> <t>Traffic direction : forward</t> <t>Congestion algorithm:flows.</dd> <dt>Traffic direction: </dt><dd>forward</dd> <dt>Congestion algorithm:</dt><dd> default TCPCongestioncongestion control <xreftarget="RFC5681"/>.target="RFC5681" format="default"/>. Implementers are also encouraged to run the experiment with an alternative TCP congestion controlalgorithm.</t> <t>Trafficalgorithm.</dd> <dt>Traffic timeline:each</dt><dd>Each short TCP flow is modeled as a sequence of file downloads interleaved with idle periods. Not all short TCP flows start at the same time,2two of them start in the ONstatestate, while rest of the8eight flows start in an OFF state. For a description of the short TCP flowmodelmodel, seetest specifictest-specific informationbelow.</t> </list></t> </list></t> <t>Test Specific Information: <list style="symbols"> <t>Short-TCPbelow.</dd> </dl> </dd> </dl> </dd> </dl> </dd> <dt>Test-specific information: </dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Short TCP trafficmodel: Themodel:</dt><dd>The short TCP model to be used in this test is described in <xreftarget="I-D.ietf-rmcat-eval-criteria"/>.</t> </list></t> </list></t>target="RFC8868" format="default"/>.</dd> </dl> </dd> </dl> </section> <sectiontitle="Medianumbered="true" toc="default"> <name>Media Pause andResume">Resume</name> <t>In this test case, more than one real-time interactive mediaflowsflow share the linkbandwidthbandwidth, and all flows reach to a steady state by utilizing the link capacity in an optimum way. At thisstagestage, one of the media flows is paused for a moment. This event will result in more available bandwidth for the rest of the flows as they are on a shared link. When the paused media flowresumesresumes, itwouldno longerhavehas the same bandwidth share on the link. It has to make its way through the other existing flows in the link to achieve a fair share of the link capacity. This test case is important specially for real-time interactivemediamedia, which consists of more than one media flows and can pause/resume media flows at any point of time during the session. This test case directly addressestherequirementnumber5 in <xreftarget="I-D.ietf-rmcat-cc-requirements"/>.target="RFC8836" section="2" sectionFormat="of" format="default"/>. One can think of it as a variation of the test case defined in <xreftarget="competing-rmcat-flow"/>.target="competing-rmcat-flow" format="default"/>. However, it is different as the candidate algorithms can use different strategies to increaseitsefficiency, forexampleexample, in terms of fairness, convergence time,reduceoscillationetc,reduction, etc., by capitalizing on the fact that they have previous information of the link.</t><t>Expected behavior: During<dl spacing="normal"> <dt>Expected behavior:</dt><dd>During the period where the third stream is paused, the two remaining flows are expected to increase their rates and reach the maximum media bit rate. When the third stream resumes, all three flows are expected to converge to the same original fair share of rates prior to the media pause/resumeevent.</t> <t>Evaluation metrics :event.</dd> <dt>Evaluation metrics:</dt> <dd> <t>Includes the following metrics in addition toasthose described in <xreftarget="EM"/>.<list style="numbers"> <t>Flowtarget="EM" format="default"/>:</t> <ol spacing="normal" type="1"> <li><t>Flow level:<list style="letters"> <t>Variation</t> <ol spacing="normal" type="A"> <li>Variation in sending bit rate and throughput. Mainly observing the frequency and magnitude ofoscillations.</t> </list></t> </list></t> <t>Testbed Topology: Sameoscillations.</li> </ol> </li> </ol> </dd> <dt>Testbed topology:</dt><dd>Same as the test case defined in <xreftarget="competing-rmcat-flow"/></t> <t>Testbed attributes: Thetarget="competing-rmcat-flow" format="default"/>.</dd> <dt>Testbed attributes:</dt> <dd> <t>The general description of the testbed parameters are the same as <xreftarget="competing-rmcat-flow"/>target="competing-rmcat-flow" format="default"/> with changes in thetest specifictest-specific setup asbelow-</t> <t><list style="symbols">below:</t> <t>Othertest specifictest-specific setup:<list style="symbols"> <t>Media</t> <dl spacing="normal"> <dt>Media flow timeline:<list style="symbols"> <t>Flow</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Flow ID:one (1)</t> <t>Start</dt><dd>one (1)</dd> <dt>Start time:0s</t> <t>Flow</dt><dd>0 s</dd> <dt>Flow duration:119s</t> <t>Pause</dt><dd>119 s</dd> <dt>Pause time:not required</t> <t>Resume</dt><dd>not required</dd> <dt>Resume time:not required</t> </list></t> <t>Media</dt><dd>not required</dd> </dl> </dd> <dt>Media flow timeline:<list style="symbols"> <t>Flow</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Flow ID:two (2)</t> <t>Start</dt><dd>two (2)</dd> <dt>Start time:0s</t> <t>Flow</dt><dd>0 s</dd> <dt>Flow duration:119s</t> <t>Pause</dt><dd>119 s</dd> <dt>Pause time:at 40s</t> <t>Resume</dt><dd>at 40 s</dd> <dt>Resume time:at 60s</t> </list></t> <t>Media</dt><dd>at 60 s</dd> </dl> </dd> <dt>Media flow timeline:<list style="symbols"> <t>Flow ID: three (3)</t> <t>Start time: 0s</t> <t>Flow duration:119s</t> <t>Pause time: not required</t> <t>Resume time: not required</t> </list></t> </list></t> </list></t></dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Flow ID:</dt><dd>three (3)</dd> <dt>Start time:</dt><dd>0 s</dd> <dt>Flow duration:</dt><dd>119 s</dd> <dt>Pause time:</dt><dd>not required</dd> <dt>Resume time:</dt><dd>not required</dd> </dl> </dd> </dl> </dd> </dl> </section> </section> <sectiontitle="Other potential test cases">numbered="true" toc="default"> <name>Other Potential Test Cases</name> <t>It has been noticed that there are other interesting test cases besides the basic test cases listed above. In many aspects, these additional test cases can help further evaluation of the candidate algorithm. They are listedasbelow.</t> <sectiontitle="Medianumbered="true" toc="default"> <name>Media Flows withPriority">Priority</name> <t>In this testcasecase, media flows will have different priority levels. Thiswill beis an extension of <xreftarget="competing-rmcat-flow"/>target="competing-rmcat-flow" format="default"/> where the same testwill beis run with different priority levels imposed on each of the media flows. For example, the first flow (S1) is assigned a priority of22, whereas the remaining two flows (S2 and S3) are assigned a priority of 1. The candidate algorithm must reflect the relative priorities assigned to each media flow. In this case, the first flow (S1) must arrive at a steady-state rate approximately twiceofthat of the other two flows (S2 and S3).</t> <t>The candidate algorithm can use a coupled congestion control mechanism <xreftarget="I-D.ietf-rmcat-coupled-cc"/>target="RFC8699" format="default"/> or use a weighted priority scheduler for the bandwidth distribution according to the respective media flow priority or use.</t> </section> <sectiontitle="Explicitnumbered="true" toc="default"> <name>Explicit Congestion NotificationUsage">Usage</name> <t>This test case requiresto runrunning all the basic test cases with the availability of Explicit Congestion Notification (ECN) <xreftarget="RFC6679"/>target="RFC6679" format="default"/> feature enabled. The goal of this test is to exhibit that the candidate algorithms do not fail when ECN signals are available. With ECN signalsenabledenabled, the algorithms are expected to perform better than their delay-based variants.</t> </section> <sectiontitle="Multiple Bottlenecks">numbered="true" toc="default"> <name>Multiple Bottlenecks</name> <t>In this testcasecase, onecongestion controlledcongestion-controlled media flow, S1->R1, traverses a path with multiple bottlenecks. As illustrated in <xreftarget="fig-eval-topo-6-2"/>,target="fig-eval-topo-6-2" format="default"/>, the first flow (S1->R1) competes with the secondcongestion controlledcongestion-controlled media flow (S2->R2) over the link between A andBB, which is close to the senderside; again,side. Again, that flow (S1->R1) competes with the thirdcongestion controlledcongestion-controlled media flow (S3->R3) over the link between C andDD, which is close to the receiver side. The goal of this test is to ensure that the candidate algorithms work properly in the presence of multiple bottleneck links on theend to endend-to-end path.</t><t>Expected behavior:<dl spacing="normal"> <dt>Expected behavior:</dt><dd> The candidate algorithm is expected to achieve full utilization at both bottleneck links without starving any of the threecongestion controlledcongestion-controlled media flows and ensuring fair share of the available bandwidth at eachbottlenecks.</t> <t><figure anchor="fig-eval-topo-6-2" title="Testbedbottleneck.</dd> </dl> <figure anchor="fig-eval-topo-6-2"> <name>Testbed Topology for MultipleBottlenecks"> <artwork><![CDATA[Bottlenecks</name> <artwork name="" type="" align="left" alt=""><![CDATA[ Forward ----> +---+ +---+ +---+ +---+ |S2 | |R2 | |S3 | |R3 | +---+ +---+ +---+ +---+ | | | | | | | | +---+ +-----+ +-----+ +-----+ +-----+ +---+ |S1|=======||======| A |------>| B |----->| C |---->| D|=======|R1|======|R1 | +---+ | |<------| |<-----| |<----| | +---+ +-----+ +-----+ +-----+ +-----+ 1st 2nd Bottleneck (A->B) Bottleneck (C->D) <------ Backward ]]></artwork></figure></t> <t>Testbed topology: Three</figure> <dl spacing="normal"> <dt>Testbed topology:</dt><dd>Three media sources S1, S2, and S3 are connected to respective destinations R1, R2, and R3. For all threeflowsflows, the media traffic is transported over the forwardpathpath, and the corresponding feedback/control traffic is transported over the backwardpath.</t> <t>Testbed attributes:</t> <t><list style="symbols"> <t>Test duration: 300s</t> <t>Pathpath.</dd> <dt>Testbed attributes:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Test duration:</dt><dd> 300 s</dd> <dt>Path characteristics:<list style="symbols"> <t>Reference</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Reference bottleneckcapacity: 2Mbps.</t> <t>Pathcapacity:</dt><dd>2 Mbps</dd> <dt>Path capacity ratio between A andB: 1.0</t> <t>PathB:</dt><dd>1.0</dd> <dt>Path capacity ratio between B andC: 4.0.</t> <t>PathC:</dt><dd>4.0</dd> <dt>Path capacity ratio between C andD: 0.75.</t> <t>One-WayD:</dt><dd>0.75</dd> <dt>One-way propagation delay:<list style="numbers"> <t>Between</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Between S1 andR1: 100ms</t> <t>BetweenR1:</dt><dd>100 ms</dd> <dt>Between S2 andR2: 40ms</t> <t>BetweenR2:</dt><dd>40 ms</dd> <dt>Between S3 andR3: 40ms</t> </list></t> </list></t> <t>Application-related: <list style="symbols"> <t>Media Source: <list style="symbols"> <t>Media type: Video <list style="symbols"> <t>Media direction: Forward</t> <t>Number of media sources: Three (3)</t> <t>MediaR3:</dt><dd>40 ms</dd> </dl> </dd> </dl> </dd> <dt>Application-related: </dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media source:</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Media type:</dt> <dd> <t>Video </t> <dl spacing="normal"> <dt>Media direction:</dt><dd>Forward</dd> <dt>Number of media sources:</dt><dd>Three (3)</dd> <dt>Media timeline:<list style="symbols"> <t>Start time: 0s.</t> <t>End</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Start time:</dt><dd> 0 s</dd> <dt>End time:299s.</t> </list></t> </list></t> <t>Media type: Audio <list style="symbols"> <t>Media direction: Forward</t> <t>Number of media sources: Three (3)</t> <t>Media</dt><dd>299 s</dd> </dl> </dd> </dl> </dd> <dt>Media type:</dt> <dd> <t>Audio</t> <dl spacing="normal"> <dt>Media direction:</dt><dd>Forward</dd> <dt>Number of media sources:</dt><dd>Three (3)</dd> <dt>Media timeline:<list style="symbols"> <t>Start time: 0s.</t> <t>End time: 299s.</t> </list></t> </list></t> </list></t> <t>Competing</dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Start time:</dt><dd>0 s</dd> <dt>End time:</dt><dd>299 s</dd> </dl> </dd> </dl> </dd> </dl> </dd> <dt>Competing traffic:<list style="symbols"> <t>Number of sources : Zero (0)</t> </list></t> </list></t> </list></t></dt> <dd><t><br/></t> <dl spacing="normal"> <dt>Number of sources:</dt><dd>Zero (0)</dd> </dl> </dd> </dl> </dd> </dl> </dd> </dl> </section> </section> <sectiontitle="Wirelessnumbered="true" toc="default"> <name>Wireless AccessLinks"> <!-- -->Links</name> <t>Additional wireless network (both cellular network andWiFiWi-Fi network) specific test cases are defined in <xreftarget="I-D.ietf-rmcat-wireless-tests"/>.</t>target="RFC8869" format="default"/>.</t> </section> <sectiontitle="Security Considerations">numbered="true" toc="default"> <name>Security Considerations</name> <t>The security considerations in <xreftarget="I-D.ietf-rmcat-eval-criteria"/>target="RFC8868" section="6" sectionFormat="of" format="default"/> and the relevant congestion control algorithms apply. The principles for congestion control are described in <xreftarget="RFC2914"/>,target="RFC2914" format="default"/>, and in particular any new method must implement safeguards to avoid congestion collapse of the Internet.</t> <t>The evaluation of the test cases are intended to be run in a controlled lab environment. Hence, the applications,simulatorssimulators, and network nodes ought to be well-behaved and should not impact the desired results. Moreover, proper measures must be taken to avoid leakingnon-responsivenonresponsive traffic from unproven congestion avoidance techniques onto the open Internet.</t> </section> <sectiontitle="IANA Considerations"> <t>There arenumbered="true" toc="default"> <name>IANA Considerations</name> <t>This document has no IANAimpacts in this memo.</t>actions.</t> </section> </middle> <back> <references> <name>References</name> <references> <name>Normative References</name> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.6679.xml"/> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.3550.xml"/> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.3551.xml"/> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.3611.xml"/> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.4585.xml"/> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.5506.xml"/> <!-- draft-ietf-rmcat-eval-criteria (8868) part of C238 --> <reference anchor="RFC8868" target="https://www.rfc-editor.org/info/rfc8868"> <front> <title>Evaluating Congestion Control for Interactive Real-time Media</title> <author initials='V' surname='Singh' fullname='Varun Singh'> <organization /> </author> <author initials='J' surname='Ott' fullname='Joerg Ott'> <organization /> </author> <author initials='S' surname='Holmer' fullname='Stefan Holmer'> <organization /> </author> <date month='July' year='2020' /> </front> <seriesInfo name="RFC" value="8868"/> <seriesInfo name="DOI" value="10.17487/RFC8868"/> </reference> <!-- draft-ietf-rmcat-wireless-tests-11 (8869) part of C238 --> <reference anchor="RFC8869" target="https://www.rfc-editor.org/info/rfc8869"> <front> <title>Evaluation Test Cases for Interactive Real-Time Media over Wireless Networks</title> <author initials="Z" surname="Sarker" fullname="Zaheduzzaman Sarker"> <organization/> </author> <author initials="X" surname="Zhu" fullname="Xiaoqing Zhu"> <organization/> </author> <author initials="J" surname="Fu" fullname="Jian Fu"> <organization/> </author> <date month='July' year='2020' /> </front> <seriesInfo name="RFC" value="8869"/> <seriesInfo name="DOI" value="10.17487/RFC8869"/> </reference> <!-- [I-D.ietf-rmcat-video-traffic-model] Published as RFC 8593 --> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8593.xml"/> <!-- draft-ietf-rmcat-cc-requirements-09: 8836 --> <reference anchor="RFC8836" target="https://www.rfc-editor.org/info/rfc8836"> <front> <title>Congestion Control Requirements for Interactive Real-Time Media</title> <author initials="R" surname="Jesup" fullname="Randell Jesup"> <organization/> </author> <author initials="Z" surname="Sarker" fullname="Zaheduzzaman Sarker" role="editor"> <organization/> </author> <date month="July" year="2020"/> </front> <seriesInfo name="RFC" value="8836" /> <seriesInfo name="DOI" value="10.17487/RFC8836"/> </reference> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.5681.xml"/> </references> <references> <name>Informative References</name> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.2914.xml"/> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8290.xml"/> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8033.xml"/> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.7567.xml"/> <reference anchor="xiph-seq" target="http://media.xiph.org/video/derf/"> <front> <title>Video Test Media</title> <author> <organization>Xiph.org</organization> </author> </front> </reference> <reference anchor="HEVC-seq" target="http://www.netlab.tkk.fi/~varun/test_sequences/"> <front> <title>Test Sequences</title> <author> <organization>HEVC</organization> </author> </front> </reference> <!-- [I-D.ietf-rmcat-coupled-cc] Published as RFC 8699 --> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8699.xml"/> </references> </references> <sectiontitle="Acknowledgements">numbered="false" toc="default"> <name>Acknowledgments</name> <t>Much of this document is derived from previous work on congestion control at the IETF.</t> <t>The content and concepts within this document are a product of the discussion carried outinwithin the Design Team.</t> </section></middle> <back> <references title="Normative References"> <?rfc include='reference.RFC.6679.xml'?> <!-- RTP related --> &rfc3550; &rfc3551; &rfc3611; &rfc4585; &rfc5506; <!--RMCAT related --> &I-D.ietf-rmcat-eval-criteria; &I-D.ietf-rmcat-wireless-tests; &I-D.ietf-rmcat-video-traffic-model; &I-D.ietf-rmcat-cc-requirements; <?rfc include='reference.RFC.5681.xml'?> <!--&rfc5681; --> <!-- Standard TCP --> </references> <references title="Informative References"> <!-- &I-D.ietf-rtcweb-use-cases-and-requirements; --> <?rfc include='reference.RFC.2914.xml'?> <?rfc include='reference.RFC.8290.xml'?> <?rfc include='reference.RFC.8033.xml'?> <?rfc include='reference.RFC.7567.xml'?> <!-- &rfc5033; --> <!-- CC Evaluation --> <!-- &rfc5166; --> <!-- CC Metrics --> <!----> <!-- --> <!----> <reference anchor="xiph-seq"> <front> <title>Video Test Media</title> <author fullname="" initials="" surname="Xiph.org"/> <date month="" year=""/> </front> <seriesInfo name="http://media.xiph.org/video/derf/" value=""/> </reference> <reference anchor="HEVC-seq"> <front> <title>Test Sequences</title> <author fullname="" initials="" surname="HEVC"/> <date month="" year=""/> </front> <seriesInfo name="http://www.netlab.tkk.fi/~varun/test_sequences/" value=""/> </reference> <?rfc include='reference.I-D.ietf-rmcat-coupled-cc'?> <!-- --> </references></back> </rfc>