rfc8867xml2.original.xml   rfc8867.xml 
<?xml version="1.0" encoding="US-ASCII"?> <?xml version='1.0' encoding='utf-8'?>
<!DOCTYPE rfc SYSTEM "rfc2629.dtd" [ <!DOCTYPE rfc SYSTEM "rfc2629-xhtml.ent">
<!ENTITY rfc2119 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/refe
rence.RFC.2119.xml">
<!ENTITY rfc3550 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/refe
rence.RFC.3550.xml">
<!ENTITY rfc3551 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/refe
rence.RFC.3551.xml">
<!ENTITY rfc3611 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/refe
rence.RFC.3611.xml">
<!ENTITY rfc4585 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/refe
rence.RFC.4585.xml">
<!ENTITY rfc5506 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/refe
rence.RFC.5506.xml">
<!ENTITY rfc5166 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/refe
rence.RFC.5166.xml">
<!ENTITY rfc5033 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/refe
rence.RFC.5033.xml">
<!ENTITY rfc5681 PUBLIC "" "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/refe
rence.RFC.5681.xml">
<!ENTITY I-D.ietf-rmcat-cc-requirements PUBLIC "" "http://xml2rfc.tools.ietf.org
/public/rfc/bibxml3/reference.I-D.ietf-rmcat-cc-requirements.xml">
<!ENTITY I-D.ietf-avtcore-rtp-circuit-breakers PUBLIC "" "http://xml2rfc.tools.i
etf.org/public/rfc/bibxml3/reference.I-D.ietf-avtcore-rtp-circuit-breakers.xml">
<!ENTITY I-D.ietf-rmcat-eval-criteria PUBLIC "" "http://xml2rfc.tools.ietf.org/p
ublic/rfc/bibxml3/reference.I-D.ietf-rmcat-eval-criteria.xml">
<!ENTITY I-D.ietf-rtcweb-use-cases-and-requirements PUBLIC "" "http://xml2rfc.to
ols.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-rtcweb-use-cases-and-requirem
ents.xml">
<!ENTITY I-D.ietf-rmcat-wireless-tests PUBLIC "" "http://xml2rfc.tools.ietf.org/
public/rfc/bibxml3/reference.I-D.ietf-rmcat-wireless-tests.xml">
<!ENTITY I-D.ietf-rmcat-video-traffic-model PUBLIC "" "http://xml2rfc.tools.ietf
.org/public/rfc/bibxml3/reference.I-D.ietf-rmcat-video-traffic-model.xml">
]>
<?xml-stylesheet type='text/xsl' href='rfc2629.xslt' ?>
<?rfc toc="yes" ?>
<?rfc compact="yes" ?>
<?rfc symrefs="yes" ?>
<rfc category="info" docName="draft-ietf-rmcat-eval-test-10" ipr="trust200902">
<!---->
<front> <rfc xmlns:xi="http://www.w3.org/2001/XInclude"
<title abbrev="Test Scenarios for RMCAT">Test Cases for Evaluating RMCAT submissionType="IETF"
Proposals</title> category="info"
consensus="true"
docName="draft-ietf-rmcat-eval-test-10"
number="8867"
ipr="trust200902"
obsoletes=""
updates=""
xml:lang="en"
tocInclude="true"
symRefs="true"
sortRefs="true"
version="3">
<!-- xml2rfc v2v3 conversion 2.43.0 -->
<front>
<title abbrev="Test Scenarios for RMCAT">Test Cases for Evaluating
Congestion Control for Interactive Real-Time Media</title>
<seriesInfo name="RFC" value="8867"/>
<author fullname="Zaheduzzaman Sarker" initials="Z" surname="Sarker"> <author fullname="Zaheduzzaman Sarker" initials="Z" surname="Sarker">
<organization>Ericsson AB</organization> <organization>Ericsson AB</organization>
<address> <address>
<postal> <postal>
<street>Torshamnsgatan 23</street> <street>Torshamnsgatan 23</street>
<city>Stockholm</city> <city>Stockholm</city>
<region>SE</region>
<code>164 83</code> <code>164 83</code>
<country>Sweden</country> <country>Sweden</country>
</postal> </postal>
<phone>+46 10 717 37 43</phone> <phone>+46 10 717 37 43</phone>
<email>zaheduzzaman.sarker@ericsson.com</email> <email>zaheduzzaman.sarker@ericsson.com</email>
</address> </address>
</author> </author>
<author fullname="Varun Singh" initials="V" surname="Singh"> <author fullname="Varun Singh" initials="V" surname="Singh">
<organization abbrev="callstats.io">Nemu Dialogue Systems <organization abbrev="callstats.io">CALLSTATS I/O Oy</organization>
Oy</organization>
<address> <address>
<postal> <postal>
<street>Runeberginkatu 4c A 4</street> <street>Rauhankatu 11 C</street>
<city>Helsinki</city>
<city>Helsinki</city> <code>00100</code>
<country>Finland</country>
<code>00100</code>
<country>Finland</country>
</postal> </postal>
<email>varun.singh@iki.fi</email>
<email>varun.singh@iki.fi</email> <uri>http://www.callstats.io/</uri>
<uri>http://www.callstats.io/</uri>
</address> </address>
</author> </author>
<author fullname="Xiaoqing Zhu" initials="X" surname="Zhu"> <author fullname="Xiaoqing Zhu" initials="X" surname="Zhu">
<organization>Cisco Systems</organization> <organization>Cisco Systems</organization>
<address> <address>
<postal> <postal>
<street>12515 Research Blvd</street> <street>12515 Research Blvd</street>
<city>Austin</city>
<city>Austing</city>
<region>TX</region> <region>TX</region>
<code>78759</code> <code>78759</code>
<country>United States of America</country>
<country>USA</country>
</postal> </postal>
<email>xiaoqzhu@cisco.com</email> <email>xiaoqzhu@cisco.com</email>
</address> </address>
</author> </author>
<author fullname="Michael A. Ramalho" initials="M. A." surname="Ramalho"> <author fullname="Michael A. Ramalho" initials="M." surname="Ramalho">
<organization abbrev="Cisco Systems">Cisco Systems, Inc.</organization> <organization abbrev="AcousticComms">AcousticComms Consulting</organizatio
n>
<address> <address>
<postal> <postal>
<street>6310 Watercrest Way Unit 203</street> <street>6310 Watercrest Way Unit 203</street>
<city>Lakewood Ranch</city> <city>Lakewood Ranch</city>
<region>FL</region> <region>FL</region>
<code>34202-5211</code> <code>34202-5211</code>
<country>USA</country> <country>USA</country>
</postal> </postal>
<phone>+1 732 832 9723</phone>
<phone>+1 919 476 2038</phone> <email>mar42@cornell.edu</email>
<uri>http://ramalho.webhop.info/</uri>
<email>mramalho@cisco.com</email>
</address> </address>
</author> </author>
<date day="23" month="May" year="2019"/> <date month="July" year="2020"/>
<area>TSV</area> <area>TSV</area>
<keyword>Multimedia</keyword> <keyword>Multimedia</keyword>
<keyword>Test cases</keyword> <keyword>Test cases</keyword>
<keyword>Congestion Control</keyword> <keyword>Congestion Control</keyword>
<abstract> <abstract>
<t>The Real-time Transport Protocol (RTP) is used to transmit media in <t>The Real-time Transport Protocol (RTP) is used to transmit media in
multimedia telephony applications. These applications are typically multimedia telephony applications. These applications are typically
required to implement congestion control. This document describes the required to implement congestion control. This document describes the
test cases to be used in the performance evaluation of such congestion test cases to be used in the performance evaluation of such congestion
control algorithms in a controlled environment.</t> control algorithms in a controlled environment.</t>
</abstract> </abstract>
</front> </front>
<middle> <middle>
<section title="Introduction"> <section numbered="true" toc="default">
<name>Introduction</name>
<t>This memo describes a set of test cases for evaluating congestion <t>This memo describes a set of test cases for evaluating congestion
control algorithm proposals in controlled environments for real-time control algorithm proposals in controlled environments for real-time
interactive media. It is based on the guidelines enumerated in <xref interactive media. It is based on the guidelines enumerated in <xref targe
target="I-D.ietf-rmcat-eval-criteria"/> and the requirements discussed t="RFC8868" format="default"/> and the requirements discussed
in <xref target="I-D.ietf-rmcat-cc-requirements"/>. The test cases cover in <xref target="RFC8836" format="default"/>. The test cases cover
basic usage scenarios and are described using a common structure, which basic usage scenarios and are described using a common structure, which
allows for additional test cases to be added to those described herein allows for additional test cases to be added to those described herein
to accommodate other topologies and/or the modelling of different path to accommodate other topologies and/or the modeling of different path
characteristics. The described test cases in this memo should be used to characteristics. The described test cases in this memo should be used to
evaluate any proposed congestion control algorithm for real-time evaluate any proposed congestion control algorithm for real-time
interactive media.</t> interactive media.</t>
</section> </section>
<section anchor="sec-terminology" numbered="true" toc="default">
<name>Terminology</name>
<section anchor="sec-terminology" title="Terminology"> <t>The terminology defined in <xref target="RFC3550" format="default">RTP<
<!--The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", /xref>,
"SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and <xref target="RFC3551" format="default">RTP Profile for Audio and Video Co
"OPTIONAL" in this document are to be interpreted as described nferences with
in BCP 14, <xref target="RFC2119" /> and indicate requirement Minimal Control</xref>, <xref target="RFC3611" format="default">RTCP Exten
levels for compliant implementations.--> ded Report
(XR)</xref>, <xref target="RFC4585" format="default">Extended RTP Profile
<t>The terminology defined in <xref target="RFC3550">RTP</xref>, <xref for RTCP-based
target="RFC3551">RTP Profile for Audio and Video Conferences with Feedback (RTP/AVPF)</xref>, and <xref target="RFC5506" format="default">Su
Minimal Control</xref>, <xref target="RFC3611">RTCP Extended Report pport for
(XR)</xref>, <xref target="RFC4585">Extended RTP Profile for RTCP-based Reduced-Size RTCP</xref> applies.</t>
Feedback (RTP/AVPF)</xref>, and <xref target="RFC5506">Support for
Reduced-Size RTCP</xref> apply.</t>
</section> </section>
<section anchor="TS" numbered="true" toc="default">
<section anchor="TS" title="Structure of Test cases"> <name>Structure of Test Cases</name>
<t>All the test cases in this document follow a basic structure allowing <t>All the test cases in this document follow a basic structure allowing
implementers to describe a new test scenario without repeatedly implementers to describe a new test scenario without repeatedly
explaining common attributes. The structure includes a general explaining common attributes. The structure includes a general
description section that describes the test case and its motivation. description section that describes the test case and its motivation.
Additionally the test case defines a set of attributes that characterize Additionally the test case defines a set of attributes that characterize
the testbed, for example, the network path between communicating peers the testbed, for example, the network path between communicating peers
and the diverse traffic sources.</t> and the diverse traffic sources.</t>
<dl spacing="normal">
<t><list style="symbols"> <dt>Define the test case:</dt><dd>
<t>Define the test case: <list style="symbols"> <t><br/></t>
<t>General description: describes the motivation and the goals <dl spacing="normal">
of the test case.</t> <dt>General description:</dt><dd>describes the motivation and the go
als
<t>Expected behavior: describes the desired rate adaptation of the test case.</dd>
behavior.</t> <dt>Expected behavior:</dt><dd>describes the desired rate adaptation
behavior.</dd>
<t>Define a list of metrics to evaluate the desired behavior: <dt>List of metrics to evaluate the desired behavior:</dt><dd>
this indicates the minimum set of metrics (e.g., link this indicates the minimum set of metrics (e.g., link
utilization, media sending rate) that a proposed algorithm needs utilization, media sending rate) that a proposed algorithm needs
to measure to validate the expected rate adaptation behavior. It to measure to validate the expected rate adaptation behavior. It
should also indicate the time granularity (e.g., averaged over should also indicate the time granularity (e.g., averaged over
10ms, 100ms, or 1s) for measuring certain metrics. Typical 10 ms, 100 ms, or 1 s) for measuring certain metrics. Typical
measurement interval is 200ms.</t> measurement interval is 200 ms.</dd>
</dl>
<!-- we agreed this should be 200ms or was it based on some IPPM d </dd>
oc? -->
</list></t>
<t>Define testbed topology: every test case needs to define an <dt>Define testbed topology:</dt><dd>
evaluation testbed topology. <xref target="fig-eval-topo"/> shows <t><br/></t>
<t>Every test case needs to define an
evaluation testbed topology. <xref target="fig-eval-topo" format="defa
ult"/> shows
such an evaluation topology. In this evaluation topology, S1..Sn are such an evaluation topology. In this evaluation topology, S1..Sn are
traffic sources. These sources generate media traffic and use the traffic sources. These sources generate media traffic and use the
congestion control algorithm(s) under investigation. R1..Rn are the congestion control algorithm(s) under investigation. R1..Rn are the
corresponding receivers. A test case can have one or more such corresponding receivers. A test case can have one or more such
traffic sources (S) and their corresponding receivers (R). The path traffic sources (S) and their corresponding receivers (R). The path
from the source to destination is denoted as "forward" and the path from the source to destination is denoted as "forward", and the path
from a destination to a source is denoted as "backward". The from a destination to a source is denoted as "backward". The
following basic structure of the test case has been described from following basic structure of the test case has been described from
the perspective of media generating endpoints attached on the the perspective of media-generating endpoints attached on the
left-hand side of <xref target="fig-eval-topo"/>. In this setup, the left-hand side of <xref target="fig-eval-topo" format="default"/>. In
media flows are transported in forward direction and corresponding this setup, the
media flows are transported in the forward direction, and the correspo
nding
feedback/control messages are transported in the backward direction. feedback/control messages are transported in the backward direction.
However, it is also possible to set up the test with media in both However, it is also possible to set up the test with media in both
forward and backward directions. In that case, unless otherwise forward and backward directions. In that case, unless otherwise
specified by the test case, it is expected that the backward path specified by the test case, it is expected that the backward path
does not introduce any congestion related impairments and has enough does not introduce any congestion-related impairments and has enough
capacity to accommodate both media and feedback/control messages. It capacity to accommodate both media and feedback/control messages. It
should be noted that depending on the test cases it is possible to should be noted that, depending on the test cases, it is possible to
have different path characteristics in either of the have different path characteristics in either of the
directions.<figure anchor="fig-eval-topo" directions.</t>
title="Example of A Testbed Topology"> <figure anchor="fig-eval-topo">
<artwork><![CDATA[ <name>Example of a Testbed Topology</name>
+---+ +---+ <artwork name="" type="" align="left" alt=""><![CDATA[
|S1 |====== \ Forward --> / =======|R1 | +---+ +---+
+---+ \\ // +---+ |S1 |====== \ Forward --> / =======|R1 |
\\ // +---+ \\ // +---+
+---+ +-----+ +-----+ +---+ \\ //
|S2 |=======| A |------------------------------>| B |=======|R2 | +---+ +-----+ +-----+ +---+
+---+ | |<------------------------------| | +---+ |S2 |=======| A |--------------------------->| B |=======|R2 |
+-----+ +-----+ +---+ | |<---------------------------| | +---+
(...) // \\ (...) +-----+ +-----+
// <-- Backward \\ (...) // \\ (...)
+---+ // \\ +---+ // <-- Backward \\
|Sn |====== / \ ======|Rn | +---+ // \\ +---+
+---+ +---+ |Sn |====== / \ ======|Rn |
+---+ +---+
]]></artwork> ]]></artwork>
</figure>In a testbed environment with real equipments, there may ---+ +---+
</figure>
<t>In a testbed environment with real equipment, there may
exist a significant amount of unwanted traffic on the portions of exist a significant amount of unwanted traffic on the portions of
the network path between the endpoints. Some of this traffic may be the network path between the endpoints. Some of this traffic may be
generated by other processes on the endpoints themselves (e.g., generated by other processes on the endpoints themselves (e.g.,
discovery protocols) or by other endpoints not presently under test. discovery protocols) or by other endpoints not presently under test.
Such unwanted traffic should be removed or avoided to the greatest Such unwanted traffic should be removed or avoided to the greatest
extent possible.</t> extent possible.</t>
</dd>
<t>Define testbed attributes: <list style="symbols"> <dt>Define testbed attributes:</dt>
<t>Duration: defines the duration of the test in seconds.</t> <dd>
<t><br/></t>
<t>Path characteristics: defines the end-to-end transport level <dl spacing="normal">
<dt>Duration:</dt><dd>defines the duration of the test in seconds.</
dd>
<dt>Path characteristics:</dt><dd>
<t>defines the end-to-end transport level
path characteristics of the testbed for a particular test case. path characteristics of the testbed for a particular test case.
Two sets of attributes describe the path characteristics, one Two sets of attributes describe the path characteristics, one
for the forward path and the other for the backward path. The for the forward path and the other for the backward path. The
path characteristics for a particular path direction is path characteristics for a particular path direction are
applicable to all the Sources "S" sending traffic on that path. applicable to all the sources "S" sending traffic on that path.
If only one attribute is specified, it is used for both path If only one attribute is specified, it is used for both path
directions, however, unless specified the reverse path has no directions; however, unless specified the reverse path has no
capacity restrictions and no path loss.<list style="symbols"> capacity restrictions and no path loss.</t>
<t>Path direction: forward or backward.</t> <dl spacing="normal">
<dt>Path direction:</dt><dd>forward or backward.</dd>
<t>Minimum bottleneck-link capacity: defines minimum capacity <dt>Minimum bottleneck-link capacity:</dt><dd>defines the minimu
of the m capacity of the
end-to-end path</t> end-to-end path.</dd>
<dt>Reference bottleneck capacity:</dt><dd>defines a reference v
<t>Reference bottleneck capacity: defines a reference value alue
for the bottleneck capacity for test cases with time-varying for the bottleneck capacity for test cases with time-varying
bottleneck capacities. All bottleneck capacities will be bottleneck capacities. All bottleneck capacities will be
specified as a ratio with respect to the reference capacity specified as a ratio with respect to the reference capacity
value.</t> value.</dd>
<dt>One-way propagation delay:</dt><dd>describes the end-to-end
<t>One-way propagation delay: describes the end-to-end
latency along the path when network queues are empty, i.e., latency along the path when network queues are empty, i.e.,
the time it takes for a packet to go from the sender to the the time it takes for a packet to go from the sender to the
receiver without encountering any queuing delay.</t> receiver without encountering any queuing delay.</dd>
<dt>Maximum end-to-end jitter:</dt><dd>defines the maximum jitte
<t>Maximum end-to-end jitter: defines the maximum jitter r
that can be observed along the path.</t> that can be observed along the path.</dd>
<dt>Bottleneck queue type:</dt><dd>for example, "tail drop" <xre
<t>Bottleneck queue type: for example, "tail drop" <xref f target="RFC7567" format="default"/>,
target="RFC7567"/>, Flow Queue -CoDel (FQ-CoDel)<xref Flow Queue Controlled Delay (FQ-CoDel) <xref target="RFC8290"
target="RFC8290"/>, or Proportional Integral controller format="default"/>,
Enhanced (PIE)<xref target="RFC8033"/>.</t> or Proportional Integral controller Enhanced (PIE)
<xref target="RFC8033" format="default"/>.</dd>
<t>Bottleneck queue size: defines the size of queue in terms <dt>Bottleneck queue size:</dt><dd>defines the size of queue in
terms
of queuing time when the queue is full (in of queuing time when the queue is full (in
milliseconds).</t> milliseconds).</dd>
<dt>Path loss ratio:</dt><dd>characterizes the non-congested,
<t>Path loss ratio: characterizes the non-congested, additive losses to be generated on the end-to-end path.
additive, losses to be generated on the end-to-end path.
This must describe the loss pattern or loss model used to This must describe the loss pattern or loss model used to
generate the losses.</t> generate the losses.</dd>
</list></t> </dl>
</dd>
<t>Application-related: defines the traffic source behavior for <dt>Application-related:</dt><dd>
implementing the test case <list style="symbols"> <t>defines the traffic source behavior for
<t>Media traffic Source: defines the characteristics of the implementing the test case:</t>
<dl spacing="normal">
<dt>Media traffic source:</dt><dd>
<t>defines the characteristics of the
media sources. When using more than one media source, the media sources. When using more than one media source, the
different attributes are enumerated separately for each different attributes are enumerated separately for each
different media source. <list style="symbols"> different media source. </t>
<t>Media type: Video/Voice</t> <dl spacing="normal">
<dt>Media type:</dt><dd>Video/Voice.</dd>
<t>Media flow direction: forward, backward or both.</t> <dt>Media flow direction:</dt><dd>forward, backward, or both
.</dd>
<t>Number of media sources: defines the total number of <dt>Number of media sources:</dt><dd>defines the total numbe
media sources</t> r of
media sources.</dd>
<t>Media codec: Constant Bit Rate (CBR) or Variable Bit <dt>Media codec:</dt><dd>Constant Bit Rate (CBR) or Variable
Rate (VBR)</t> Bit
Rate (VBR).</dd>
<t>Media source behavior: describes the media encoder <dt>Media source behavior:</dt><dd>
<t>describes the media encoder
behavior. It defines the main parameters that affect the behavior. It defines the main parameters that affect the
adaptation behavior. This may include but is not limited adaptation behavior. This may include but is not limited
to: <list style="symbols"> to the following: </t>
<t>Adaptability: describes the adaptation options. <dl spacing="normal">
For example, in the case of video it defines the <dt>Adaptability:</dt><dd>describes the adaptation optio
ns.
For example, in the case of video, it defines the
following ranges of adaptation: bit rate, frame following ranges of adaptation: bit rate, frame
rate, video resolution. Similarly, in the case of rate, and video resolution. Similarly, in the case of
voice, it defines the range of bit rate adaptation, voice, it defines the range of bit rate adaptation,
the sampling rate variation, and the variation in the sampling rate variation, and the variation in
packetization interval.</t> packetization interval.</dd>
<dt>Output variation:</dt><dd>for a VBR encoder, it defi
<t>Output variation : for a VBR encoder it defines nes
the encoder output variation from the average target the encoder output variation from the average target
rate over a particular measurement interval. For rate over a particular measurement interval. For
example, on average the encoder output may vary example, on average the encoder output may vary
between 5% to 15% above or below the average target between 5% to 15% above or below the average target
bit rate when measured over a 100 ms time window. bit rate when measured over a 100 ms time window.
The time interval over which the variation is The time interval over which the variation is
specified must be provided.</t> specified must be provided.</dd>
<dt>Responsiveness to a new bit rate request:</dt><dd>
<!-- --> the lag
<t>Responsiveness to a new bit rate request: the lag
in time between a new bit rate request from the in time between a new bit rate request from the
congestion control algorithm and actual rate changes congestion control algorithm and actual rate changes
in encoder output. Depending on the encoder, this in encoder output. Depending on the encoder, this
value may be specified in absolute time (e.g. 10ms value may be specified in absolute time (e.g., 10 ms
to 1000ms) or other appropriate metric (e.g. next to 1000 ms) or other appropriate metric (e.g., next
frame interval time).</t> frame interval time).</dd>
</list> More detailed discussions on expected media </dl>
<t> More detailed discussions on expected media
source behavior, including those from synthetic video source behavior, including those from synthetic video
traffic sources, is at <xref traffic sources, can be found in <xref target="RFC8593" fo
target="I-D.ietf-rmcat-video-traffic-model"/>.</t> rmat="default"/>.</t>
</dd>
<t>Media content: describes the chosen video scenario. <dt>Media content:</dt><dd>describes the chosen video scenar
For example, video test sequences are available at: io.
<xref target="xiph-seq"/> and <xref target="HEVC-seq"/>. For example, video test sequences are available at
Different video scenarios give different distribution of <xref target="xiph-seq" format="default"/> and <xref targe
t="HEVC-seq" format="default"/>.
Different video scenarios give different distributions of
video frames produced by the video encoder. Hence, it is video frames produced by the video encoder. Hence, it is
important to specify the media content used in a important to specify the media content used in a
particular test. If a synthetic video traffic souce particular test. If a synthetic video traffic source
<xref target="I-D.ietf-rmcat-video-traffic-model"/> is <xref target="RFC8593" format="default"/> is
used, then the synthetic video traffic source needs to used, then the synthetic video traffic source needs to
configure according to the characteristics of the media be configured according to the characteristics of the medi
content specified.</t> a
content specified.</dd>
<t>Media timeline: describes the point when the media <dt>Media timeline:</dt><dd>describes the point when the med
ia
source is introduced and removed from the testbed. For source is introduced and removed from the testbed. For
example, the media source may start transmitting example, the media source may start transmitting
immediately when the test case begins, or after a few immediately when the test case begins, or after a few
seconds.</t> seconds.</dd>
<dt>Startup behavior:</dt><dd>the media starts at a defined
<t>Startup behavior: the media starts at a defined bit bit
rate, which may be the minimum, maximum bit rate, or a rate, which may be the minimum, maximum bit rate, or a
value in between (in Kbps).</t> value in between (in Kbps).</dd>
</list></t> </dl>
</dd>
<t>Competing traffic source: describes the characteristics <dt>Competing traffic source:</dt><dd>
<t>describes the characteristics
of the competing traffic source, the different types of of the competing traffic source, the different types of
competing flows are enumerated in <xref competing flows are enumerated in <xref target="RFC8868" forma
target="I-D.ietf-rmcat-eval-criteria"/>. <list t="default"/>. </t>
style="symbols"> <dl spacing="normal">
<t>Traffic direction: forward, backward or both.</t> <dt>Traffic direction:</dt><dd>forward, backward, or both.</
dd>
<t>Type of sources: defines the types of competing <dt>Type of sources:</dt><dd>defines the types of competing
traffic sources. Types of competing traffic flows are traffic sources. Types of competing traffic flows are
listed in <xref target="I-D.ietf-rmcat-eval-criteria"/>. listed in <xref target="RFC8868" format="default"/>.
For example, the number of TCP flows connected to a web For example, the number of TCP flows connected to a web
browser, the mean size and distribution of the content browser, the mean size and distribution of the content
downloaded.</t> downloaded.</dd>
<dt>Number of sources:</dt><dd>defines the total number of
<t>Number of sources: defines the total number of
competing sources of each media type per traffic competing sources of each media type per traffic
direction.</t> direction.</dd>
<dt>Congestion control:</dt><dd>enumerates the congestion co
<t>Congestion control: enumerates the congestion control ntrol
used by each type of competing traffic.</t> used by each type of competing traffic.</dd>
<dt>Traffic timeline:</dt><dd>describes when the competing
<t>Traffic timeline: describes when the competing traffic starts and ends in the test case.</dd>
traffic starts and ends in the test case.</t> </dl>
</list></t> </dd>
</list></t> </dl>
</dd>
<t>Additional attributes: describes attributes essential for <dt>Additional attributes:</dt><dd>describes attributes essential fo
implementing a test case which are not included in the above r
implementing a test case that are not included in the above
structure. These attributes must be well defined, so that the structure. These attributes must be well defined, so that the
other implementers of that particular test case are able to other implementers of that particular test case are able to
implement it easily.</t> implement it easily.</dd>
</list></t> </dl>
</list></t> </dd>
</dl>
<t>Any attribute can have a set of values (enclosed within "[]"). Each <t>Any attribute can have a set of values (enclosed within "[]"). Each
member value of such a set must be treated as different value for the member value of such a set must be treated as different value for the
same attribute. It is desired to run separate tests for each such same attribute. It is desired to run separate tests for each such
attribute value.</t> attribute value.</t>
<t>The test cases described in this document follow the above <t>The test cases described in this document follow the above
structure.</t> structure.</t>
</section> </section>
<section anchor="sec-rec-eval" numbered="true" toc="default">
<section anchor="sec-rec-eval" title="Recommended Evaluation Settings"> <name>Recommended Evaluation Settings</name>
<t>This section describes recommended test case settings and could be <t>This section describes recommended test case settings and could be
overwritten by the respective test cases.</t> overwritten by the respective test cases.</t>
<section anchor="EM" numbered="true" toc="default">
<section anchor="EM" title="Evaluation metrics"> <name>Evaluation Metrics</name>
<t>To evaluate the performance of the candidate algorithms the <t>To evaluate the performance of the candidate algorithms, the
implementers must log enough information to visualize the following implementers must log enough information to visualize the following
metrics at a fine enough time granularity: <list style="numbers"> metrics at a fine enough time granularity: </t>
<t>Flow level: <list style="letters"> <ol spacing="normal" type="1">
<t>End-to-end delay for the congestion controlled media <li>
flow(s). For example - end-to-end delay observed on IP packet <t>Flow level: </t>
level, video frame level.</t> <ol spacing="normal" type="A">
<li>End-to-end delay for the congestion-controlled media
<t>Variation in sending bit rate and throughput. Mainly flow(s). For example, end-to-end delay observed on the IP packet
observing the frequency and magnitude of oscillations.</t> level and the video frame level.</li>
<li>Variation in sending bit rate and throughput. Mainly
<t>Packet losses observed at the receiving endpoint.</t> observing the frequency and magnitude of oscillations.</li>
<li>Packet losses observed at the receiving endpoint.</li>
<t>Feedback message overhead.</t> <li>Feedback message overhead.</li>
<li>Convergence time. Time to reach steady state for the
<t>Convergence time - time to reach steady state for the congestion-controlled media flow(s). Each occurrence of
congestion controlled media flow(s). Each occurrence of convergence during the test period needs to be presented.</li>
convergence during the test period need to be presented.</t> </ol>
</list></t> </li>
<li>
<t>Transport level: <list style="letters"> <t>Transport level: </t>
<t>Bandwidth utilization.</t> <ol spacing="normal" type="A">
<li>Bandwidth utilization.</li>
<t>Queue length (milliseconds at specified path capacity).</t> <li>Queue length (milliseconds at specified path capacity).</li>
</list></t> </ol>
</list></t> </li>
</ol>
</section> </section>
<section anchor="PC" numbered="true" toc="default">
<section anchor="PC" title="Path characteristics"> <name>Path Characteristics</name>
<t>Each path between a sender and receiver as described in <xref <t>Each path between a sender and receiver as described in
target="fig-eval-topo"/> have the following characteristics unless <xref target="fig-eval-topo" format="default"/> has the following characteristic
otherwise specified in the test case. <list style="symbols"> s unless
<t>Path direction: forward and backward.</t> otherwise specified in the test case: </t>
<dl newline="false" spacing="normal">
<t>Reference bottleneck capacity: 1Mbps.</t> <dt>Path direction: </dt><dd>forward and backward.</dd>
<dt>Reference bottleneck capacity: </dt><dd>1 Mbps.</dd>
<t>One-Way propagation delay: 50ms. Implementers are encouraged to <dt>One-way propagation delay: </dt><dd>50 ms. Implementers are encour
aged to
run the experiment with additional propagation delays mentioned in run the experiment with additional propagation delays mentioned in
<xref target="I-D.ietf-rmcat-eval-criteria"/></t> <xref target="RFC8868" format="default"/>.</dd>
<dt>Maximum end-to-end jitter: </dt><dd>30 ms. Jitter models are descr
<t>Maximum end-to-end jitter: 30ms. Jitter models are described in ibed in
<xref target="I-D.ietf-rmcat-eval-criteria"/></t> <xref target="RFC8868" format="default"/>.</dd>
<dt>Bottleneck queue type: </dt><dd>"tail drop". Implementers are enco
<t>Bottleneck queue type: "tail drop". Implementers are encouraged uraged
to run the experiment with other AQM schemes, such as FQ-CoDel and to run the experiment with other Active Queue Management (AQM) schem
PIE.</t> es, such as FQ-CoDel and
PIE.</dd>
<t>Bottleneck queue size: 300ms.</t> <dt>Bottleneck queue size: </dt><dd>300 ms.</dd>
<dt>Path loss ratio: </dt><dd>0%.</dd>
<t>Path loss ratio: 0%.</t> </dl>
</list></t> <t>Examples of additional network parameters are discussed in <xref targ
et="RFC8868" format="default"/>.</t>
<t>Examples of additional network parameters are discussed in <xref
target="I-D.ietf-rmcat-eval-criteria"/>.</t>
<t>For test cases involving time-varying bottleneck capacity, all <t>For test cases involving time-varying bottleneck capacity, all
capacity values are specified as a ratio with respect to a reference capacity values are specified as a ratio with respect to a reference
capacity value, so as to allow flexible scaling of capacity values capacity value, so as to allow flexible scaling of capacity values
along with media source rate range. There exist two different along with media source rate range. There exist two different
mechanisms for inducing path capacity variation: a) by explicitly mechanisms for inducing path capacity variation: a) by explicitly
modifying the value of physical link capacity; or b) by introducing modifying the value of physical link capacity, or b) by introducing
background non-adaptive UDP traffic with time-varying traffic rate. background non-adaptive UDP traffic with time-varying traffic rate.
Implementers are encouraged to run the experiments with both Implementers are encouraged to run the experiments with both
mechanisms for test cases specified in <xref target="VACS"/>, <xref mechanisms for test cases specified in <xref target="VACS" format="defau
target="VACM"/>, and <xref target="CFL"/>.</t> lt"/>, <xref target="VACM" format="default"/>, and <xref target="CFL" format="de
fault"/>.</t>
</section> </section>
<section anchor="MS" numbered="true" toc="default">
<section anchor="MS" title="Media source"> <name>Media Source</name>
<t>Unless otherwise specified, each test case will include one or more <t>Unless otherwise specified, each test case will include one or more
media sources as described below. <list style="symbols"> media sources as described below: </t>
<t>Media type: Video <list style="symbols">
<t>Media codec: VBR</t>
<t>Media source behavior: <list style="symbols"> <dl newline="false" spacing="normal">
<t>Adaptability: <list style="symbols"> <dt>Media type:</dt>
<t>Bit rate range: 150 Kbps - 1.5 Mbps. In real-life <dd>
applications the bit rate range can vary a lot <t>Video</t>
depending on the provided service, for example, the <dl newline="false" spacing="normal">
maximum bit rate can be up to 4Mbps. However, for <dt>Media codec:</dt> <dd>VBR</dd>
<dt>Media source behavior:</dt>
<dd>
<t><br/></t>
<dl newline="false" spacing="normal">
<dt>Adaptability:</dt>
<dd>
<t><br/></t>
<dl newline="false" spacing="normal">
<dt>Bit rate range:</dt>
<dd>
150 Kbps - 1.5 Mbps. In real-life
applications, the bit rate range can vary a lot
depending on the provided service; for example, the
maximum bit rate can be up to 4 Mbps. However, for
running tests to evaluate the congestion control running tests to evaluate the congestion control
algorithms it is more important to have a look at how algorithms, it is more important to have a look at how
they are reacting to certain amount of bandwidth they react to a certain amount of bandwidth
change. Also it is possible that the media traffic change. Also it is possible that the media traffic
generator used in a particular simulator or testbed is generator used in a particular simulator or testbed is
not capable of generating higher bit rate. Hence we not capable of generating a higher bit rate. Hence, we
have selected a suitable bit rate range typical of have selected a suitable bit rate range typical of
consumer-grade video conferencing applications in consumer-grade video conferencing applications in
designing the test case. If a different bit rate range designing the test case. If a different bit rate range
is used in the test cases, then the end-to-end path is used in the test cases, then the end-to-end path
capacity values will also need to be scaled capacity values will also need to be scaled
accordingly.</t> accordingly.</dd>
<dt>Frame resolution: </dt> <dd>144p - 720p (o
<t>Frame resolution: 144p - 720p (or 1080p). This r 1080p). This
resolution range is selected based on the bit rate resolution range is selected based on the bit rate
range. If a different bit rate range is used in the range. If a different bit rate range is used in the
test cases then the frame resolution range also need test cases, then a suitable frame resolution range also
to be selected suitably.</t> needs
to be selected.</dd>
<t>Frame rate: 10fps - 30fps. This frame rate range is <dt>Frame rate: </dt> <dd>10 fps - 30 fps. Thi
s frame rate range is
selected based on the bit rate range. If a different selected based on the bit rate range. If a different
bit rate range is used in the test cases then the bit rate range is used in the test cases, then the
frame rate range also need to be adjusted frame rate range also needs to be suitably adjusted.</dd
suitably.</t> >
</list></t> </dl>
</dd>
<t>Variation from target bit rate: +/-5%. Unless otherwise <dt>Variation from target bit rate: </dt> <dd>+/-5%.
Unless otherwise
specified in the test case(s), bit rate variation should specified in the test case(s), bit rate variation should
be calculated over one (1) second period of time.</t> be calculated over a one (1) second period of time.</dd>
<dt>Responsiveness to new bit rate request: </dt>
<t>Responsiveness to new bit rate request: 100ms</t> <dd>100 ms</dd>
</list></t> </dl>
</dd>
<t>Media content: The media content should represent a typical <dt>Media content:</dt> <dd>The media content should repres
ent a typical
video conversational scenario with head and shoulder movement. video conversational scenario with head and shoulder movement.
We recommend to use Foreman video sequence<xref We recommend using the Foreman video sequence <xref target="xiph
target="xiph-seq"/>.</t> -seq" format="default"/>.</dd>
<dt>Media startup behavior:</dt> <dd>150 Kbps. It should be
<t>Media startup behavior: 150Kbps. It should be noted that noted that
applications can use smart ways to select an optimal startup applications can use smart ways to select an optimal startup
bit rate value for a certain network condition. In such cases bit rate value for a certain network condition. In such cases,
the candidate proposals may show the effectiveness of such the candidate proposals may show the effectiveness of such a
smart approach as an additional information for the evaluation smart approach as additional information for the evaluation
process.</t> process.</dd>
</list></t> </dl>
</list><list style="symbols"> </dd>
<t>Media type: Audio<list style="symbols"> <dt>Media type:</dt> <dd> <t>Audio</t>
<t>Media codec: CBR</t> <dl spacing="normal">
<dt>Media codec: </dt><dd>CBR</dd>
<t>Media bit rate: 20Kbps</t> <dt>Media bit rate:</dt><dd>20 Kbps</dd>
</list></t> </dl>
</list></t> </dd>
</dl>
</section> </section>
</section> </section>
<section anchor="TC" numbered="true" toc="default">
<section anchor="TC" title="Basic Test Cases"> <name>Basic Test Cases</name>
<section anchor="VACS" <section anchor="VACS" numbered="true" toc="default">
title="Variable Available Capacity with a Single Flow"> <name>Variable Available Capacity with a Single Flow</name>
<t>In this test case the minimum bottleneck-link capacity between the tw <t>In this test case, the minimum bottleneck-link capacity between the t
o wo
endpoints varies over time. This test is designed to measure the endpoints varies over time. This test is designed to measure the
responsiveness of the candidate algorithm. This test tries to address responsiveness of the candidate algorithm. This test tries to address
the requirements in <xref target="I-D.ietf-rmcat-cc-requirements"/>, the requirements in <xref target="RFC8836" format="default"/>,
which requires the algorithm to adapt the flow(s) and provide lower which requires the algorithm to adapt the flow(s) and provide lower
end-to-end latency when there exists: <list style="symbols"> end-to-end latency when there exists: </t>
<t>an intermediate bottleneck</t> <ul spacing="normal">
<li>an intermediate bottleneck</li>
<t>change in available capacity (e.g., due to interface change, <li>change in available capacity (e.g., due to interface change,
routing change, abrupt arrival/departure of background routing change, abrupt arrival/departure of background
non-adaptive traffic).</t> non-adaptive traffic).</li>
<li>maximum media bit rate is greater than link capacity. In this
<t>maximum media bit rate is greater than link capacity. In this
case, when the application tries to ramp up to its maximum bit case, when the application tries to ramp up to its maximum bit
rate, since the link capacity is limited to a value lower, the rate, since the link capacity is limited to a lower value, the
congestion control scheme is expected to stabilize the sending bit congestion control scheme is expected to stabilize the sending bit
rate close to the available bottleneck capacity.</t> rate close to the available bottleneck capacity.</li>
</ul>
<!-- --> <t>It should be noted that the exact variation in available
</list>It should be noted that the exact variation in available
capacity due to any of the above depends on the underlying capacity due to any of the above depends on the underlying
technologies. Hence, we describe a set of known factors, which may be technologies. Hence, we describe a set of known factors, which may be
extended to devise a more specific test case targeting certain extended to devise a more specific test case targeting certain
behaviors in a certain network environment.</t> behaviors in a certain network environment.</t>
<dl spacing="normal">
<t>Expected behavior: the candidate algorithm is expected to detect <dt>Expected behavior:</dt><dd>The candidate algorithm is expected to de
tect
the path capacity constraint, converge to the bottleneck link's the path capacity constraint, converge to the bottleneck link's
capacity and adapt the flow to avoid unwanted media rate oscillation capacity, and adapt the flow to avoid unwanted media rate oscillation
when the sending bit rate is approaching the bottleneck link's when the sending bit rate is approaching the bottleneck link's
capacity. Such oscillations might occur when the media flow(s) capacity. Such oscillations might occur when the media flow(s)
attempts to reach its maximum bit rate but overshoots the usage of the attempts to reach its maximum bit rate but overshoots the usage of the
available bottleneck capacity then to rectify, it reduces the bit rate available bottleneck capacity, then to rectify, it reduces the bit rate
and starts to ramp up again.</t> and starts to ramp up again.</dd>
<dt>Evaluation metrics:</dt><dd>As described in <xref target="EM" format
<t>Evaluation metrics : as described in <xref target="EM"/>.</t> ="default"/>.</dd>
<dt>Testbed topology:</dt><dd>One media source S1 is connected to the
<t>Testbed topology: One media source S1 is connected to the
corresponding R1. The media traffic is transported over the forward corresponding R1. The media traffic is transported over the forward
path and corresponding feedback/control traffic is transported over path and corresponding feedback/control traffic is transported over
the backward path. <figure anchor="fig-eval-topo-4-2" the backward path. </dd>
title="Testbed Topology for Limited Link Capacity"> </dl>
<artwork><![CDATA[ <figure anchor="fig-eval-topo-4-2">
<name>Testbed Topology for Limited Link Capacity</name>
<artwork name="" type="" align="left" alt=""><![CDATA[
Forward --> Forward -->
+---+ +-----+ +-----+ +---+ +---+ +-----+ +-----+ +---+
|S1 |=======| A |------------------------------>| B |=======|R1 | |S1 |=======| A |------------------------------>| B |=======|R1 |
+---+ | |<------------------------------| | +---+ +---+ | |<------------------------------| | +---+
+-----+ +-----+ +-----+ +-----+
<-- Backward <-- Backward
]]></artwork> ]]></artwork>
</figure></t> </figure>
<dl spacing="normal">
<t>Testbed attributes:</t> <dt>Testbed attributes:</dt>
<dd>
<t><list style="symbols"> <t><br/></t>
<t>Test duration: 100s</t> <dl spacing="normal">
<dt>Test duration:</dt><dd>100 s</dd>
<t>Path characteristics: as described in <xref target="PC"/></t> <dt>Path characteristics:</dt><dd>as described in <xref target="PC" fo
rmat="default"/></dd>
<t>Application-related: <list style="symbols"> <dt>Application-related:</dt>
<t>Media Traffic:<list style="symbols"> <dd>
<t>Media type: Video<list style="symbols"> <t><br/></t>
<t>Media direction: forward.</t> <dl spacing="normal">
<dt>Media Traffic:</dt>
<t>Number of media sources: one (1)</t> <dd>
<t><br/></t>
<t>Media timeline: <list style="symbols"> <dl spacing="normal">
<t>Start time: 0s.</t> <dt>Media type:</dt>
<dd>
<t>End time: 99s.</t> <t>Video</t>
</list></t> <dl spacing="normal">
</list></t> <dt>Media direction:</dt><dd>forward</dd>
<dt>Number of media sources:</dt><dd>one (1)</dd>
<t>Media type: Audio<list style="symbols"> <dt>Media timeline:</dt>
<t>Media direction: forward.</t> <dd>
<t><br/></t>
<t>Number of media sources: one (1)</t> <dl spacing="normal">
<dt>Start time:</dt><dd>0 s</dd>
<t>Media timeline: <list style="symbols"> <dt>End time:</dt><dd>99 s</dd>
<t>Start time: 0s.</t> </dl>
</dd>
<t>End time: 99s.</t> </dl>
</list></t> </dd>
</list></t> <dt>Media type:</dt>
</list></t> <dd>
<t>Audio</t>
<t>Competing traffic: <list style="symbols"> <dl spacing="normal">
<t>Number of sources : zero (0)</t> <dt>Media direction:</dt><dd>forward</dd>
</list></t> <dt>Number of media sources:</dt><dd>one (1)</dd>
</list></t> <dt>Media timeline:</dt>
<dd>
<t>Test Specific Information: <list style="symbols"> <t><br/></t>
<t>One-way propagation delay: [ 50 ms, 100 ms]. on the forward <dl spacing="normal">
path direction</t> <dt>Start time:</dt><dd>0 s</dd>
<dt>End time:</dt><dd>99 s</dd>
<t>This test uses bottleneck path capacity variation as listed </dl>
in <xref target="VACS_US"/></t> </dd>
</dl>
<t>When using background non-adaptive UDP traffic to induce </dd>
time-varying bottleneck , the physical path capacity remains </dl>
at 4Mbps and the UDP traffic source rate changes over time as </dd>
(4 - (Y x r)), where r is the Reference bottleneck capacity in <dt>Competing traffic:</dt>
Mbps and Y is the path capacity ratio specified in <xref <dd>
target="VACS_US"/></t> <t><br/></t>
</list></t> <dl spacing="normal">
</list></t> <dt>Number of sources:</dt><dd>zero (0)</dd>
</dl>
<texttable anchor="VACS_US" </dd>
title="Path capacity variation pattern for forward direction" </dl>
> </dd>
<ttcol>Variation pattern index</ttcol> </dl>
</dd>
<ttcol>Path direction</ttcol> <dt>Test-specific information:</dt>
<dd>
<ttcol>Start time</ttcol> <t><br/></t>
<dl spacing="normal">
<ttcol>Path capacity ratio</ttcol> <dt>One-way propagation delay:</dt> <dd>[50 ms, 100 ms]. On the fo
rward path direction.</dd>
<c>One</c> </dl>
<t>This test uses bottleneck path capacity variation as listed
<c>Forward</c> in <xref target="VACS_US" format="default"/>.</t>
<t>When using background non-adaptive UDP traffic to induce a
<c>0s</c> time-varying bottleneck, the physical path capacity remains
at 4 Mbps, and the UDP traffic source rate changes over time
<c>1.0</c> as
(4 - (Y x r)), where r is the Reference bottleneck capacity
<c>Two</c> in
Mbps, and Y is the path capacity ratio specified in
<c>Forward</c> <xref target="VACS_US" format="default"/>.</t>
</dd>
<c>40s</c> </dl>
<table anchor="VACS_US" align="center">
<c>2.5</c> <name>Path Capacity Variation Pattern for the Forward Direction</name>
<thead>
<c>Three</c> <tr>
<th align="left">Variation pattern index</th>
<c>Forward</c> <th align="left">Path direction</th>
<th align="left">Start time</th>
<c>60s</c> <th align="left">Path capacity ratio</th>
</tr>
<c>0.6</c> </thead>
<tbody>
<c>Four</c> <tr>
<td align="left">One</td>
<c>Forward</c> <td align="left">Forward</td>
<td align="left">0 s</td>
<c>80s</c> <td align="left">1.0</td>
</tr>
<c>1.0</c> <tr>
<td align="left">Two</td>
<!-- <postamble>Table 1: Path capacity variation pattern for <td align="left">Forward</td>
forward direction</postamble> --> <td align="left">40 s</td>
</texttable> <td align="left">2.5</td>
</tr>
<tr>
<td align="left">Three</td>
<td align="left">Forward</td>
<td align="left">60 s</td>
<td align="left">0.6</td>
</tr>
<tr>
<td align="left">Four</td>
<td align="left">Forward</td>
<td align="left">80 s</td>
<td align="left">1.0</td>
</tr>
</tbody>
</table>
<t/> <t/>
</section> </section>
<section anchor="VACM" numbered="true" toc="default">
<section anchor="VACM" <name>Variable Available Capacity with Multiple Flows</name>
title="Variable Available Capacity with Multiple Flows"> <t>This test case is similar to <xref target="VACS" format="default"/>.
<t>This test case is similar to <xref target="VACS"/>. However in However,
addition this test will also consider persistent network load due to this test will also consider persistent network load due to
competing traffic.</t> competing traffic.</t>
<dl spacing="normal">
<t>Expected behavior: the candidate algorithm is expected to detect <dt>Expected behavior:</dt><dd>The candidate algorithm is expected to de
tect
the variation in available capacity and adapt the media stream(s) the variation in available capacity and adapt the media stream(s)
accordingly. The flows stabilize around their maximum bit rate as the accordingly. The flows stabilize around their maximum bit rate as the
maximum link capacity is large enough to accommodate the flows. When maximum link capacity is large enough to accommodate the flows. When
the available capacity drops, the flows adapt by decreasing their the available capacity drops, the flows adapt by decreasing their
sending bit rate, and when congestion disappears, the flows are again sending bit rate, and when congestion disappears, the flows are again
expected to ramp up.</t> expected to ramp up.</dd>
<dt>Evaluation metrics:</dt><dd>As described in <xref target="EM" format
<t>Evaluation metrics : as described in <xref target="EM"/>.</t> ="default"/>.</dd>
<dt>Testbed topology:</dt><dd>Two (2) media sources S1 and S2 are connec
<t>Testbed Topology: Two (2) media sources S1 and S2 are connected to ted to
their corresponding destinations R1 and R2. The media traffic is their corresponding destinations R1 and R2. The media traffic is
transported over the forward path and corresponding feedback/control transported over the forward path and corresponding feedback/control
traffic is transported over the backward path. <figure traffic is transported over the backward path. </dd>
anchor="fig-eval-topo-4-1" </dl>
title="Testbed Topology for Variable Available Capacity"> <figure anchor="fig-eval-topo-4-1">
<artwork><![CDATA[ <name>Testbed Topology for Variable Available Capacity</name>
<artwork name="" type="" align="left" alt=""><![CDATA[
+---+ +---+ +---+ +---+
|S1 |===== \ / =======|R1 | |S1 |===== \ / =======|R1 |
+---+ \\ Forward --> // +---+ +---+ \\ Forward --> // +---+
\\ // \\ //
+-----+ +-----+ +-----+ +-----+
| A |------------------------------>| B | | A |------------------------------>| B |
| |<------------------------------| | | |<------------------------------| |
+-----+ +-----+ +-----+ +-----+
// \\ // \\
// <-- Backward \\ // <-- Backward \\
+---+ // \\ +---+ +---+ // \\ +---+
|S2 |====== / \ ======|R2 | |S2 |====== / \ ======|R2 |
+---+ +---+ +---+ +---+
]]></artwork> ]]></artwork>
</figure></t> </figure>
<dl spacing="normal">
<t>Testbed attributes:</t> <dt>Testbed attributes:</dt>
<dd>Testbed attributes are similar to those described in <xref target="V
<t>Testbed attributes are similar as described in <xref ACS" format="default"/>,
target="VACS"/> except the test specific capacity variation setup.</t> except for the test-specific capacity variation setup.</dd>
<dt>Test-specific information:</dt>
<t>Test Specific Information: This test uses path capacity variation <dd>This test uses path capacity variation
as listed in <xref target="VACM_US"/> with a corresponding end time of as listed in <xref target="VACM_US" format="default"/> with a correspond
125 seconds. The reference bottleneck capacity is 2Mbps. When using ing end time of
125 seconds. The reference bottleneck capacity is 2 Mbps. When using
background non-adaptive UDP traffic to induce time-varying bottleneck background non-adaptive UDP traffic to induce time-varying bottleneck
for congestion controlled media flows, the physical path capacity is for congestion-controlled media flows, the physical path capacity is
4Mbps and the UDP traffic source rate changes over time as (4 - (Y x 4 Mbps, and the UDP traffic source rate changes over time as (4 - (Y x
r)), where r is the Reference bottleneck capacity in Mbps and Y is the r)), where r is the Reference bottleneck capacity in Mbps, and Y is the
path capacity ratio specified in <xref target="VACM_US"/>.</t> path capacity ratio specified in <xref target="VACM_US" format="default"
/>.</dd>
<texttable anchor="VACM_US" </dl>
title="Path capacity variation pattern for forward direction" <table anchor="VACM_US" align="center">
> <name>Path Capacity Variation Pattern for the Forward Direction</name>
<ttcol>Variation pattern index</ttcol> <thead>
<tr>
<ttcol>Path direction</ttcol> <th align="left">Variation pattern index</th>
<th align="left">Path direction</th>
<ttcol>Start time</ttcol> <th align="left">Start time</th>
<th align="left">Path capacity ratio</th>
<ttcol>Path capacity ratio</ttcol> </tr>
</thead>
<c>One</c> <tbody>
<tr>
<c>Forward</c> <td align="left">One</td>
<td align="left">Forward</td>
<c>0s</c> <td align="left">0 s</td>
<td align="left">2.0</td>
<c>2.0</c> </tr>
<tr>
<c>Two</c> <td align="left">Two</td>
<td align="left">Forward</td>
<c>Forward</c> <td align="left">25 s</td>
<td align="left">1.0</td>
<c>25s</c> </tr>
<tr>
<c>1.0</c> <td align="left">Three</td>
<td align="left">Forward</td>
<c>Three</c> <td align="left">50 s</td>
<td align="left">1.75</td>
<c>Forward</c> </tr>
<tr>
<c>50s</c> <td align="left">Four</td>
<td align="left">Forward</td>
<c>1.75</c> <td align="left">75 s</td>
<td align="left">0.5</td>
<c>Four</c> </tr>
<tr>
<c>Forward</c> <td align="left">Five</td>
<td align="left">Forward</td>
<c>75s</c> <td align="left">100 s</td>
<td align="left">1.0</td>
<c>0.5</c> </tr>
</tbody>
<c>Five</c> </table>
<c>Forward</c>
<c>100s</c>
<c>1.0</c>
<!-- <postamble>Table 2: Path capacity variation pattern for
forward direction</postamble> -->
</texttable>
</section> </section>
<section anchor="CFL" numbered="true" toc="default">
<section anchor="CFL" <name>Congested Feedback Link with Bi-directional Media Flows</name>
title="Congested Feedback Link with Bi-directional Media Flows"> <t>Real-time interactive media uses RTP; hence it is assumed that RTCP,
<t>Real-time interactive media uses RTP hence it is assumed that RTCP, RTP header extension, or such would be used by the congestion control
RTP header extension or such would be used by the congestion control algorithm in the back channel. Due to the asymmetric nature of the link
algorithm in the backchannel. Due to the asymmetric nature of the link between communicating peers, it is possible for a participating peer to
between communicating peers it is possible for a participating peer to
not receive such feedback information due to an impaired or congested not receive such feedback information due to an impaired or congested
backchannel (even when the forward channel might not be impaired). back channel (even when the forward channel might not be impaired).
This test case is designed to observe the candidate congestion control This test case is designed to observe the candidate congestion control
behavior in such an event.</t> behavior in such an event.</t>
<dl spacing="normal">
<t>Expected behavior: It is expected that the candidate algorithms are <dt>Expected behavior:</dt>
able to cope with the lack of feedback information and adapt to <dd>
<t>It is expected that the candidate algorithms are
able to cope with the lack of feedback information and to adapt to
minimize the performance degradation of media flows in the forward minimize the performance degradation of media flows in the forward
channel.</t> channel.</t>
<t>It should be noted that for this test case, logs are compared with
<t>It should be noted that for this test case: logs are compared with the reference case, i.e., when the backward channel has no
the reference case, i.e, when the backward channel has no
impairments.</t> impairments.</t>
</dd>
<t>Evaluation metrics : as described in <xref target="EM"/>.</t> <dt>Evaluation metrics:</dt><dd>As described in <xref target="EM" format
="default"/>.</dd>
<t>Testbed topology: One (1) media source S1 is connected to <dt>Testbed topology:</dt><dd>One (1) media source S1 is connected to
corresponding R1, but both endpoints are additionally receiving and corresponding R1, but both endpoints are additionally receiving and
sending data, respectively. The media traffic (S1-&gt;R1) is sending data, respectively. The media traffic (S1-&gt;R1) is
transported over the forward path and corresponding feedback/control transported over the forward path, and the corresponding feedback/contro
traffic is transported over the backward path. Likewise media traffic l
(S2-&gt;R2) is transported over the backward path and corresponding traffic is transported over the backward path. Likewise, media traffic
feedback/control traffic is transported over the forward path.</t> (S2-&gt;R2) is transported over the backward path, and the corresponding
feedback/control traffic is transported over the forward path.</dd>
<t><figure anchor="fig-eval-topo-4-6" </dl>
title="Testbed Topology for Congested Feedback Link"> <figure anchor="fig-eval-topo-4-6">
<artwork><![CDATA[ <name>Testbed Topology for Congested Feedback Link</name>
+---+ +---+ <artwork name="" type="" align="left" alt=""><![CDATA[
|S1 |===== \ Forward --> / =======|R1 | +---+ +---+
+---+ \\ // +---+ |S1 |====== \ Forward --> / =======|R1 |
\\ // +---+ \\ // +---+
+-----+ +-----+ \\ //
| A |------------------------------>| B | +-----+ +-----+
| |<------------------------------| | | A |------------------------------>| B |
+-----+ +-----+ | |<------------------------------| |
// \\ +-----+ +-----+
// <-- Backward \\ // \\
+---+ // \\ +---+ // <-- Backward \\
|R2 |===== / \ ======|S2 | +---+ // \\ +---+
+---+ +---+ |R2 |===== / \ ======|S2 |
+---+ +---+
]]></artwork> ]]></artwork>
</figure></t> ---+ +---+
</figure>
<t>Testbed attributes:</t> <dl spacing="normal">
<dt>Testbed attributes:</dt>
<t><list style="symbols"> <dd><t><br/></t>
<t>Test duration: 100s</t> <dl spacing="normal">
<dt>Test duration:</dt><dd>100 s</dd>
<t>Path characteristics: <list style="symbols"> <dt>Path characteristics:</dt>
<t>Reference bottleneck capacity: 1Mbps.</t> <dd><t><br/></t>
</list></t> <dl spacing="normal">
<dt>Reference bottleneck capacity:</dt><dd>1 Mbps</dd>
<t>Application-related: <list style="symbols"> </dl>
<t>Media Source: <list style="symbols"> </dd>
<t>Media type: Video<list style="symbols"> <dt>Application-related:</dt>
<t>Media direction: forward and backward</t> <dd><t><br/></t>
<dl spacing="normal">
<t>Number of media sources: two (2)</t> <dt>Media source:</dt>
<dd><t><br/></t>
<t>Media timeline: <list style="symbols"> <dl spacing="normal">
<t>Start time: 0s.</t> <dt>Media type:</dt>
<dd>
<t>End time: 99s.</t> <t>Video</t>
</list></t> <dl spacing="normal">
</list></t> <dt>Media direction:</dt><dd>forward and backward</dd>
<dt>Number of media sources:</dt><dd>two (2)</dd>
<t>Media type: Audio<list style="symbols"> <dt>Media timeline:</dt>
<t>Media direction: forward and backward</t> <dd><t><br/></t>
<dl spacing="normal">
<t>Number of media sources: two (2)</t> <dt>Start time:</dt><dd>0 s</dd>
<dt>End time:</dt><dd>99 s</dd>
<t>Media timeline: <list style="symbols"> </dl>
<t>Start time: 0s.</t> </dd>
</dl>
<t>End time: 99s.</t> </dd>
</list></t> <dt>Media type:</dt>
</list></t> <dd>
</list></t> <t>Audio</t>
<dl spacing="normal">
<t>Competing traffic: <list style="symbols"> <dt>Media direction:</dt><dd>forward and backward</dd>
<t>Number of sources : zero (0)</t> <dt>Number of media sources:</dt><dd>two (2)</dd>
</list></t> <dt>Media timeline: </dt>
</list></t> <dd><t><br/></t>
<dl spacing="normal">
<t>Test Specific Information: this test uses path capacity <dt>Start time:</dt><dd>0 s</dd>
variations to create congested feedback link. <xref <dt>End time:</dt><dd>99 s</dd>
target="CFL_US"> </xref> lists the variation patterns applied to </dl>
the forward path and <xref target="CFL_DS"/> lists the variation </dd>
</dl>
</dd>
</dl>
</dd>
<dt>Competing traffic: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Number of sources:</dt><dd>zero (0)</dd>
</dl>
</dd>
</dl>
</dd>
</dl>
</dd>
<dt>Test-specific information:</dt>
<dd>This test uses path capacity
variations to create a congested feedback link. <xref target="CFL_US
" format="default"/>
lists the variation patterns applied to
the forward path, and <xref target="CFL_DS" format="default"/> lists
the variation
patterns applied to the backward path. When using background patterns applied to the backward path. When using background
non-adaptive UDP traffic to induce time-varying bottleneck for non-adaptive UDP traffic to induce a time-varying bottleneck for
congestion controlled media flows, the physical path capacity is congestion-controlled media flows, the physical path capacity is
4Mbps for both directions and the UDP traffic source rate changes 4 Mbps for both directions, and the UDP traffic source rate changes
over time as (4-x)Mbps in each direction, where x is the over time as (4-x) Mbps in each direction, where x is the
bottleneck capacity specified in <xref target="CFL_DS"/>.</t> bottleneck capacity specified in <xref target="CFL_DS" format="defau
</list></t> lt"/>.</dd>
</dl>
<texttable anchor="CFL_US" <table anchor="CFL_US" align="center">
title="Path capacity variation pattern for forward direction" <name>Path Capacity Variation Pattern for the Forward Direction</name>
> <thead>
<ttcol>Variation pattern index</ttcol> <tr>
<th align="left">Variation pattern index</th>
<ttcol>Path direction</ttcol> <th align="left">Path direction</th>
<th align="left">Start time</th>
<ttcol>Start time</ttcol> <th align="left">Path capacity ratio</th>
</tr>
<ttcol>Path capacity ratio</ttcol> </thead>
<tbody>
<c>One</c> <tr>
<td align="left">One</td>
<c>Forward</c> <td align="left">Forward</td>
<td align="left">0 s</td>
<c>0s</c> <td align="left">2.0</td>
</tr>
<c>2.0</c> <tr>
<td align="left">Two</td>
<c>Two</c> <td align="left">Forward</td>
<td align="left">20 s</td>
<c>Forward</c> <td align="left">1.0</td>
</tr>
<c>20s</c> <tr>
<td align="left">Three</td>
<c>1.0</c> <td align="left">Forward</td>
<td align="left">40 s</td>
<c>Three</c> <td align="left">0.5</td>
</tr>
<c>Forward</c> <tr>
<td align="left">Four</td>
<c>40s</c> <td align="left">Forward</td>
<td align="left">60 s</td>
<c>0.5</c> <td align="left">2.0</td>
</tr>
<c>Four</c> </tbody>
</table>
<c>Forward</c>
<c>60s</c>
<c>2.0</c>
<!-- <postamble>Table 3: Path capacity variation pattern for
forward direction</postamble> -->
</texttable>
<t/> <t/>
<table anchor="CFL_DS" align="center">
<texttable anchor="CFL_DS" <name>Path Capacity Variation Pattern for the Backward Direction</name
title="Path capacity variation pattern for backward direction >
"> <thead>
<ttcol>Variation pattern index</ttcol> <tr>
<th align="left">Variation pattern index</th>
<ttcol>Path direction</ttcol> <th align="left">Path direction</th>
<th align="left">Start time</th>
<ttcol>Start time</ttcol> <th align="left">Path capacity ratio</th>
</tr>
<ttcol>Path capacity ratio</ttcol> </thead>
<tbody>
<c>One</c> <tr>
<td align="left">One</td>
<c>Backward</c> <td align="left">Backward</td>
<td align="left">0 s</td>
<c>0s</c> <td align="left">2.0</td>
</tr>
<c>2.0</c> <tr>
<td align="left">Two</td>
<c>Two</c> <td align="left">Backward</td>
<td align="left">35 s</td>
<c>Backward</c> <td align="left">0.8</td>
</tr>
<c>35s</c> <tr>
<td align="left">Three</td>
<c>0.8</c> <td align="left">Backward</td>
<td align="left">70 s</td>
<c>Three</c> <td align="left">2.0</td>
</tr>
<c>Backward</c> </tbody>
</table>
<c>70s</c>
<c>2.0</c>
<!-- <postamble>Table 4: Path capacity variation pattern for
backward direction</postamble> -->
</texttable>
</section> </section>
<section anchor="competing-rmcat-flow" numbered="true" toc="default">
<section anchor="competing-rmcat-flow" <name>Competing Media Flows with the Same Congestion Control Algorithm</
title="Competing Media Flows with same Congestion Control Algorit name>
hm">
<t>In this test case, more than one media flow share the bottleneck <t>In this test case, more than one media flow share the bottleneck
link and each of them uses the same congestion control algorithm. This link, and each of them uses the same congestion control algorithm. This
is a typical scenario where a real-time interactive application sends is a typical scenario where a real-time interactive application sends
more than one media flow to the same destination and these flows are more than one media flow to the same destination, and these flows are
multiplexed over the same port. In such a scenario it is likely that multiplexed over the same port. In such a scenario, it is likely that
the flows will be routed via the same path and need to share the the flows will be routed via the same path and need to share the
available bandwidth amongst themselves. For the sake of simplicity it available bandwidth amongst themselves. For the sake of simplicity, it
is assumed that there are no other competing traffic sources in the is assumed that there are no other competing traffic sources in the
bottleneck link and that there is sufficient capacity to accommodate bottleneck link and that there is sufficient capacity to accommodate
all the flows individually. While this appears to be a variant of the all the flows individually. While this appears to be a variant of the
test case defined in <xref target="VACM"/>, it focuses on the capacity test case defined in <xref target="VACM" format="default"/>, it focuses
sharing aspect of the candidate algorithm. The previous test case, on on the capacity-sharing aspect of the candidate algorithm. The previous
test case, on
the other hand, measures adaptability, stability, and responsiveness the other hand, measures adaptability, stability, and responsiveness
of the candidate algorithm.</t> of the candidate algorithm.</t>
<dl spacing="normal">
<t>Expected behavior: It is expected that the competing flows will <dt>Expected behavior:</dt><dd>It is expected that the competing flows w
ill
converge to an optimum bit rate to accommodate all the flows with converge to an optimum bit rate to accommodate all the flows with
minimum possible latency and loss. Specifically, the test introduces minimum possible latency and loss. Specifically, the test introduces
three media flows at different time instances, when the second flow three media flows at different time instances. When the second flow
appears there should still be room to accommodate another flow on the appears, there should still be room to accommodate another flow on the
bottleneck link. Lastly, when the third flow appears the bottleneck bottleneck link. Lastly, when the third flow appears, the bottleneck
link should be saturated.</t> link should be saturated.</dd>
<dt>Evaluation metrics:</dt><dd>As described in <xref target="EM" format
<t>Evaluation metrics : as described in <xref target="EM"/>.</t> ="default"/>.</dd>
<dt>Testbed topology: </dt><dd>Three media sources S1, S2, and S3 are co
<t>Testbed topology: Three media sources S1, S2, S3 are connected to nnected to
R1, R2, R3 respectively. The media traffic is transported over the R1, R2, and R3, respectively. The media traffic is transported over the
forward path and corresponding feedback/control traffic is transported forward path, and the corresponding feedback/control traffic is transpor
over the backward path. <figure anchor="fig-eval-topo-4-3" ted
title="Testbed Topology for Multiple congestion controlled media Flo over the backward path. </dd>
ws"> </dl>
<artwork><![CDATA[ <figure anchor="fig-eval-topo-4-3">
<name>Testbed Topology for Multiple Congestion-Controlled Media Flows<
/name>
<artwork name="" type="" align="left" alt=""><![CDATA[
+---+ +---+ +---+ +---+
|S1 |===== \ Forward --> / =======|R1 | |S1 |===== \ Forward --> / =======|R1 |
+---+ \\ // +---+ +---+ \\ // +---+
\\ // \\ //
+---+ +-----+ +-----+ +---+ +---+ +-----+ +-----+ +---+
|S2 |=======| A |------------------------------>| B |=======|R2 | |S2 |=======| A |------------------------------>| B |=======|R2 |
+---+ | |<------------------------------| | +---+ +---+ | |<------------------------------| | +---+
+-----+ +-----+ +-----+ +-----+
// <-- Backward \\ // <-- Backward \\
+---+ // \\ +---+ +---+ // \\ +---+
|S3 |===== / \ ======|R3 | |S3 |===== / \ ======|R3 |
+---+ +---+ +---+ +---+
]]></artwork> ]]></artwork>
</figure></t> </figure>
<t>Testbed attributes:</t>
<t><list style="symbols">
<t>Test duration: 120s</t>
<t>Path characteristics: <list style="symbols">
<t>Reference bottleneck capacity: 3.5Mbps</t>
<t>Path capacity ratio: 1.0</t>
<!-- <t>One-Way propagation delay: [10ms, 50ms]</t> -->
</list></t>
<t>Application-related: <list style="symbols">
<t>Media Source: <list style="symbols">
<t>Media type: Video<list style="symbols">
<t>Media direction: forward.</t>
<t>Number of media sources: three (3)</t>
<t>Media timeline: new media flows are added
sequentially, at short time intervals. See test
specific setup below.</t>
</list></t>
<t>Media type: Audio<list style="symbols">
<t>Media direction: forward.</t>
<t>Number of media sources: three (3)</t>
<t>Media timeline: new media flows are added
sequentially, at short time intervals. See test
specific setup below.</t>
</list></t>
</list></t>
<t>Competing traffic: <list style="symbols">
<t>Number of sources : zero (0)</t>
</list></t>
</list></t>
<t>Test Specific Information: <xref target="MTL_CF"/> defines the
media timeline for both media type.</t>
</list></t>
<texttable anchor="MTL_CF"
title="Media Timeline for Video and Audio media sources">
<ttcol>Flow ID</ttcol>
<ttcol>Media type</ttcol>
<ttcol>Start time</ttcol>
<ttcol>End time</ttcol>
<c>1</c>
<c>Video</c>
<c>0s</c>
<c>119s</c>
<c>2</c>
<c>Video</c>
<c>20s</c>
<c>119s</c>
<c>3</c>
<c>Video</c>
<c>40s</c>
<c>119s</c>
<c>4</c>
<c>Audio</c>
<c>0s</c>
<c>119s</c>
<c>5</c>
<c>Audio</c>
<c>20s</c>
<c>119s</c>
<c>6</c>
<c>Audio</c>
<c>40s</c>
<c>119s</c> <dl spacing="normal">
</texttable> <dt>Testbed attributes:</dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Test duration:</dt><dd>120 s</dd>
<dt>Path characteristics:</dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Reference bottleneck capacity:</dt><dd>3.5 Mbps</dd>
<dt>Path capacity ratio:</dt><dd>1.0</dd>
</dl>
</dd>
<dt>Application-related: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Media Source: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Media type:</dt>
<dd>
<t>Video</t>
<dl spacing="normal">
<dt>Media direction:</dt><dd>forward</dd>
<dt>Number of media sources:</dt><dd>three (3)</dd>
<dt>Media timeline:</dt><dd>new media flows are added
sequentially, at short time intervals. See the
test-specific setup below.</dd>
</dl>
</dd>
<dt>Media type:</dt>
<dd>
<t>Audio</t>
<dl spacing="normal">
<dt>Media direction:</dt><dd>forward</dd>
<dt>Number of media sources:</dt><dd>three (3)</dd>
<dt>Media timeline:</dt><dd>new media flows are added
sequentially, at short time intervals. See the test-spec
ific setup below.</dd>
</dl>
</dd>
</dl>
</dd>
<dt>Competing traffic: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Number of sources:</dt><dd>zero (0)</dd>
</dl>
</dd>
</dl>
</dd>
</dl>
</dd>
<dt>Test-specific information:</dt><dd> <xref target="MTL_CF" format="default"
/> defines the
media timeline for both media types.</dd>
</dl>
<table anchor="MTL_CF" align="center">
<name>Media Timelines for Video and Audio Media Sources</name>
<thead>
<tr>
<th align="left">Flow ID</th>
<th align="left">Media type</th>
<th align="left">Start time</th>
<th align="left">End time</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left">1</td>
<td align="left">Video</td>
<td align="left">0 s</td>
<td align="left">119 s</td>
</tr>
<tr>
<td align="left">2</td>
<td align="left">Video</td>
<td align="left">20 s</td>
<td align="left">119 s</td>
</tr>
<tr>
<td align="left">3</td>
<td align="left">Video</td>
<td align="left">40 s</td>
<td align="left">119 s</td>
</tr>
<tr>
<td align="left">4</td>
<td align="left">Audio</td>
<td align="left">0 s</td>
<td align="left">119 s</td>
</tr>
<tr>
<td align="left">5</td>
<td align="left">Audio</td>
<td align="left">20 s</td>
<td align="left">119 s</td>
</tr>
<tr>
<td align="left">6</td>
<td align="left">Audio</td>
<td align="left">40 s</td>
<td align="left">119 s</td>
</tr>
</tbody>
</table>
</section> </section>
<section numbered="true" toc="default">
<section title="Round Trip Time Fairness"> <name>Round Trip Time Fairness</name>
<t>In this test case, multiple media flows share the bottleneck link, <t>In this test case, multiple media flows share the bottleneck link,
but the one-way propagation delay for each flow is different. For the but the one-way propagation delay for each flow is different. For the
sake of simplicity it is assumed that there are no other competing sake of simplicity, it is assumed that there are no other competing
traffic sources in the bottleneck link and that there is sufficient traffic sources in the bottleneck link and that there is sufficient
capacity to accommodate all the flows. While this appears to be a capacity to accommodate all the flows. While this appears to be a
variant of test case 5.2, it focuses on the capacity sharing aspect of variant of test case 5.2 (<xref target="VACM" format="default"/>),
it focuses on the capacity-sharing aspect of
the candidate algorithm under different RTTs.</t> the candidate algorithm under different RTTs.</t>
<dl spacing="normal">
<t>Expected behavior: It is expected that the competing flows will <dt>Expected behavior:</dt><dd>It is expected that the competing flows will
converge to bit rates to accommodate all the flows with minimum converge to bit rates to accommodate all the flows with minimum
possible latency and loss. The effectiveness of the algorithm depends possible latency and loss. The effectiveness of the algorithm depends
on how fast and fairly the competing flows converge to their steady on how fast and fairly the competing flows converge to their steady
states irrespective of the RTT observed.</t> states irrespective of the RTT observed.</dd>
<dt>Evaluation metrics:</dt><dd>As described in <xref target="EM" format="defa
<t>Evaluation metrics : as described in <xref target="EM"/>.</t> ult"/>.</dd>
<dt>Testbed topology: </dt><dd>Five (5) media sources S1..S5 are connected
<t>Testbed Topology: Five (5) media sources S1,S2,..,S5 are connected to their corresponding media sinks R1..R5. The media traffic is
to their corresponding media sinks R1,R2,..,R5. The media traffic is transported over the forward path, and the corresponding feedback/contro
transported over the forward path and corresponding feedback/control l
traffic is transported over the backward path. The topology is the traffic is transported over the backward path. The topology is the
same as in <xref target="competing-rmcat-flow"/>.</t> same as in <xref target="competing-rmcat-flow" format="default"/>.</dd>
<dt>Testbed attributes: </dt>
<t>Testbed attributes:</t> <dd><t><br/></t>
<dl spacing="normal">
<t><list style="symbols"> <dt>Test duration:</dt><dd>300 s</dd>
<t>Test duration: 300s</t> <dt>Path characteristics: </dt>
<dd><t><br/></t>
<t>Path characteristics: <list style="symbols"> <dl spacing="normal">
<t>Reference bottleneck capacity: 4Mbps</t> <dt>Reference bottleneck capacity:</dt><dd>4 Mbps</dd>
<dt>Path capacity ratio:</dt><dd>1.0</dd>
<t>Path capacity ratio: 1.0</t> <dt>One-way propagation delay for each flow:</dt><dd>10 ms for S1-
R1,
<t>One-Way propagation delay for each flow: 10ms for S1-R1, 25 ms for S2-R2, 50 ms for S3-R3, 100 ms for S4-R4, and 150 ms
25ms for S2-R2, 50ms for S3-R3, 100ms for S4-R4, and 150ms S5-R5.</dd>
S5-R5.</t> </dl>
</list></t> </dd>
<dt>Application-related: </dt>
<t>Application-related: <list style="symbols"> <dd><t><br/></t>
<t>Media Source: <list style="symbols"> <dl spacing="normal">
<t>Media type: Video<list style="symbols"> <dt>Media source: </dt>
<t>Media direction: forward</t> <dd><t><br/></t>
<dl spacing="normal">
<t>Number of media sources: five (5)</t> <dt>Media type:</dt>
<dd>
<t>Media timeline: new media flows are added <t>Video</t>
sequentially, at short time intervals. See test <dl spacing="normal">
specific setup below.</t> <dt>Media direction:</dt><dd>forward</dd>
</list></t> <dt>Number of media sources:</dt><dd>five (5)</dd>
<dt>Media timeline:</dt><dd>new media flows are added
<t>Media type: Audio<list style="symbols"> sequentially, at short time intervals. See the
<t>Media direction: forward.</t> test-specific setup below.</dd>
</dl>
<t>Number of media sources: five (5)</t> </dd>
<dt>Media type:</dt>
<t>Media timeline: new media flows are added <dd>
sequentially, at short time intervals. See test <t>Audio</t>
specific setup below.</t> <dl spacing="normal">
</list></t> <dt>Media direction:</dt><dd>forward</dd>
</list></t> <dt>Number of media sources:</dt><dd> five (5)</dd>
<dt>Media timeline:</dt><dd>New media flows are added
<t>Competing traffic: <list style="symbols"> sequentially, at short time intervals. See the
<t>Number of sources : zero (0)</t> test-specific setup below.</dd>
</list></t> </dl>
</list></t> </dd>
</dl>
<t>Test Specific Information: <xref target="MTL_RTT"/> defines the </dd>
media timeline for both media type.</t> <dt>Competing traffic: </dt>
</list></t> <dd><t><br/></t>
<dl spacing="normal">
<texttable anchor="MTL_RTT" <dt>Number of sources:</dt><dd>zero (0)</dd>
title="Media Timeline for Video and Audio media sources"> </dl>
<ttcol>Flow IF</ttcol> </dd>
</dl>
<ttcol>Media type</ttcol> </dd>
</dl>
<ttcol>Start time</ttcol> </dd>
<dt>Test-specific information: </dt><dd><xref target="MTL_RTT" format="default
<ttcol>End time</ttcol> "/> defines the
media timeline for both media types.</dd>
<c>1</c> </dl>
<table anchor="MTL_RTT" align="center">
<c>Video</c> <name>Media Timeline for Video and Audio Media Sources</name>
<thead>
<c>0s</c> <tr>
<th align="left">Flow ID</th>
<c>299s</c> <th align="left">Media type</th>
<th align="left">Start time</th>
<c>2</c> <th align="left">End time</th>
</tr>
<c>Video</c> </thead>
<tbody>
<c>10s</c> <tr>
<td align="left">1</td>
<c>299s</c> <td align="left">Video</td>
<td align="left">0 s</td>
<c>3</c> <td align="left">299 s</td>
</tr>
<c>Video</c> <tr>
<td align="left">2</td>
<c>20s</c> <td align="left">Video</td>
<td align="left">10 s</td>
<c>299s</c> <td align="left">299 s</td>
</tr>
<c>4</c> <tr>
<td align="left">3</td>
<c>Video</c> <td align="left">Video</td>
<td align="left">20 s</td>
<c>30s</c> <td align="left">299 s</td>
</tr>
<c>299s</c> <tr>
<td align="left">4</td>
<c>5</c> <td align="left">Video</td>
<td align="left">30 s</td>
<c>Video</c> <td align="left">299 s</td>
</tr>
<c>40s</c> <tr>
<td align="left">5</td>
<c>299s</c> <td align="left">Video</td>
<td align="left">40 s</td>
<c>6</c> <td align="left">299 s</td>
</tr>
<c>Audio</c> <tr>
<td align="left">6</td>
<c>0</c> <td align="left">Audio</td>
<td align="left">0 s</td>
<c>299s</c> <td align="left">299 s</td>
</tr>
<c>7</c> <tr>
<td align="left">7</td>
<c>Audio</c> <td align="left">Audio</td>
<td align="left">10 s</td>
<c>10s</c> <td align="left">299 s</td>
</tr>
<c>299s</c> <tr>
<td align="left">8</td>
<c>8</c> <td align="left">Audio</td>
<td align="left">20 s</td>
<c>Audio</c> <td align="left">299 s</td>
</tr>
<c>20s</c> <tr>
<td align="left">9</td>
<c>299s</c> <td align="left">Audio</td>
<td align="left">30 s</td>
<c>9</c> <td align="left">299 s</td>
</tr>
<c>Audio</c> <tr>
<td align="left">10</td>
<c>30s</c> <td align="left">Audio</td>
<td align="left">40 s</td>
<c>299s</c> <td align="left">299 s</td>
</tr>
<c>10</c> </tbody>
</table>
<c>Audio</c>
<c>40s</c>
<c>299s</c>
</texttable>
</section> </section>
<section numbered="true" toc="default">
<section title="Media Flow Competing with a Long TCP Flow"> <name>Media Flow Competing with a Long TCP Flow</name>
<t>In this test case, one or more media flows share the bottleneck <t>In this test case, one or more media flows share the bottleneck
link with at least one long lived TCP flow. Long lived TCP flows link with at least one long-lived TCP flow. Long-lived TCP flows
download data throughout the session and are expected to have infinite download data throughout the session and are expected to have infinite
amount of data to send and receive. This is a scenario where a amount of data to send and receive. This is a scenario where a
multimedia application co-exists with a large file download. The test multimedia application coexists with a large file download. The test
case measures the adaptivity of the candidate algorithm to competing case measures the adaptivity of the candidate algorithm to competing
traffic. It addresses the requirement 3 in <xref traffic. It addresses requirement 3 in <xref target="RFC8836" section="2
target="I-D.ietf-rmcat-cc-requirements"/>.</t> " sectionFormat="of" format="default"/>.</t>
<dl spacing="normal">
<t>Expected behavior: depending on the convergence observed in test <dt>Expected behavior:</dt><dd>Depending on the convergence observed in
case 5.1 and 5.2, the candidate algorithm may be able to avoid test
cases 5.1 and 5.2, the candidate algorithm may be able to avoid
congestion collapse. In the worst case, the media stream will fall to congestion collapse. In the worst case, the media stream will fall to
the minimum media bit rate.</t> the minimum media bit rate.</dd>
<dt>Evaluation metrics:</dt>
<t>Evaluation metrics : following metrics in addition to as described <dd><t>Includes the following metrics in addition to those described
in <xref target="EM"/>. <list style="numbers"> in <xref target="EM" format="default"/>: </t>
<t>Flow level: <list style="letters"> <ol spacing="normal" type="1">
<t>TCP throughput.</t> <li> <t>Flow level: </t>
<ol spacing="normal" type="a">
<t>Loss for the TCP flow</t> <li>TCP throughput</li>
</list></t> <li>Loss for the TCP flow</li>
</list></t> </ol>
</li>
<t>Testbed topology: One (1) media source S1 is connected to the </ol>
corresponding media sink, R1. In addition, there is a long-live TCP </dd>
<dt>Testbed topology:</dt><dd>One (1) media source S1 is connected to th
e
corresponding media sink, R1. In addition, there is a long-lived TCP
flow sharing the same bottleneck link. The media traffic is flow sharing the same bottleneck link. The media traffic is
transported over the forward path and corresponding feedback/control transported over the forward path, and the corresponding feedback/contro l
traffic is transported over the backward path. The TCP traffic goes traffic is transported over the backward path. The TCP traffic goes
over the forward path from, S_tcp with acknowledgment packets go over over the forward path from S_tcp with acknowledgment packets going over
the backward path from, R_tcp.</t> the backward path from R_tcp.</dd>
</dl>
<t><figure anchor="fig-eval-topo-4-4" <figure anchor="fig-eval-topo-4-4">
title="Testbed Topology for TCP vs congestion controlled media Flows <name>Testbed Topology for TCP vs Congestion-Controlled Media Flows</n
"> ame>
<artwork><![CDATA[ <artwork name="" type="" align="left" alt=""><![CDATA[
+--+ +--+ +--+ +--+
|S1|===== \ Forward --> / =====|R1| |S1|===== \ Forward --> / =====|R1|
+--+ \\ // +--+ +--+ \\ // +--+
\\ // \\ //
+-----+ +-----+ +-----+ +-----+
| A |---------------------------->| B | | A |---------------------------->| B |
| |<----------------------------| | | |<----------------------------| |
+-----+ +-----+ +-----+ +-----+
// <-- Backward \\ // <-- Backward \\
+-----+ // \\ +-----+ +-----+ // \\ +-----+
|S_tcp|=== / \ ===|R_tcp| |S_tcp|=== / \ ===|R_tcp|
+-----+ +-----+ +-----+ +-----+
]]></artwork> ]]></artwork>
</figure></t> </figure>
<dl spacing="normal">
<t>Testbed attributes:</t> <dt>Testbed attributes:</dt>
<dd><t><br/></t>
<t><list style="symbols"> <dl spacing="normal">
<t>Test duration: 120s</t> <dt>Test duration:</dt><dd>120 s</dd>
<dt>Path characteristics:</dt>
<t>Path characteristics: <list style="symbols"> <dd><t><br/></t>
<t>Reference bottleneck capacity: 2Mbps</t> <dl spacing="normal">
<dt>Reference bottleneck capacity:</dt><dd>2 Mbps</dd>
<t>Path capacity ratio: 1.0</t> <dt>Path capacity ratio:</dt><dd>1.0</dd>
<dt>Bottleneck queue size:</dt><dd>[300 ms, 1000 ms]</dd>
<!-- <t>One-Way propagation delay: [10ms, 150ms]</t> --> </dl>
</dd>
<t>Bottleneck queue size: [300ms, 1000ms]</t> <dt>Application-related:</dt>
</list></t> <dd><t><br/></t>
<dl spacing="normal">
<t>Application-related: <list style="symbols"> <dt>Media source:</dt>
<t>Media Source: <list style="symbols"> <dd><t><br/></t>
<t>Media type: Video<list style="symbols"> <dl spacing="normal">
<t>Media direction: forward</t> <dt>Media type:</dt>
<dd>
<t>Number of media sources: one (1)</t> <t>Video</t>
<dl spacing="normal">
<t>Media timeline: <list style="symbols"> <dt>Media direction:</dt><dd>forward</dd>
<t>Start time: 5s.</t> <dt>Number of media sources:</dt><dd>one (1)</dd>
<dt>Media timeline:</dt>
<t>End time: 119s.</t> <dd><t><br/></t>
</list></t> <dl spacing="normal">
</list></t> <dt>Start time:</dt><dd>5 s</dd>
<dt>End time:</dt><dd>119 s</dd>
<t>Media type: Audio<list style="symbols"> </dl>
<t>Media direction: forward</t> </dd>
</dl>
<t>Number of media sources: one (1)</t> </dd>
<dt>Media type:</dt>
<t>Media timeline: <list style="symbols"> <dd>
<t>Start time: 5s.</t> <t>Audio</t>
<dl spacing="normal">
<t>End time: 119s.</t> <dt>Media direction:</dt><dd>forward</dd>
</list></t> <dt>Number of media sources:</dt><dd>one (1)</dd>
</list></t> <dt>Media timeline:</dt>
</list></t> <dd><t><br/></t>
<dl spacing="normal">
<t>Additionally, implementers are encouraged to run the <dt>Start time:</dt><dd>5 s</dd>
<dt>End time:</dt><dd>119 s</dd>
</dl>
</dd>
</dl>
</dd>
</dl>
<t>Additionally, implementers are encouraged to run the
experiment with multiple media sources.</t> experiment with multiple media sources.</t>
</dd>
<t>Competing traffic: <list style="symbols"> <dt>Competing traffic:</dt>
<t>Number and Types of sources : one (1) and long-lived <dd><t><br/></t>
TCP</t> <dl spacing="normal">
<dt>Number and types of sources:</dt><dd>one (1) and long-liv
<t>Traffic direction : forward</t> ed TCP</dd>
<dt>Traffic direction:</dt><dd>forward</dd>
<t>Congestion control: default TCP congestion control<xref <dt>Congestion control:</dt><dd>default TCP congestion contro
target="RFC5681"/>. Implementers are also encouraged to l
run the experiment with alternative TCP congestion control <xref target="RFC5681" format="default"/>. Implementers a
algorithm.<!----></t> re also encouraged to
run the experiment with alternative TCP congestion contro
<t>Traffic timeline: <list style="symbols"> l algorithms.</dd>
<t>Start time: 0s.</t> <dt>Traffic timeline: </dt>
<dd><t><br/></t>
<!----> <dl spacing="normal">
<dt>Start time:</dt><dd>0 s</dd>
<t>End time: 119s.</t> <dt>End time:</dt><dd>119 s</dd>
</list></t> </dl>
</list></t> </dd>
</list></t> </dl>
</dd>
<t>Test Specific Information: none</t> </dl>
</list></t> </dd>
</dl>
</dd>
<dt>Test-specific information:</dt><dd>none</dd>
</dl>
</section> </section>
<section numbered="true" toc="default">
<section title="Media Flow Competing with Short TCP Flows"> <name>Media Flow Competing with Short TCP Flows</name>
<t>In this test case, one or more congestion controlled media flow <t>In this test case, one or more congestion-controlled media flows
shares the bottleneck link with multiple short-lived TCP flows. share the bottleneck link with multiple short-lived TCP flows.
Short-lived TCP flows resemble the on/off pattern observed in the web Short-lived TCP flows resemble the on/off pattern observed in web
traffic, wherein clients (for example, browsers) connect to a server traffic, wherein clients (for example, browsers) connect to a server
and download a resource (typically a web page, few images, text files, and download a resource (typically a web page, few images, text files,
etc.) using several TCP connections. This scenario shows the etc.) using several TCP connections. This scenario shows the
performance of a multimedia application when several browser windows performance of a multimedia application when several browser windows
are active. The test case measures the adaptivity of the candidate are active. The test case measures the adaptivity of the candidate
algorithm to competing web traffic, it addresses the requirements 1.E algorithm to competing web traffic, and it addresses requirement 1.E
in <xref target="I-D.ietf-rmcat-cc-requirements"/>.</t> in <xref target="RFC8836" section="2" sectionFormat="of" format="default
"/>.</t>
<t>Depending on the number of short TCP flows, the cross-traffic <t>Depending on the number of short TCP flows, the cross traffic
either appears as a short burst flow or resembles a long TCP flow. The either appears as a short burst flow or resembles a long-lived TCP flow.
intention of this test is to observe the impact of short-term burst on The
intention of this test is to observe the impact of a short-term burst on
the behavior of the candidate algorithm.</t> the behavior of the candidate algorithm.</t>
<dl spacing="normal">
<t>Expected behavior: The candidate algorithm is expected to avoid <dt>Expected behavior:</dt><dd> The candidate algorithm is expected to avoid
flow starvation during the presence of short and bursty competing TCP flow starvation during the presence of short and bursty competing TCP
flows, streaming at least at the minimum media bit rate. After flows, streaming at least at the minimum media bit rate. After
competing TCP flows terminate, the media streams are expected to be competing TCP flows terminate, the media streams are expected to be
robust enough to eventually recover to previous steady state behavior, robust enough to eventually recover to previous steady state behavior,
and at the very least, avoid persistent starvation.</t> and at the very least, avoid persistent starvation.</dd>
<dt>Evaluation metrics:</dt>
<t>Evaluation metrics : following metrics in addition to as described <dd>
in <xref target="EM"/>.<list style="numbers"> <t>Includes the following metrics in addition to those described
<t>Flow level: <list style="letters"> in <xref target="EM" format="default"/>:</t>
<t>Variation in the sending rate of the TCP flow.</t> <ol spacing="normal" type="1">
<li><t>Flow level: </t>
<t>TCP throughput.</t> <ol spacing="normal" type="A">
</list></t> <li>Variation in the sending rate of the TCP flow</li>
</list></t> <li>TCP throughput</li>
</ol>
<t>Testbed topology: The topology described here is same as the one </li>
described in <xref target="fig-eval-topo-4-4"/>.</t> </ol>
</dd>
<t>Testbed attributes:</t> <dt>Testbed topology:</dt><dd>The topology described here is the same as the o
ne
<t><list style="symbols"> described in <xref target="fig-eval-topo-4-4" format="default"/>.</dd>
<t>Test duration: 300s</t> <dt>Testbed attributes:</dt>
<dd><t><br/></t>
<t>Path characteristics: <list style="symbols"> <dl spacing="normal">
<t>Reference bottleneck capacity: 2.0Mbps</t> <dt>Test duration:</dt><dd>300 s</dd>
<dt>Path characteristics:</dt>
<t>Path capacity ratio: 1.0</t> <dd><t><br/></t>
<dl spacing="normal">
<!-- <t>One-Way propagation delay: [10ms, 150ms]</t> --> <dt>Reference bottleneck capacity:</dt><dd>2.0 Mbps</dd>
</list></t> <dt>Path capacity ratio:</dt><dd>1.0</dd>
</dl>
<t>Application-related: <list style="symbols"> </dd>
<t>Media source: <list style="symbols"> <dt>Application-related: </dt>
<t>Media type: Video<list style="symbols"> <dd><t><br/></t>
<t>Media direction: forward</t> <dl spacing="normal">
<dt>Media source: </dt>
<t>Number of media sources: two (2)</t> <dd><t><br/></t>
<dl spacing="normal">
<t>Media timeline: <list style="symbols"> <dt>Media type:</dt>
<t>Start time: 5s.</t> <dd>
<t>Video</t>
<t>End time: 299s.</t> <dl spacing="normal">
</list></t> <dt>Media direction:</dt><dd>forward</dd>
</list></t> <dt>Number of media sources:</dt><dd>two (2)</dd>
<dt>Media timeline: </dt>
<t>Media type: Audio<list style="symbols"> <dd><t><br/></t>
<t>Media direction: forward</t> <dl spacing="normal">
<dt>Start time:</dt><dd>5 s</dd>
<t>Number of media sources: two (2)</t> <dt>End time:</dt><dd>299 s</dd>
</dl>
<t>Media timeline: <list style="symbols"> </dd>
<t>Start time: 5s.</t> </dl>
</dd>
<t>End time: 299s.</t> <dt>Media type:</dt>
</list></t> <dd>
</list></t> <t>Audio</t>
</list></t> <dl spacing="normal">
<dt>Media direction:</dt><dd>forward</dd>
<t>Competing traffic: <list style="symbols"> <dt>Number of media sources:</dt><dd> two (2)</dd>
<t>Number and Types of sources : ten (10), short-lived TCP <dt>Media timeline: </dt>
flows.</t> <dd><t><br/></t>
<dl spacing="normal">
<t>Traffic direction : forward</t> <dt>Start time:</dt><dd> 5 s</dd>
<dt>End time: </dt><dd>299 s</dd>
<t>Congestion algorithm: default TCP Congestion control </dl>
<xref target="RFC5681"/>. Implementers are also encouraged </dd>
to run the experiment with alternative TCP congestion </dl>
control algorithm.</t> </dd>
</dl>
<t>Traffic timeline: each short TCP flow is modeled as a </dd>
<dt>Competing traffic: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Number and types of sources:</dt><dd> ten (10), short-live
d TCP flows.</dd>
<dt>Traffic direction: </dt><dd>forward</dd>
<dt>Congestion algorithm:</dt><dd> default TCP congestion cont
rol
<xref target="RFC5681" format="default"/>. Implementers are
also encouraged
to run the experiment with an alternative TCP congestion
control algorithm.</dd>
<dt>Traffic timeline: </dt><dd>Each short TCP flow is modeled
as a
sequence of file downloads interleaved with idle periods. sequence of file downloads interleaved with idle periods.
Not all short TCP flows start at the same time, 2 of them Not all short TCP flows start at the same time, two of them
start in the ON state while rest of the 8 flows start in start in the ON state, while rest of the eight flows start i
an OFF state. For description of short TCP flow model see n
test specific information below.</t> an OFF state. For a description of the short TCP flow model,
</list></t> see
</list></t> test-specific information below.</dd>
</dl>
</dd>
</dl>
</dd>
</dl>
</dd>
<dt>Test-specific information: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Short TCP traffic model:</dt><dd>The short TCP model to be used in
this test is described in <xref target="RFC8868" format="default"/>.
</dd>
</dl>
</dd>
</dl>
<t>Test Specific Information: <list style="symbols">
<t>Short-TCP traffic model: The short TCP model to be used in
this test is described in <xref
target="I-D.ietf-rmcat-eval-criteria"/>.</t>
</list></t>
</list></t>
</section> </section>
<section numbered="true" toc="default">
<section title="Media Pause and Resume"> <name>Media Pause and Resume</name>
<t>In this test case, more than one real-time interactive media flows <t>In this test case, more than one real-time interactive media flow
share the link bandwidth and all flows reach to a steady state by share the link bandwidth, and all flows reach to a steady state by
utilizing the link capacity in an optimum way. At this stage one of utilizing the link capacity in an optimum way. At this stage, one of
the media flows is paused for a moment. This event will result in more the media flows is paused for a moment. This event will result in more
available bandwidth for the rest of the flows as they are on a shared available bandwidth for the rest of the flows as they are on a shared
link. When the paused media flow resumes it would no longer have the link. When the paused media flow resumes, it no longer has the
same bandwidth share on the link. It has to make its way through the same bandwidth share on the link. It has to make its way through the
other existing flows in the link to achieve a fair share of the link other existing flows in the link to achieve a fair share of the link
capacity. This test case is important specially for real-time capacity. This test case is important specially for real-time
interactive media which consists of more than one media flows and can interactive media, which consists of more than one media flows and can
pause/resume media flows at any point of time during the session. This pause/resume media flows at any point of time during the session. This
test case directly addresses the requirement number 5 in <xref test case directly addresses requirement 5 in
target="I-D.ietf-rmcat-cc-requirements"/>. One can think it as a <xref target="RFC8836" section="2" sectionFormat="of" format="default"/>
variation of test case defined in <xref . One can think of it as a
target="competing-rmcat-flow"/>. However, it is different as the variation of the test case defined in
candidate algorithms can use different strategies to increase its <xref target="competing-rmcat-flow" format="default"/>.
efficiency, for example in terms of fairness, convergence time, reduce However, it is different as the
oscillation etc, by capitalizing the fact that they have previous candidate algorithms can use different strategies to increase
efficiency, for example, in terms of fairness, convergence time,
oscillation reduction, etc., by capitalizing on the fact that they have
previous
information of the link.</t> information of the link.</t>
<dl spacing="normal">
<t>Expected behavior: During the period where the third stream is <dt>Expected behavior:</dt><dd>During the period where the third stream is
paused, the two remaining flows are expected to increase their rates paused, the two remaining flows are expected to increase their rates
and reach the maximum media bit rate. When the third stream resumes, and reach the maximum media bit rate. When the third stream resumes,
all three flows are expected to converge to the same original fair all three flows are expected to converge to the same original fair
share of rates prior to the media pause/resume event.</t> share of rates prior to the media pause/resume event.</dd>
<dt>Evaluation metrics:</dt>
<t>Evaluation metrics : following metrics in addition to as described <dd>
in <xref target="EM"/>.<list style="numbers"> <t>Includes the following metrics in addition to those described
<t>Flow level: <list style="letters"> in <xref target="EM" format="default"/>:</t>
<t>Variation in sending bit rate and throughput. Mainly <ol spacing="normal" type="1">
observing the frequency and magnitude of oscillations.</t> <li><t>Flow level: </t>
</list></t> <ol spacing="normal" type="A">
</list></t> <li>Variation in sending bit rate and throughput. Mainly
observing the frequency and magnitude of oscillations.</li>
<t>Testbed Topology: Same as test case defined in <xref </ol>
target="competing-rmcat-flow"/></t> </li>
</ol>
<t>Testbed attributes: The general description of the testbed </dd>
parameters are same as <xref target="competing-rmcat-flow"/> with <dt>Testbed topology:</dt><dd>Same as the test case defined in <xref target="c
changes in the test specific setup as below-</t> ompeting-rmcat-flow" format="default"/>.</dd>
<dt>Testbed attributes:</dt>
<t><list style="symbols"> <dd>
<t>Other test specific setup: <list style="symbols"> <t>The general description of the testbed parameters are
<t>Media flow timeline: <list style="symbols"> the same as <xref target="competing-rmcat-flow" format="default"/>
<t>Flow ID: one (1)</t> with changes in the test-specific setup as below:</t>
<t>Other test-specific setup: </t>
<t>Start time: 0s</t> <dl spacing="normal">
<dt>Media flow timeline: </dt>
<t>Flow duration: 119s</t> <dd><t><br/></t>
<dl spacing="normal">
<t>Pause time: not required</t> <dt>Flow ID: </dt><dd>one (1)</dd>
<dt>Start time: </dt><dd>0 s</dd>
<t>Resume time: not required</t> <dt>Flow duration: </dt><dd>119 s</dd>
</list></t> <dt>Pause time: </dt><dd>not required</dd>
<dt>Resume time: </dt><dd>not required</dd>
<t>Media flow timeline: <list style="symbols"> </dl>
<t>Flow ID: two (2)</t> </dd>
<dt>Media flow timeline: </dt>
<t>Start time: 0s</t> <dd><t><br/></t>
<dl spacing="normal">
<t>Flow duration: 119s</t> <dt>Flow ID: </dt><dd>two (2)</dd>
<dt>Start time: </dt><dd>0 s</dd>
<t>Pause time: at 40s</t> <dt>Flow duration: </dt><dd>119 s</dd>
<dt>Pause time: </dt><dd>at 40 s</dd>
<t>Resume time: at 60s</t> <dt>Resume time: </dt><dd>at 60 s</dd>
</list></t> </dl>
</dd>
<t>Media flow timeline: <list style="symbols"> <dt>Media flow timeline: </dt>
<t>Flow ID: three (3)</t> <dd><t><br/></t>
<dl spacing="normal">
<t>Start time: 0s</t> <dt>Flow ID:</dt><dd>three (3)</dd>
<dt>Start time:</dt><dd>0 s</dd>
<t>Flow duration:119s</t> <dt>Flow duration:</dt><dd>119 s</dd>
<dt>Pause time:</dt><dd>not required</dd>
<t>Pause time: not required</t> <dt>Resume time:</dt><dd>not required</dd>
</dl>
</dd>
</dl>
</dd>
</dl>
<t>Resume time: not required</t>
</list></t>
</list></t>
</list></t>
</section> </section>
</section> </section>
<section numbered="true" toc="default">
<section title="Other potential test cases"> <name>Other Potential Test Cases</name>
<t>It has been noticed that there are other interesting test cases <t>It has been noticed that there are other interesting test cases
besides the basic test cases listed above. In many aspects, these besides the basic test cases listed above. In many aspects, these
additional test cases can help further evaluation of the candidate additional test cases can help further evaluation of the candidate
algorithm. They are listed as below.</t> algorithm. They are listed below.</t>
<section numbered="true" toc="default">
<section title="Media Flows with Priority"> <name>Media Flows with Priority</name>
<t>In this test case media flows will have different priority levels. <t>In this test case, media flows will have different priority levels.
This will be an extension of <xref target="competing-rmcat-flow"/> This is an extension of <xref target="competing-rmcat-flow" format="defa
where the same test will be run with different priority levels imposed ult"/>
where the same test is run with different priority levels imposed
on each of the media flows. For example, the first flow (S1) is on each of the media flows. For example, the first flow (S1) is
assigned a priority of 2 whereas the remaining two flows (S2 and S3) assigned a priority of 2, whereas the remaining two flows (S2 and S3)
are assigned a priority of 1. The candidate algorithm must reflect the are assigned a priority of 1. The candidate algorithm must reflect the
relative priorities assigned to each media flow. In this case, the relative priorities assigned to each media flow. In this case, the
first flow (S1) must arrive at a steady-state rate approximately twice first flow (S1) must arrive at a steady-state rate approximately twice
of that of the other two flows (S2 and S3).</t> that of the other two flows (S2 and S3).</t>
<t>The candidate algorithm can use a coupled congestion control <t>The candidate algorithm can use a coupled congestion control
mechanism <xref target="I-D.ietf-rmcat-coupled-cc"/> or use a weighted mechanism <xref target="RFC8699" format="default"/> or use a weighted
priority scheduler for the bandwidth distribution according to the priority scheduler for the bandwidth distribution according to the
respective media flow priority or use.</t> respective media flow priority or use.</t>
</section> </section>
<section numbered="true" toc="default">
<section title="Explicit Congestion Notification Usage"> <name>Explicit Congestion Notification Usage</name>
<t>This test case requires to run all the basic test cases with the <t>This test case requires running all the basic test cases with the
availability of Explicit Congestion Notification (ECN) <xref availability of Explicit Congestion Notification (ECN)
target="RFC6679"/> feature enabled. The goal of this test is to <xref target="RFC6679" format="default"/> feature enabled. The goal of t
his test is to
exhibit that the candidate algorithms do not fail when ECN signals are exhibit that the candidate algorithms do not fail when ECN signals are
available. With ECN signals enabled the algorithms are expected to available. With ECN signals enabled, the algorithms are expected to
perform better than their delay-based variants.</t> perform better than their delay-based variants.</t>
</section> </section>
<section numbered="true" toc="default">
<section title="Multiple Bottlenecks"> <name>Multiple Bottlenecks</name>
<t>In this test case one congestion controlled media flow, S1-&gt;R1, <t>In this test case, one congestion-controlled media flow, S1-&gt;R1,
traverses a path with multiple bottlenecks. As illustrated in <xref traverses a path with multiple bottlenecks. As illustrated in
target="fig-eval-topo-6-2"/>, the first flow (S1-&gt;R1) competes with <xref target="fig-eval-topo-6-2" format="default"/>, the first flow (S1-
the second congestion controlled media flow (S2-&gt;R2) over the link &gt;R1) competes with
between A and B which is close to the sender side; again, that flow the second congestion-controlled media flow (S2-&gt;R2) over the link
(S1-&gt;R1) competes with the third congestion controlled media flow between A and B, which is close to the sender side. Again, that flow
(S3-&gt;R3) over the link between C and D which is close to the (S1-&gt;R1) competes with the third congestion-controlled media flow
(S3-&gt;R3) over the link between C and D, which is close to the
receiver side. The goal of this test is to ensure that the candidate receiver side. The goal of this test is to ensure that the candidate
algorithms work properly in the presence of multiple bottleneck links algorithms work properly in the presence of multiple bottleneck links
on the end to end path.</t> on the end-to-end path.</t>
<dl spacing="normal">
<t>Expected behavior: The candidate algorithm is expected to achieve <dt>Expected behavior:</dt><dd> The candidate algorithm is expected to achieve
full utilization at both bottleneck links without starving any of the full utilization at both bottleneck links without starving any of the
three congestion controlled media flows and ensuring fair share of the three congestion-controlled media flows and ensuring fair share of the
available bandwidth at each bottlenecks.</t> available bandwidth at each bottleneck.</dd>
</dl>
<t><figure anchor="fig-eval-topo-6-2" <figure anchor="fig-eval-topo-6-2">
title="Testbed Topology for Multiple Bottlenecks"> <name>Testbed Topology for Multiple Bottlenecks</name>
<artwork><![CDATA[ <artwork name="" type="" align="left" alt=""><![CDATA[
Forward ----> Forward ---->
+---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+
|S2 | |R2 | |S3 | |R3 | |S2 | |R2 | |S3 | |R3 |
+---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+
| | | | | | | |
| | | | | | | |
+---+ +-----+ +-----+ +-----+ +-----+ +---+ +---+ +-----+ +-----+ +-----+ +-----+ +---+
|S1 |=======| A |------>| B |----->| C |---->| D |=======|R1 | |S1 |======| A |------>| B |----->| C |---->| D |======|R1 |
+---+ | |<------| |<-----| |<----| | +---+ +---+ | |<------| |<-----| |<----| | +---+
+-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+
1st 2nd
Bottleneck (A->B) Bottleneck (C->D)
<------ Backward 1st 2nd
Bottleneck (A->B) Bottleneck (C->D)
<------ Backward
]]></artwork> ]]></artwork>
</figure></t> </figure>
<t>Testbed topology: Three media sources S1, S2, and S3 are connected
to respective destinations R1, R2, and R3. For all three flows the
media traffic is transported over the forward path and corresponding
feedback/control traffic is transported over the backward path.</t>
<t>Testbed attributes:</t>
<t><list style="symbols">
<t>Test duration: 300s</t>
<t>Path characteristics: <list style="symbols">
<t>Reference bottleneck capacity: 2Mbps.</t>
<t>Path capacity ratio between A and B: 1.0</t>
<t>Path capacity ratio between B and C: 4.0.</t>
<t>Path capacity ratio between C and D: 0.75.</t>
<t>One-Way propagation delay: <list style="numbers">
<t>Between S1 and R1: 100ms</t>
<t>Between S2 and R2: 40ms</t>
<t>Between S3 and R3: 40ms</t>
</list></t>
</list></t>
<t>Application-related: <list style="symbols">
<t>Media Source: <list style="symbols">
<t>Media type: Video <list style="symbols">
<t>Media direction: Forward</t>
<t>Number of media sources: Three (3)</t>
<t>Media timeline: <list style="symbols">
<t>Start time: 0s.</t>
<t>End time: 299s.</t>
</list></t>
</list></t>
<t>Media type: Audio <list style="symbols">
<t>Media direction: Forward</t>
<t>Number of media sources: Three (3)</t>
<t>Media timeline: <list style="symbols">
<t>Start time: 0s.</t>
<t>End time: 299s.</t>
</list></t>
</list></t>
</list></t>
<t>Competing traffic: <list style="symbols"> <dl spacing="normal">
<t>Number of sources : Zero (0)</t> <dt>Testbed topology:</dt><dd>Three media sources S1, S2, and S3 are connected
</list></t> to respective destinations R1, R2, and R3. For all three flows, the
</list></t> media traffic is transported over the forward path, and the corresponding
</list></t> feedback/control traffic is transported over the backward path.</dd>
<dt>Testbed attributes:</dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Test duration:</dt><dd> 300 s</dd>
<dt>Path characteristics: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Reference bottleneck capacity:</dt><dd>2 Mbps</dd>
<dt>Path capacity ratio between A and B:</dt><dd>1.0</dd>
<dt>Path capacity ratio between B and C:</dt><dd>4.0</dd>
<dt>Path capacity ratio between C and D:</dt><dd>0.75</dd>
<dt>One-way propagation delay: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Between S1 and R1:</dt><dd>100 ms</dd>
<dt>Between S2 and R2:</dt><dd>40 ms</dd>
<dt>Between S3 and R3:</dt><dd>40 ms</dd>
</dl>
</dd>
</dl>
</dd>
<dt>Application-related: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Media source:</dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Media type:</dt>
<dd>
<t>Video </t>
<dl spacing="normal">
<dt>Media direction:</dt><dd>Forward</dd>
<dt>Number of media sources:</dt><dd>Three (3)</dd>
<dt>Media timeline: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Start time:</dt><dd> 0 s</dd>
<dt>End time: </dt><dd>299 s</dd>
</dl>
</dd>
</dl>
</dd>
<dt>Media type:</dt>
<dd>
<t>Audio</t>
<dl spacing="normal">
<dt>Media direction:</dt><dd>Forward</dd>
<dt>Number of media sources:</dt><dd>Three (3)</dd>
<dt>Media timeline: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Start time:</dt><dd>0 s</dd>
<dt>End time:</dt><dd>299 s</dd>
</dl>
</dd>
</dl>
</dd>
</dl>
</dd>
<dt>Competing traffic: </dt>
<dd><t><br/></t>
<dl spacing="normal">
<dt>Number of sources:</dt><dd>Zero (0)</dd>
</dl>
</dd>
</dl>
</dd>
</dl>
</dd>
</dl>
</section> </section>
</section> </section>
<section numbered="true" toc="default">
<name>Wireless Access Links</name>
<section title="Wireless Access Links"> <t>Additional wireless network (both cellular network and Wi-Fi network)
<!-- --> specific test cases are defined in <xref target="RFC8869" format="default"
/>.</t>
<t>Additional wireless network (both cellular network and WiFi network)
specific test cases are defined in <xref
target="I-D.ietf-rmcat-wireless-tests"/>.</t>
</section> </section>
<section numbered="true" toc="default">
<section title="Security Considerations"> <name>Security Considerations</name>
<t>The security considerations in <xref <t>The security considerations in <xref target="RFC8868" section="6" secti
target="I-D.ietf-rmcat-eval-criteria"/> and the relevant congestion onFormat="of" format="default"/> and the relevant congestion
control algorithms apply. The principles for congestion control are control algorithms apply. The principles for congestion control are
described in <xref target="RFC2914"/>, and in particular any new method described in <xref target="RFC2914" format="default"/>, and in particular any new method
must implement safeguards to avoid congestion collapse of the must implement safeguards to avoid congestion collapse of the
Internet.</t> Internet.</t>
<t>The evaluation of the test cases are intended to be run in a <t>The evaluation of the test cases are intended to be run in a
controlled lab environment. Hence, the applications, simulators and controlled lab environment. Hence, the applications, simulators, and
network nodes ought to be well-behaved and should not impact the desired network nodes ought to be well-behaved and should not impact the desired
results. Moreover, proper measures must be taken to avoid leaking results. Moreover, proper measures must be taken to avoid leaking
non-responsive traffic from unproven congestion avoidance techniques nonresponsive traffic from unproven congestion avoidance techniques
onto the open Internet.</t> onto the open Internet.</t>
</section> </section>
<section numbered="true" toc="default">
<section title="IANA Considerations"> <name>IANA Considerations</name>
<t>There are no IANA impacts in this memo.</t> <t>This document has no IANA actions.</t>
</section>
<section title="Acknowledgements">
<t>Much of this document is derived from previous work on congestion
control at the IETF.</t>
<t>The content and concepts within this document are a product of the
discussion carried out in the Design Team.</t>
</section> </section>
</middle> </middle>
<back> <back>
<references title="Normative References"> <references>
<?rfc include='reference.RFC.6679.xml'?> <name>References</name>
<references>
<!-- RTP related --> <name>Normative References</name>
<xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/refer
&rfc3550; ence.RFC.6679.xml"/>
<xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/refer
&rfc3551; ence.RFC.3550.xml"/>
<xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/refer
&rfc3611; ence.RFC.3551.xml"/>
<xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/refer
&rfc4585; ence.RFC.3611.xml"/>
<xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/refer
&rfc5506; ence.RFC.4585.xml"/>
<xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/refer
<!--RMCAT related --> ence.RFC.5506.xml"/>
&I-D.ietf-rmcat-eval-criteria;
&I-D.ietf-rmcat-wireless-tests;
&I-D.ietf-rmcat-video-traffic-model;
&I-D.ietf-rmcat-cc-requirements;
<?rfc include='reference.RFC.5681.xml'?>
<!--&rfc5681; -->
<!-- Standard TCP -->
</references>
<references title="Informative References">
<!--
&I-D.ietf-rtcweb-use-cases-and-requirements; -->
<?rfc include='reference.RFC.2914.xml'?>
<?rfc include='reference.RFC.8290.xml'?>
<?rfc include='reference.RFC.8033.xml'?> <!-- draft-ietf-rmcat-eval-criteria (8868) part of C238 -->
<reference anchor="RFC8868" target="https://www.rfc-editor.org/info/rfc8868">
<front>
<title>Evaluating Congestion Control for Interactive Real-time Media</title>
<?rfc include='reference.RFC.7567.xml'?> <author initials='V' surname='Singh' fullname='Varun Singh'>
<organization />
</author>
<!-- &rfc5033; --> <author initials='J' surname='Ott' fullname='Joerg Ott'>
<organization />
</author>
<!-- CC Evaluation --> <author initials='S' surname='Holmer' fullname='Stefan Holmer'>
<organization />
</author>
<!-- &rfc5166; --> <date month='July' year='2020' />
<!-- CC Metrics --> </front>
<!----> <seriesInfo name="RFC" value="8868"/>
<seriesInfo name="DOI" value="10.17487/RFC8868"/>
</reference>
<!-- <!-- draft-ietf-rmcat-wireless-tests-11 (8869) part of C238 -->
<reference anchor="RFC8869" target="https://www.rfc-editor.org/info/rfc8869">
<front>
<title>Evaluation Test Cases for Interactive Real-Time Media over Wireless Netwo
rks</title>
<!----> <author initials="Z" surname="Sarker" fullname="Zaheduzzaman Sarker">
<organization/>
</author>
<reference anchor="xiph-seq"> <author initials="X" surname="Zhu" fullname="Xiaoqing Zhu">
<front> <organization/>
<title>Video Test Media</title> </author>
<author fullname="" initials="" surname="Xiph.org"/> <author initials="J" surname="Fu" fullname="Jian Fu">
<organization/>
</author>
<date month="" year=""/> <date month='July' year='2020' />
</front>
<seriesInfo name="http://media.xiph.org/video/derf/" value=""/> </front>
</reference> <seriesInfo name="RFC" value="8869"/>
<seriesInfo name="DOI" value="10.17487/RFC8869"/>
</reference>
<reference anchor="HEVC-seq"> <!-- [I-D.ietf-rmcat-video-traffic-model] Published as RFC 8593 -->
<front> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC
<title>Test Sequences</title> .8593.xml"/>
<author fullname="" initials="" surname="HEVC"/> <!-- draft-ietf-rmcat-cc-requirements-09: 8836 -->
<reference anchor="RFC8836" target="https://www.rfc-editor.org/info/rfc8836">
<front>
<title>Congestion Control Requirements for Interactive Real-Time Media</titl
e>
<author initials="R" surname="Jesup" fullname="Randell Jesup">
<organization/>
</author>
<author initials="Z" surname="Sarker" fullname="Zaheduzzaman Sarker" role="e
ditor">
<organization/>
</author>
<date month="July" year="2020"/>
</front>
<seriesInfo name="RFC" value="8836" />
<seriesInfo name="DOI" value="10.17487/RFC8836"/>
</reference>
<date month="" year=""/> <xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/refer
</front> ence.RFC.5681.xml"/>
</references>
<references>
<name>Informative References</name>
<xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/refer
ence.RFC.2914.xml"/>
<xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/refer
ence.RFC.8290.xml"/>
<xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/refer
ence.RFC.8033.xml"/>
<xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/refer
ence.RFC.7567.xml"/>
<seriesInfo name="http://www.netlab.tkk.fi/~varun/test_sequences/" <reference anchor="xiph-seq" target="http://media.xiph.org/video/derf/">
value=""/> <front>
</reference> <title>Video Test Media</title>
<author>
<organization>Xiph.org</organization>
</author>
</front>
</reference>
<?rfc include='reference.I-D.ietf-rmcat-coupled-cc'?> <reference anchor="HEVC-seq" target="http://www.netlab.tkk.fi/~varun/tes
t_sequences/">
<front>
<title>Test Sequences</title>
<author>
<organization>HEVC</organization>
</author>
</front>
</reference>
<!-- --> <!-- [I-D.ietf-rmcat-coupled-cc] Published as RFC 8699 -->
<xi:include href="https://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC
.8699.xml"/>
</references>
</references> </references>
<section numbered="false" toc="default">
<name>Acknowledgments</name>
<t>Much of this document is derived from previous work on congestion
control at the IETF.</t>
<t>The content and concepts within this document are a product of the
discussion carried out within the Design Team.</t>
</section>
</back> </back>
</rfc> </rfc>
 End of changes. 193 change blocks. 
1535 lines changed or deleted 1623 lines changed or added

This html diff was produced by rfcdiff 1.45. The latest version is available from http://tools.ietf.org/tools/rfcdiff/