Bug 1325536 - ice telemetry. r=drno,chutten,bsmedberg
MozReview-Commit-ID: 8pZBXA8Pjea
--- a/media/mtransport/nricectx.cpp
+++ b/media/mtransport/nricectx.cpp
@@ -681,27 +681,61 @@ void NrIceCtx::internal_DeinitializeGlob
nr_crypto_vtbl = nullptr;
initialized = false;
}
void NrIceCtx::internal_SetTimerAccelarator(int divider) {
ctx_->test_timer_divider = divider;
}
-NrIceCtx::~NrIceCtx() {
+void NrIceCtx::AccumulateStats(const NrIceStats& stats) {
+ nr_ice_accumulate_count(&(ctx_->stats.stun_retransmits),
+ stats.stun_retransmits);
+ nr_ice_accumulate_count(&(ctx_->stats.turn_401s), stats.turn_401s);
+ nr_ice_accumulate_count(&(ctx_->stats.turn_403s), stats.turn_403s);
+ nr_ice_accumulate_count(&(ctx_->stats.turn_438s), stats.turn_438s);
+}
+
+NrIceStats NrIceCtx::Destroy() {
+ // designed to be called more than once so if stats are desired, this can be
+ // called just prior to the destructor
MOZ_MTLOG(ML_DEBUG, "Destroying ICE ctx '" << name_ <<"'");
for (auto stream = streams_.begin(); stream != streams_.end(); stream++) {
if (*stream) {
(*stream)->Close();
}
}
- nr_ice_peer_ctx_destroy(&peer_);
- nr_ice_ctx_destroy(&ctx_);
+
+ NrIceStats stats;
+ if (ctx_) {
+ stats.stun_retransmits = ctx_->stats.stun_retransmits;
+ stats.turn_401s = ctx_->stats.turn_401s;
+ stats.turn_403s = ctx_->stats.turn_403s;
+ stats.turn_438s = ctx_->stats.turn_438s;
+ }
+
+ if (peer_) {
+ nr_ice_peer_ctx_destroy(&peer_);
+ }
+ if (ctx_) {
+ nr_ice_ctx_destroy(&ctx_);
+ }
+
delete ice_handler_vtbl_;
delete ice_handler_;
+
+ ice_handler_vtbl_ = 0;
+ ice_handler_ = 0;
+ streams_.clear();
+
+ return stats;
+}
+
+NrIceCtx::~NrIceCtx() {
+ Destroy();
}
void
NrIceCtx::SetStream(size_t index, NrIceMediaStream* stream) {
if (index >= streams_.size()) {
streams_.resize(index + 1);
}
--- a/media/mtransport/nricectx.h
+++ b/media/mtransport/nricectx.h
@@ -187,16 +187,24 @@ class NrIceProxyServer {
private:
std::string host_;
uint16_t port_;
std::string alpn_;
};
class TestNat;
+class NrIceStats {
+ public:
+ uint16_t stun_retransmits;
+ uint16_t turn_401s;
+ uint16_t turn_403s;
+ uint16_t turn_438s;
+};
+
class NrIceCtx {
friend class NrIceCtxHandler;
public:
enum ConnectionState { ICE_CTX_INIT,
ICE_CTX_CHECKING,
ICE_CTX_CONNECTED,
ICE_CTX_COMPLETED,
ICE_CTX_FAILED,
@@ -317,16 +325,19 @@ class NrIceCtx {
// Notify that the network has gone online/offline
void UpdateNetworkState(bool online);
// Finalize the ICE negotiation. I.e., there will be no
// more forking.
nsresult Finalize();
+ void AccumulateStats(const NrIceStats& stats);
+ NrIceStats Destroy();
+
// Are we trickling?
bool generating_trickle() const { return trickle_; }
// Signals to indicate events. API users can (and should)
// register for these.
sigslot::signal2<NrIceCtx*, NrIceCtx::GatheringState>
SignalGatheringStateChange;
sigslot::signal2<NrIceCtx*, NrIceCtx::ConnectionState>
--- a/media/mtransport/nricectxhandler.cpp
+++ b/media/mtransport/nricectxhandler.cpp
@@ -136,25 +136,56 @@ NrIceCtxHandler::BeginIceRestart(RefPtr<
current_ctx = new_ctx;
return true;
}
void
NrIceCtxHandler::FinalizeIceRestart()
{
+ if (old_ctx) {
+ // Fixup the telemetry by transferring old stats to current ctx.
+ NrIceStats stats = old_ctx->Destroy();
+ current_ctx->AccumulateStats(stats);
+ }
+
// no harm calling this even if we're not in the middle of restarting
old_ctx = nullptr;
}
void
NrIceCtxHandler::RollbackIceRestart()
{
if (old_ctx == nullptr) {
return;
}
current_ctx = old_ctx;
old_ctx = nullptr;
}
+NrIceStats NrIceCtxHandler::Destroy()
+{
+ NrIceStats stats;
+
+ // designed to be called more than once so if stats are desired, this can be
+ // called just prior to the destructor
+ if (old_ctx && current_ctx) {
+ stats = old_ctx->Destroy();
+ current_ctx->AccumulateStats(stats);
+ }
+
+ if (current_ctx) {
+ stats = current_ctx->Destroy();
+ }
+
+ old_ctx = nullptr;
+ current_ctx = nullptr;
+
+ return stats;
+}
+
+NrIceCtxHandler::~NrIceCtxHandler()
+{
+ Destroy();
+}
} // close namespace
--- a/media/mtransport/nricectxhandler.h
+++ b/media/mtransport/nricectxhandler.h
@@ -26,25 +26,26 @@ public:
RefPtr<NrIceCtx> ctx() { return current_ctx; }
bool BeginIceRestart(RefPtr<NrIceCtx> new_ctx);
bool IsRestarting() const { return (old_ctx != nullptr); }
void FinalizeIceRestart();
void RollbackIceRestart();
+ NrIceStats Destroy();
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(NrIceCtxHandler)
private:
NrIceCtxHandler(const std::string& name,
bool offerer,
NrIceCtx::Policy policy);
NrIceCtxHandler() = delete;
- ~NrIceCtxHandler() {}
+ ~NrIceCtxHandler();
DISALLOW_COPY_ASSIGN(NrIceCtxHandler);
RefPtr<NrIceCtx> current_ctx;
RefPtr<NrIceCtx> old_ctx; // for while restart is in progress
int restart_count; // used to differentiate streams between restarted ctx
};
} // close namespace
--- a/media/mtransport/third_party/nICEr/src/ice/ice_candidate.c
+++ b/media/mtransport/third_party/nICEr/src/ice/ice_candidate.c
@@ -316,16 +316,30 @@ int nr_ice_candidate_destroy(nr_ice_cand
nr_ice_candidate_mark_done(cand, NR_ICE_CAND_STATE_FAILED);
}
switch(cand->type){
case HOST:
break;
#ifdef USE_TURN
case RELAYED:
+ // record stats back to the ice ctx on destruction
+ if (cand->u.relayed.turn) {
+ nr_ice_accumulate_count(&(cand->ctx->stats.turn_401s), cand->u.relayed.turn->cnt_401s);
+ nr_ice_accumulate_count(&(cand->ctx->stats.turn_403s), cand->u.relayed.turn->cnt_403s);
+ nr_ice_accumulate_count(&(cand->ctx->stats.turn_438s), cand->u.relayed.turn->cnt_438s);
+
+ nr_turn_stun_ctx* stun_ctx;
+ stun_ctx = STAILQ_FIRST(&cand->u.relayed.turn->stun_ctxs);
+ while (stun_ctx) {
+ nr_ice_accumulate_count(&(cand->ctx->stats.stun_retransmits), stun_ctx->stun->retransmit_ct);
+
+ stun_ctx = STAILQ_NEXT(stun_ctx, entry);
+ }
+ }
if (cand->u.relayed.turn_handle)
nr_ice_socket_deregister(cand->isock, cand->u.relayed.turn_handle);
if (cand->u.relayed.srvflx_candidate)
cand->u.relayed.srvflx_candidate->u.srvrflx.relay_candidate=0;
nr_turn_client_ctx_destroy(&cand->u.relayed.turn);
nr_socket_destroy(&cand->u.relayed.turn_sock);
break;
#endif /* USE_TURN */
--- a/media/mtransport/third_party/nICEr/src/ice/ice_candidate_pair.c
+++ b/media/mtransport/third_party/nICEr/src/ice/ice_candidate_pair.c
@@ -147,16 +147,21 @@ int nr_ice_candidate_pair_destroy(nr_ice
nr_ice_cand_pair *pair;
if(!pairp || !*pairp)
return(0);
pair=*pairp;
*pairp=0;
+ // record stats back to the ice ctx on destruction
+ if (pair->stun_client) {
+ nr_ice_accumulate_count(&(pair->local->ctx->stats.stun_retransmits), pair->stun_client->retransmit_ct);
+ }
+
RFREE(pair->as_string);
RFREE(pair->foundation);
nr_ice_socket_deregister(pair->local->isock,pair->stun_client_handle);
if (pair->stun_client) {
RFREE(pair->stun_client->params.ice_binding_request.username);
RFREE(pair->stun_client->params.ice_binding_request.password.data);
nr_stun_client_ctx_destroy(&pair->stun_client);
}
--- a/media/mtransport/third_party/nICEr/src/ice/ice_ctx.c
+++ b/media/mtransport/third_party/nICEr/src/ice/ice_ctx.c
@@ -1050,8 +1050,21 @@ int nr_ice_get_new_ice_pwd(char** pwd)
abort:
if(_status) {
RFREE(*pwd);
*pwd = 0;
}
return(_status);
}
+#ifndef UINT2_MAX
+#define UINT2_MAX ((UINT2)(65535U))
+#endif
+
+void nr_ice_accumulate_count(UINT2* orig_count, UINT2 new_count)
+ {
+ if (UINT2_MAX - new_count < *orig_count) {
+ // don't rollover, just stop accumulating at MAX value
+ *orig_count = UINT2_MAX;
+ } else {
+ *orig_count += new_count;
+ }
+ }
--- a/media/mtransport/third_party/nICEr/src/ice/ice_ctx.h
+++ b/media/mtransport/third_party/nICEr/src/ice/ice_ctx.h
@@ -107,16 +107,23 @@ typedef void (*nr_ice_trickle_candidate_
typedef struct nr_ice_stun_id_ {
UCHAR id[12];
STAILQ_ENTRY(nr_ice_stun_id_) entry;
} nr_ice_stun_id;
typedef STAILQ_HEAD(nr_ice_stun_id_head_,nr_ice_stun_id_) nr_ice_stun_id_head;
+typedef struct nr_ice_stats_ {
+ UINT2 stun_retransmits;
+ UINT2 turn_401s;
+ UINT2 turn_403s;
+ UINT2 turn_438s;
+} nr_ice_stats;
+
struct nr_ice_ctx_ {
UINT4 flags;
char *label;
char *ufrag;
char *pwd;
UINT4 Ta;
@@ -150,16 +157,17 @@ struct nr_ice_ctx_ {
NR_async_cb done_cb;
void *cb_arg;
nr_ice_trickle_candidate_cb trickle_cb;
void *trickle_cb_arg;
char force_net_interface[MAXIFNAME];
+ nr_ice_stats stats;
};
int nr_ice_ctx_create(char *label, UINT4 flags, nr_ice_ctx **ctxp);
int nr_ice_ctx_create_with_credentials(char *label, UINT4 flags, char* ufrag, char* pwd, nr_ice_ctx **ctxp);
#define NR_ICE_CTX_FLAGS_OFFERER 1
#define NR_ICE_CTX_FLAGS_ANSWERER (1<<1)
#define NR_ICE_CTX_FLAGS_AGGRESSIVE_NOMINATION (1<<2)
#define NR_ICE_CTX_FLAGS_LITE (1<<3)
@@ -187,16 +195,18 @@ int nr_ice_ctx_copy_turn_servers(nr_ice_
int nr_ice_ctx_set_resolver(nr_ice_ctx *ctx, nr_resolver *resolver);
int nr_ice_ctx_set_interface_prioritizer(nr_ice_ctx *ctx, nr_interface_prioritizer *prioritizer);
int nr_ice_ctx_set_turn_tcp_socket_wrapper(nr_ice_ctx *ctx, nr_socket_wrapper_factory *wrapper);
void nr_ice_ctx_set_socket_factory(nr_ice_ctx *ctx, nr_socket_factory *factory);
int nr_ice_ctx_set_trickle_cb(nr_ice_ctx *ctx, nr_ice_trickle_candidate_cb cb, void *cb_arg);
int nr_ice_ctx_hide_candidate(nr_ice_ctx *ctx, nr_ice_candidate *cand);
int nr_ice_get_new_ice_ufrag(char** ufrag);
int nr_ice_get_new_ice_pwd(char** pwd);
+// accumulate a count without worrying about rollover
+void nr_ice_accumulate_count(UINT2* orig_count, UINT2 new_count);
#define NR_ICE_MAX_ATTRIBUTE_SIZE 256
extern int LOG_ICE;
#ifdef __cplusplus
}
#endif /* __cplusplus */
--- a/media/mtransport/third_party/nICEr/src/stun/stun_client_ctx.c
+++ b/media/mtransport/third_party/nICEr/src/stun/stun_client_ctx.c
@@ -247,16 +247,19 @@ static void nr_stun_client_timer_expired
r_log(NR_LOG_STUN,LOG_INFO,"STUN-CLIENT(%s): Timed out",ctx->label);
ctx->state=NR_STUN_CLIENT_STATE_TIMED_OUT;
ABORT(R_FAILED);
}
if (ctx->state != NR_STUN_CLIENT_STATE_RUNNING)
ABORT(R_NOT_PERMITTED);
+ // track retransmits for ice telemetry
+ nr_ice_accumulate_count(&(ctx->retransmit_ct), 1);
+
/* as a side effect will reset the timer */
nr_stun_client_send_request(ctx);
_status = 0;
abort:
if (ctx->state != NR_STUN_CLIENT_STATE_RUNNING) {
/* Cancel the timer firing */
if (ctx->timer_handle){
--- a/media/mtransport/third_party/nICEr/src/stun/stun_client_ctx.h
+++ b/media/mtransport/third_party/nICEr/src/stun/stun_client_ctx.h
@@ -163,16 +163,17 @@ struct nr_stun_client_ctx_ {
nr_socket *sock;
nr_stun_client_auth_params auth_params;
nr_stun_client_params params;
nr_stun_client_results results;
char *nonce;
char *realm;
void *timer_handle;
int request_ct;
+ UINT2 retransmit_ct;
UINT4 rto_ms; /* retransmission time out */
double retransmission_backoff_factor;
UINT4 maximum_transmits;
UINT4 maximum_transmits_timeout_ms;
UINT4 mapped_addr_check_mask; /* What checks to run on mapped addresses */
int timeout_ms;
struct timeval timer_set;
int retry_ct;
--- a/media/mtransport/third_party/nICEr/src/stun/turn_client_ctx.c
+++ b/media/mtransport/third_party/nICEr/src/stun/turn_client_ctx.c
@@ -250,18 +250,26 @@ static void nr_turn_stun_ctx_cb(NR_SOCKE
break;
case NR_STUN_CLIENT_STATE_FAILED:
/* Special case: if this is an authentication error,
we retry once. This allows the 401/438 nonce retry
paradigm. After that, we fail */
/* TODO(ekr@rtfm.com): 401 needs a #define */
/* TODO(ekr@rtfm.com): Add alternate-server (Mozilla bug 857688) */
+ if (ctx->stun->error_code == 438) {
+ // track 438s for ice telemetry
+ nr_ice_accumulate_count(&(ctx->tctx->cnt_438s), 1);
+ }
if (ctx->stun->error_code == 401 || ctx->stun->error_code == 438) {
if (ctx->retry_ct > 0) {
+ if (ctx->stun->error_code == 401) {
+ // track 401s for ice telemetry
+ nr_ice_accumulate_count(&(ctx->tctx->cnt_401s), 1);
+ }
r_log(NR_LOG_TURN, LOG_WARNING, "TURN(%s): Exceeded the number of retries", ctx->tctx->label);
ABORT(R_FAILED);
}
if (!ctx->stun->nonce) {
r_log(NR_LOG_TURN, LOG_WARNING, "TURN(%s): 401 but no nonce", ctx->tctx->label);
ABORT(R_FAILED);
}
@@ -601,16 +609,18 @@ static void nr_turn_client_error_cb(NR_S
nr_turn_client_failed(ctx->tctx);
}
static void nr_turn_client_permission_error_cb(NR_SOCKET s, int how, void *arg)
{
nr_turn_stun_ctx *ctx = (nr_turn_stun_ctx *)arg;
if (ctx->last_error_code == 403) {
+ // track 403s for ice telemetry
+ nr_ice_accumulate_count(&(ctx->tctx->cnt_403s), 1);
r_log(NR_LOG_TURN, LOG_WARNING, "TURN(%s): mode %d, permission denied",
ctx->tctx->label, ctx->mode);
} else{
nr_turn_client_error_cb(0, 0, ctx);
}
}
--- a/media/mtransport/third_party/nICEr/src/stun/turn_client_ctx.h
+++ b/media/mtransport/third_party/nICEr/src/stun/turn_client_ctx.h
@@ -96,16 +96,21 @@ typedef struct nr_turn_client_ctx_ {
nr_turn_stun_ctx_head stun_ctxs;
nr_turn_permission_head permissions;
NR_async_cb finished_cb;
void *cb_arg;
void *connected_timer_handle;
void *refresh_timer_handle;
+
+ // ice telemetry
+ UINT2 cnt_401s;
+ UINT2 cnt_403s;
+ UINT2 cnt_438s;
} nr_turn_client_ctx;
extern int NR_LOG_TURN;
int nr_turn_client_ctx_create(const char *label, nr_socket *sock,
const char *username, Data *password,
nr_transport_addr *addr,
nr_turn_client_ctx **ctxp);
--- a/media/webrtc/signaling/src/peerconnection/PeerConnectionMedia.cpp
+++ b/media/webrtc/signaling/src/peerconnection/PeerConnectionMedia.cpp
@@ -48,16 +48,17 @@
#include "nsIProtocolProxyService.h"
#include "nsProxyRelease.h"
#if !defined(MOZILLA_EXTERNAL_LINKAGE)
#include "MediaStreamList.h"
#include "nsIScriptGlobalObject.h"
#include "mozilla/Preferences.h"
+#include "mozilla/Telemetry.h"
#include "mozilla/dom/RTCStatsReportBinding.h"
#include "MediaStreamTrack.h"
#include "VideoStreamTrack.h"
#include "MediaStreamError.h"
#include "MediaManager.h"
#endif
@@ -786,16 +787,21 @@ PeerConnectionMedia::RollbackIceRestart_
if (!aFlow) continue;
TransportLayerIce* ice =
static_cast<TransportLayerIce*>(aFlow->GetLayer(TransportLayerIce::ID()));
ice->RestoreOldStream();
}
mIceCtxHdlr->RollbackIceRestart();
ConnectSignals(mIceCtxHdlr->ctx().get(), restartCtx.get());
+
+ // Fixup the telemetry by transferring abandoned ctx stats to current ctx.
+ NrIceStats stats = restartCtx->Destroy();
+ restartCtx = nullptr;
+ mIceCtxHdlr->ctx()->AccumulateStats(stats);
}
bool
PeerConnectionMedia::GetPrefDefaultAddressOnly() const
{
ASSERT_ON_THREAD(mMainThread); // will crash on STS thread
#if !defined(MOZILLA_EXTERNAL_LINKAGE)
@@ -1096,16 +1102,35 @@ PeerConnectionMedia::ShutdownMediaTransp
}
for (uint32_t i=0; i < mRemoteSourceStreams.Length(); ++i) {
mRemoteSourceStreams[i]->DetachTransport_s();
}
disconnect_all();
mTransportFlows.clear();
+
+#if !defined(MOZILLA_EXTERNAL_LINKAGE)
+ NrIceStats stats = mIceCtxHdlr->Destroy();
+
+ CSFLogDebug(logTag, "Ice Telemetry: stun (retransmits: %d)"
+ " turn (401s: %d 403s: %d 438s: %d)",
+ stats.stun_retransmits, stats.turn_401s, stats.turn_403s,
+ stats.turn_438s);
+
+ Telemetry::ScalarAdd(Telemetry::ScalarID::WEBRTC_NICER_STUN_RETRANSMITS,
+ stats.stun_retransmits);
+ Telemetry::ScalarAdd(Telemetry::ScalarID::WEBRTC_NICER_TURN_401S,
+ stats.turn_401s);
+ Telemetry::ScalarAdd(Telemetry::ScalarID::WEBRTC_NICER_TURN_403S,
+ stats.turn_403s);
+ Telemetry::ScalarAdd(Telemetry::ScalarID::WEBRTC_NICER_TURN_438S,
+ stats.turn_438s);
+#endif
+
mIceCtxHdlr = nullptr;
mMainThread->Dispatch(WrapRunnable(this, &PeerConnectionMedia::SelfDestruct_m),
NS_DISPATCH_NORMAL);
}
LocalSourceStreamInfo*
PeerConnectionMedia::GetLocalStreamByIndex(int aIndex)
--- a/toolkit/components/telemetry/Scalars.yaml
+++ b/toolkit/components/telemetry/Scalars.yaml
@@ -315,8 +315,95 @@ telemetry.test:
- 1278556
description: A testing string scalar; not meant to be touched.
expires: never
kind: string
notification_emails:
- telemetry-client-dev@mozilla.com
record_in_processes:
- 'all_childs'
+
+# The following section contains WebRTC nICEr scalars
+# For more info on ICE, see https://tools.ietf.org/html/rfc5245
+# For more info on STUN, see https://tools.ietf.org/html/rfc5389
+# For more info on TURN, see https://tools.ietf.org/html/rfc5766
+webrtc.nicer:
+ stun_retransmits:
+ bug_numbers:
+ - 1325536
+ description: >
+ The count of STUN message retransmissions during a WebRTC call.
+ When sending STUN request messages over UDP, messages may be
+ dropped by the network. Retransmissions are the mechanism used to
+ accomplish reliability of the STUN request/response transaction.
+ This can happen during both successful and unsuccessful WebRTC
+ calls.
+ For more info on ICE, see https://tools.ietf.org/html/rfc5245
+ For more info on STUN, see https://tools.ietf.org/html/rfc5389
+ For more info on TURN, see https://tools.ietf.org/html/rfc5766
+ expires: "57"
+ kind: uint
+ notification_emails:
+ - webrtc-ice-telemetry-alerts@mozilla.com
+ release_channel_collection: opt-in
+ record_in_processes:
+ - 'main'
+ - 'content'
+
+ turn_401s:
+ bug_numbers:
+ - 1325536
+ description: >
+ The count of TURN 401 (Unauthorized) responses to allocation
+ requests. Only 401 responses beyond the first, expected 401 are
+ counted. More than one 401 repsonse indicates the client is
+ experiencing difficulty authenticating with the TURN server. This
+ can happen during both successful and unsuccessful WebRTC calls.
+ For more info on ICE, see https://tools.ietf.org/html/rfc5245
+ For more info on STUN, see https://tools.ietf.org/html/rfc5389
+ For more info on TURN, see https://tools.ietf.org/html/rfc5766
+ expires: "57"
+ kind: uint
+ notification_emails:
+ - webrtc-ice-telemetry-alerts@mozilla.com
+ release_channel_collection: opt-in
+ record_in_processes:
+ - 'main'
+ - 'content'
+
+ turn_403s:
+ bug_numbers:
+ - 1325536
+ description: >
+ The count of TURN 403 (Forbidden) responses to CreatePermission or
+ ChannelBind requests. This indicates that the TURN server is
+ refusing the request for an IP address or IP address/port
+ combination, likely due to administrative restrictions.
+ For more info on ICE, see https://tools.ietf.org/html/rfc5245
+ For more info on STUN, see https://tools.ietf.org/html/rfc5389
+ For more info on TURN, see https://tools.ietf.org/html/rfc5766
+ expires: "57"
+ kind: uint
+ notification_emails:
+ - webrtc-ice-telemetry-alerts@mozilla.com
+ release_channel_collection: opt-in
+ record_in_processes:
+ - 'main'
+ - 'content'
+
+ turn_438s:
+ bug_numbers:
+ - 1325536
+ description: >
+ The count of TURN 438 (Stale Nonce) responses to allocation
+ requests. This can happen during both successful and unsuccessful
+ WebRTC calls.
+ For more info on ICE, see https://tools.ietf.org/html/rfc5245
+ For more info on STUN, see https://tools.ietf.org/html/rfc5389
+ For more info on TURN, see https://tools.ietf.org/html/rfc5766
+ expires: "57"
+ kind: uint
+ notification_emails:
+ - webrtc-ice-telemetry-alerts@mozilla.com
+ release_channel_collection: opt-in
+ record_in_processes:
+ - 'main'
+ - 'content'