import{s as Mn,o as gn}from"../chunks/scheduler.cd324960.js";import{S as vn,i as bn,e as i,s as n,c as o,q as T,H as Os,h as Tn,a as r,d as s,b as l,f as mt,g as m,j as p,r as U,u as Ks,k as hn,l as g,m as a,n as c,t as h,o as d,p as y}from"../chunks/index.d5c3adcc.js";import{T as dn}from"../chunks/Tip.889bec11.js";import{C as Un,H as C,E as Jn}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.481f2bec.js";import{C as w}from"../chunks/CodeBlock.a252a8e7.js";function jn(lt){let u;return{c(){u=T(`If you face an authentication issue when loading the dataset, ensure that you have accepted the dataset's terms of use
    on the Hugging Face Hub through the following link: https://huggingface.co/datasets/mozilla-foundation/common_voice_13_0`)},l(f){u=U(f,`If you face an authentication issue when loading the dataset, ensure that you have accepted the dataset's terms of use
    on the Hugging Face Hub through the following link: https://huggingface.co/datasets/mozilla-foundation/common_voice_13_0`)},m(f,v){a(f,u,v)},d(f){f&&s(u)}}}function xn(lt){let u;return{c(){u=T("If you experience a CUDA out-of-memory (OOM) when running the above cell, incrementally reduce the `batch_size` by\n    factors of 2 until you find a batch size that fits your device.")},l(f){u=U(f,"If you experience a CUDA out-of-memory (OOM) when running the above cell, incrementally reduce the `batch_size` by\n    factors of 2 until you find a batch size that fits your device.")},m(f,v){a(f,u,v)},d(f){f&&s(u)}}}function Cn(lt){let u,f,v,ht,$,dt,R,yt,W,pa=`If you’re familiar with the <a href="https://en.wikipedia.org/wiki/Levenshtein_distance" rel="nofollow">Levenshtein distance</a> from NLP, the
metrics for assessing speech recognition systems will be familiar! Don’t worry if you’re not, we’ll go through the
explanations start-to-finish to make sure you know the different metrics and understand what they mean.`,ut,E,oa=`When assessing speech recognition systems, we compare the system’s predictions to the target text transcriptions,
annotating any errors that are present. We categorise these errors into one of three categories:`,wt,I,ma="<li>Substitutions (S): where we transcribe the <strong>wrong word</strong> in our prediction (“sit” instead of “sat”)</li> <li>Insertions (I): where we add an <strong>extra word</strong> in our prediction</li> <li>Deletions (D): where we <strong>remove a word</strong> in our prediction</li>",ft,k,ca=`These error categories are the same for all speech recognition metrics. What differs is the level at which we compute
these errors: we can either compute them on the <em>word level</em> or on the <em>character level</em>.`,Mt,B,ha="We’ll use a running example for each of the metric definitions. Here, we have a <em>ground truth</em> or <em>reference</em> text sequence:",gt,_,vt,Z,da="And a predicted sequence from the speech recognition system that we’re trying to assess:",bt,G,Tt,H,ya=`We can see that the prediction is pretty close, but some words are not quite right. We’ll evaluate this prediction
against the reference for the three most popular speech recognition metrics and see what sort of numbers we get for each.`,Ut,V,Jt,S,ua=`The <em>word error rate (WER)</em> metric is the ‘de facto’ metric for speech recognition. It calculates substitutions,
insertions and deletions on the <em>word level</em>. This means errors are annotated on a word-by-word basis. Take our example:`,jt,F,wa="<thead><tr><th>Reference:</th> <th>the</th> <th>cat</th> <th>sat</th> <th>on</th> <th>the</th> <th>mat</th></tr></thead> <tbody><tr><td>Prediction:</td> <td>the</td> <td>cat</td> <td><strong>sit</strong></td> <td>on</td> <td>the</td> <td></td></tr> <tr><td>Label:</td> <td>✅</td> <td>✅</td> <td>S</td> <td>✅</td> <td>✅</td> <td>D</td></tr></tbody>",xt,X,fa="Here, we have:",Ct,N,Ma="<li>1 substitution (“sit” instead of “sat”)</li> <li>0 insertions</li> <li>1 deletion (“mat” is missing)</li>",$t,z,ea,Rt,yn=`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mtable rowspacing="0.25em" columnalign="right left" columnspacing="0em"><mtr><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow><mi>W</mi><mi>E</mi><mi>R</mi></mrow></mstyle></mtd><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow><mrow></mrow><mo>=</mo><mfrac><mrow><mi>S</mi><mo>+</mo><mi>I</mi><mo>+</mo><mi>D</mi></mrow><mi>N</mi></mfrac></mrow></mstyle></mtd></mtr><mtr><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow></mrow></mstyle></mtd><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow><mrow></mrow><mo>=</mo><mfrac><mrow><mn>1</mn><mo>+</mo><mn>0</mn><mo>+</mo><mn>1</mn></mrow><mn>6</mn></mfrac></mrow></mstyle></mtd></mtr><mtr><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow></mrow></mstyle></mtd><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow><mrow></mrow><mo>=</mo><mn>0.333</mn></mrow></mstyle></mtd></mtr></mtable><annotation encoding="application/x-tex">
\\begin{aligned}
WER &amp;= \\frac{S + I + D}{N} \\\\
&amp;= \\frac{1 + 0 + 1}{6} \\\\
&amp;= 0.333
\\end{aligned}
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:6.1538em;vertical-align:-2.8269em;"></span><span class="mord"><span class="mtable"><span class="col-align-r"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:3.3269em;"><span style="top:-5.3269em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">W</span><span class="mord mathnormal" style="margin-right:0.00773em;">ER</span></span></span><span style="top:-3.0194em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"></span></span><span style="top:-1.1934em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:2.8269em;"><span></span></span></span></span></span><span class="col-align-l"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:3.3269em;"><span style="top:-5.3269em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"><span class="mord"></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3603em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.10903em;">N</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.05764em;">S</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal" style="margin-right:0.07847em;">I</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal" style="margin-right:0.02778em;">D</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span><span style="top:-3.0194em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"><span class="mord"></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">6</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord">0</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span><span style="top:-1.1934em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"><span class="mord"></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">0.333</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:2.8269em;"><span></span></span></span></span></span></span></span></span></span></span></span>`,Wt,A,ga=`Alright! So we have a WER of 0.333, or 33.3%. Notice how the word “sit” only has one character that is wrong, but the
entire word is marked incorrect. This is a defining feature of the WER: spelling errors are penalised heavily, no matter
how minor they are.`,Et,Y,va=`The WER is defined such that <em>lower is better</em>: a lower WER means there are fewer errors in our prediction, so a perfect
speech recognition system would have a WER of zero (no errors).`,It,Q,ba=`Let’s see how we can compute the WER using 🤗 Evaluate. We’ll need two packages to compute our WER metric: 🤗 Evaluate
for the API interface, and JIWER to do the heavy lifting of running the calculation:`,kt,L,Bt,P,Ta="Great! We can now load up the WER metric and compute the figure for our example:",_t,q,Zt,D,Ua="<strong>Print Output:</strong>",Gt,O,Ht,K,Ja="0.33, or 33.3%, as expected! We now know what’s going on under-the-hood with this WER calculation.",Vt,ee,ja=`Now, here’s something that’s quite confusing… What do you think the upper limit of the WER is? You would expect it to be
1 or 100% right? Nuh uh! Since the WER is the ratio of errors to number of words (N), there is no upper limit on the WER!
Let’s take an example were we predict 10 words and the target only has 2 words. If all of our predictions were wrong (10 errors),
we’d have a WER of 10 / 2 = 5, or 500%! This is something to bear in mind if you train an ASR system and see a WER of over
100%. Although if you’re seeing this, something has likely gone wrong… 😅`,St,te,Ft,b,ta,it,xa="inverse real-time factor (RTFx)",sa,Xt,un=`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mtext>RTFx</mtext><mo>=</mo><mfrac><mtext>Audio Duration</mtext><mtext>Processing Time</mtext></mfrac></mrow><annotation encoding="application/x-tex">
\\text{RTFx} = \\frac{\\text{Audio Duration}}{\\text{Processing Time}}
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord text"><span class="mord">RTFx</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.2519em;vertical-align:-0.8804em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3714em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord text"><span class="mord">Processing Time</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord text"><span class="mord">Audio Duration</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.8804em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span>`,Nt,se,Ca=`For example, if it takes 10 seconds to transcribe 100 seconds of audio, the RTFx is 100/10 = 10. An RTFx greater than 1.0
means the system can transcribe audio faster than real-time, which is essential for live transcription applications like
video conferencing or live captioning. An RTFx of 1.0 means the system processes at exactly real-time speed, while values
below 1.0 indicate slower-than-real-time processing.`,zt,ae,$a="Key points about RTFx:",At,ne,Ra="<li><strong>Higher is better</strong>: Higher RTFx means faster processing</li> <li><strong>RTFx &gt; 1.0</strong>: Faster than real-time (good for streaming applications)</li> <li><strong>RTFx = 1.0</strong>: Processes at exactly real-time speed</li> <li><strong>RTFx &lt; 1.0</strong>: Slower than real-time (may be acceptable for batch processing)</li>",Yt,le,Wa="RTFx is hardware-dependent and varies based on factors like:",Qt,ie,Ea="<li>Model size (larger models typically have lower RTFx)</li> <li>Hardware acceleration (GPU vs CPU)</li> <li>Batch size</li> <li>Audio characteristics (sampling rate, number of channels)</li>",Lt,re,Ia=`When evaluating ASR systems, it’s important to consider both WER and RTFx together. A model with excellent WER but very
low RTFx may not be practical for real-time applications, while a model with slightly higher WER but high RTFx might be
more suitable for latency-sensitive use cases.`,Pt,pe,qt,M,aa,rt,ka="higher is better",na,pt,Ba="word accuracy (WAcc)",la,Dt,wn=`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mtable rowspacing="0.16em" columnspacing="1em"><mtr><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow><mi>W</mi><mi>A</mi><mi>c</mi><mi>c</mi><mo>=</mo><mn>1</mn><mo>−</mo><mi>W</mi><mi>E</mi><mi>R</mi></mrow></mstyle></mtd></mtr></mtable><annotation encoding="application/x-tex">
\\begin{equation}
WAcc = 1 - WER \\nonumber
\\end{equation}
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.2em;vertical-align:-0.35em;"></span><span class="mord"><span class="mtable"><span class="col-align-c"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.85em;"><span style="top:-3.01em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">W</span><span class="mord mathnormal">A</span><span class="mord mathnormal">cc</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">W</span><span class="mord mathnormal" style="margin-right:0.00773em;">ER</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.35em;"><span></span></span></span></span></span></span></span></span></span></span></span>`,Ot,oe,_a=`The WAcc is also measured on the word-level, it’s just the WER reformulated as an accuracy metric rather than an error
metric. The WAcc is very infrequently quoted in the speech literature - we think of our system predictions in terms of
word errors, and so prefer error rate metrics that are more associated with these error type annotations.`,Kt,me,es,ce,Za=`It seems a bit unfair that we marked the entire word for “sit” wrong when in fact only one letter was incorrect.
That’s because we were evaluating our system on the word level, thereby annotating errors on a word-by-word basis.
The <em>character error rate (CER)</em> assesses systems on the <em>character level</em>. This means we divide up our words into their
individual characters, and annotate errors on a character-by-character basis:`,ts,he,Ga="<thead><tr><th>Reference:</th> <th>t</th> <th>h</th> <th>e</th> <th></th> <th>c</th> <th>a</th> <th>t</th> <th></th> <th>s</th> <th>a</th> <th>t</th> <th></th> <th>o</th> <th>n</th> <th></th> <th>t</th> <th>h</th> <th>e</th> <th></th> <th>m</th> <th>a</th> <th>t</th></tr></thead> <tbody><tr><td>Prediction:</td> <td>t</td> <td>h</td> <td>e</td> <td></td> <td>c</td> <td>a</td> <td>t</td> <td></td> <td>s</td> <td><strong>i</strong></td> <td>t</td> <td></td> <td>o</td> <td>n</td> <td></td> <td>t</td> <td>h</td> <td>e</td> <td></td> <td></td> <td></td> <td></td></tr> <tr><td>Label:</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>S</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>✅</td> <td>D</td> <td>D</td> <td>D</td> <td>D</td></tr></tbody>",ss,de,Ha=`We can see now that for the word “sit”, the “s” and “t” are marked as correct. It’s only the “i” which is labelled as a
substitution error (S). Thus, we reward our system for the partially correct prediction 🤝`,as,ye,ia,ns,fn=`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mtable rowspacing="0.25em" columnalign="right left" columnspacing="0em"><mtr><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow><mi>C</mi><mi>E</mi><mi>R</mi></mrow></mstyle></mtd><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow><mrow></mrow><mo>=</mo><mfrac><mrow><mi>S</mi><mo>+</mo><mi>I</mi><mo>+</mo><mi>D</mi></mrow><mi>N</mi></mfrac></mrow></mstyle></mtd></mtr><mtr><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow></mrow></mstyle></mtd><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow><mrow></mrow><mo>=</mo><mfrac><mrow><mn>1</mn><mo>+</mo><mn>0</mn><mo>+</mo><mn>4</mn></mrow><mn>22</mn></mfrac></mrow></mstyle></mtd></mtr><mtr><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow></mrow></mstyle></mtd><mtd><mstyle scriptlevel="0" displaystyle="true"><mrow><mrow></mrow><mo>=</mo><mn>0.227</mn></mrow></mstyle></mtd></mtr></mtable><annotation encoding="application/x-tex">
\\begin{aligned}
CER &amp;= \\frac{S + I + D}{N} \\\\
&amp;= \\frac{1 + 0 + 4}{22} \\\\
&amp;= 0.227
\\end{aligned}
</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:6.1538em;vertical-align:-2.8269em;"></span><span class="mord"><span class="mtable"><span class="col-align-r"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:3.3269em;"><span style="top:-5.3269em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.00773em;">CER</span></span></span><span style="top:-3.0194em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"></span></span><span style="top:-1.1934em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:2.8269em;"><span></span></span></span></span></span><span class="col-align-l"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:3.3269em;"><span style="top:-5.3269em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"><span class="mord"></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3603em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.10903em;">N</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.05764em;">S</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal" style="margin-right:0.07847em;">I</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord mathnormal" style="margin-right:0.02778em;">D</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span><span style="top:-3.0194em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"><span class="mord"></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">22</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord">0</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord">4</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span><span style="top:-1.1934em;"><span class="pstrut" style="height:3.3603em;"></span><span class="mord"><span class="mord"></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mord">0.227</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:2.8269em;"><span></span></span></span></span></span></span></span></span></span></span></span>`,ls,ue,Va="Right! We have a CER of 0.227, or 22.7%. Notice how this is lower than our WER - we penalised the spelling error much less.",is,we,rs,fe,Sa=`In general, the WER is used far more than the CER for assessing speech systems. This is because the WER requires systems
to have greater understanding of the context of the predictions. In our example, “sit” is in the wrong tense. A system
that understands the relationship between the verb and tense of the sentence would have predicted the correct verb tense
of “sat”. We want to encourage this level of understanding from our speech systems. So although the WER is less forgiving than
the CER, it’s also more conducive to the kinds of intelligible systems we want to develop. Therefore, we typically use
the WER and would encourage you to as well! However, there are circumstances where it is not possible to use the WER.
Certain languages, such as Mandarin and Japanese, have no notion of ‘words’, and so the WER is meaningless. Here, we revert
to using the CER.`,ps,Me,Fa=`In our example, we only used one sentence when computing the WER. We would typically use an entire test set consisting
of several thousand sentences when evaluating a real system. When evaluating over multiple sentences, we aggregate S, I, D
and N across all sentences, and then compute the WER according to the formula defined above. This gives a better estimate
of the WER for unseen data.`,os,ge,ms,ve,Xa=`If we train an ASR model on data with punctuation and casing, it will learn to predict casing and punctuation in its
transcriptions. This is great when we want to use our model for actual speech recognition applications, such as
transcribing meetings or dictation, since the predicted transcriptions will be fully formatted with casing and punctuation,
a style referred to as <em>orthographic</em>.`,cs,be,Na=`However, we also have the option of <em>normalising</em> the dataset to remove any casing and punctuation. Normalising the
dataset makes the speech recognition task easier: the model no longer needs to distinguish between upper and lower case
characters, or have to predict punctuation from the audio data alone (e.g. what sound does a semi-colon make?).
Because of this, the word error rates are naturally lower (meaning the results are better). The Whisper paper demonstrates
the drastic effect that normalising transcriptions can have on WER results (<em>c.f.</em> Section 4.4 of the <a href="https://cdn.openai.com/papers/whisper.pdf" rel="nofollow">Whisper paper</a>).
While we get lower WERs, the model isn’t necessarily better for production. The lack of casing and punctuation makes the predicted
text from the model significantly harder to read. Take the example from the <a href="asr_models">previous section</a>, where we ran
Wav2Vec2 and Whisper on the same audio sample from the LibriSpeech dataset. The Wav2Vec2 model predicts neither punctuation
nor casing, whereas Whisper predicts both. Comparing the transcriptions side-by-side, we see that the Whisper transcription
is far easier to read:`,hs,Te,ds,Ue,za=`The Whisper transcription is orthographic and thus ready to go - it’s formatted as we’d expect for a meeting transcription
or dictation script with both punctuation and casing. On the contrary, we would need to use additional post-processing
to restore punctuation and casing in our Wav2Vec2 predictions if we wanted to use it for downstream applications.`,ys,Je,Aa=`There is a happy medium between normalising and not normalising: we can train our systems on orthographic transcriptions,
and then normalise the predictions and targets before computing the WER. This way, we train our systems to predict fully
formatted text, but also benefit from the WER improvements we get by normalising the transcriptions.`,us,je,Ya=`The Whisper model was released with a normaliser that effectively handles the normalisation of casing, punctuation and
number formatting among others. Let’s apply the normaliser to the Whisper transcriptions to demonstrate how we can
normalise them:`,ws,xe,fs,Ce,Qa="<strong>Output:</strong>",Ms,$e,gs,Re,La=`Great! We can see that the text has been fully lower-cased and all punctuation removed. Let’s now define the reference
transcription and then compute the normalised WER between the reference and prediction:`,vs,We,bs,Ee,Pa="<strong>Output:</strong>",Ts,Ie,Us,ke,qa=`6.25% - that’s about what we’d expect for the Whisper base model on the LibriSpeech validation set. As we see here,
we’ve predicted an orthographic transcription, but benefited from the WER boost obtained by normalising the reference and
prediction prior to computing the WER.`,Js,Be,Da=`The choice of how you normalise the transcriptions is ultimately down to your needs. We recommend training on
orthographic text and evaluating on normalised text to get the best of both worlds.`,js,_e,xs,Ze,Oa=`Alright! We’ve covered three topics so far in this Unit: pre-trained models, dataset selection and evaluation.
Let’s have some fun and put them together in one end-to-end example 🚀 We’re going to set ourselves up for the next
section on fine-tuning by evaluating the pre-trained Whisper model on the Common Voice 13 Dhivehi test set. We’ll use
the WER number we get as a <em>baseline</em> for our fine-tuning run, or a target number that we’ll try and beat 🥊`,Cs,Ge,Ka=`First, we’ll load the pre-trained Whisper model using the <code>pipeline()</code> class. This process will be extremely familiar by now!
The only new thing we’ll do is load the model in half-precision (float16) if running on a GPU - this will speed up
inference at almost no cost to WER accuracy.`,$s,He,Rs,Ve,en=`Next, we’ll load the Dhivehi test split of Common Voice 13. You’ll remember from the previous section that the Common
Voice 13 is <em>gated</em>, meaning we had to agree to the dataset terms of use before gaining access to the dataset. We can
now link our Hugging Face account to our notebook, so that we have access to the dataset from the machine we’re currently
using.`,Ws,Se,tn=`Linking the notebook to the Hub is straightforward - it simply requires entering your Hub authentication token when prompted.
Find your Hub authentication token <a href="https://huggingface.co/settings/tokens" rel="nofollow">here</a> and enter it when prompted:`,Es,Fe,Is,Xe,sn=`Great! Once we’ve linked the notebook to our Hugging Face account, we can proceed with downloading the Common Voice
dataset. This will take a few minutes to download and pre-process, fetching the data from the Hugging Face Hub and
preparing it automatically on your notebook:`,ks,Ne,Bs,J,_s,ze,an=`Evaluating over an entire dataset can be done in much the same way as over a single example - all we have to do is <strong>loop</strong>
over the input audios, rather than inferring just a single sample. To do this, we first transform our dataset into a
<code>KeyDataset</code>. All this does is pick out the particular dataset column that we want to forward to the model (in our case, that’s
the <code>&quot;audio&quot;</code> column), ignoring the rest (like the target transcriptions, which we don’t want to use for inference). We
then iterate over this transformed datasets, appending the model outputs to a list to save the predictions. The
following code cell will take approximately five minutes if running on a GPU with half-precision, peaking at 12GB memory:`,Zs,Ae,Gs,j,Hs,Ye,nn="And finally, we can compute the WER. Let’s first compute the orthographic WER, i.e. the WER without any post-processing:",Vs,Qe,Ss,Le,ln="<strong>Output:</strong>",Fs,Pe,Xs,qe,rn=`Okay… 167% essentially means our model is outputting garbage 😜 Not to worry, it’ll be our aim to improve this by
fine-tuning the model on the Dhivehi training set!`,Ns,De,pn=`Next, we’ll evaluate the normalised WER, i.e. the WER with normalisation post-processing. We have to filter out samples
that would be empty after normalisation, as otherwise the total number of words in our reference (N) would be zero, which
would give a division by zero error in our calculation:`,zs,Oe,As,Ke,on="<strong>Output:</strong>",Ys,et,Qs,tt,mn=`Again we see the drastic reduction in WER we achieve by normalising our references and predictions: the baseline model
achieves an orthographic test WER of 168%, while the normalised WER is 126%.`,Ls,st,cn=`Right then! These are the numbers that we want to try and beat when we fine-tune the model, in order to improve the Whisper
model for Dhivehi speech recognition. Continue reading to get hands-on with a fine-tuning example 🚀`,Ps,at,qs,ct,Ds;return $=new Un({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),R=new C({props:{title:"Evaluation metrics for ASR",local:"evaluation-metrics-for-asr",headingTag:"h1"}}),_=new w({props:{code:"cmVmZXJlbmNlJTIwJTNEJTIwJTIydGhlJTIwY2F0JTIwc2F0JTIwb24lMjB0aGUlMjBtYXQlMjI=",highlighted:'reference = <span class="hljs-string">&quot;the cat sat on the mat&quot;</span>',wrap:!1}}),G=new w({props:{code:"cHJlZGljdGlvbiUyMCUzRCUyMCUyMnRoZSUyMGNhdCUyMHNpdCUyMG9uJTIwdGhlJTIy",highlighted:'prediction = <span class="hljs-string">&quot;the cat sit on the&quot;</span>',wrap:!1}}),V=new C({props:{title:"Word Error Rate",local:"word-error-rate",headingTag:"h2"}}),L=new w({props:{code:"cGlwJTIwaW5zdGFsbCUyMC0tdXBncmFkZSUyMGV2YWx1YXRlJTIwaml3ZXI=",highlighted:'pip install <span class="hljs-comment">--upgrade evaluate jiwer</span>',wrap:!1}}),q=new w({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwbG9hZCUwQSUwQXdlcl9tZXRyaWMlMjAlM0QlMjBsb2FkKCUyMndlciUyMiklMEElMEF3ZXIlMjAlM0QlMjB3ZXJfbWV0cmljLmNvbXB1dGUocmVmZXJlbmNlcyUzRCU1QnJlZmVyZW5jZSU1RCUyQyUyMHByZWRpY3Rpb25zJTNEJTVCcHJlZGljdGlvbiU1RCklMEElMEFwcmludCh3ZXIp",highlighted:`<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> load

wer_metric = load(<span class="hljs-string">&quot;wer&quot;</span>)

wer = wer_metric.compute(references=[reference], predictions=[prediction])

<span class="hljs-built_in">print</span>(wer)`,wrap:!1}}),O=new w({props:{code:"MC4zMzMzMzMzMzMzMzMzMzMz",highlighted:'<span class="hljs-attribute">0</span>.<span class="hljs-number">3333333333333333</span>',wrap:!1}}),te=new C({props:{title:"Inverse Real-Time Factor (RTFx)",local:"inverse-real-time-factor-rtfx",headingTag:"h2"}}),pe=new C({props:{title:"Word Accuracy",local:"word-accuracy",headingTag:"h2"}}),me=new C({props:{title:"Character Error Rate",local:"character-error-rate",headingTag:"h2"}}),we=new C({props:{title:"Which metric should I use?",local:"which-metric-should-i-use",headingTag:"h2"}}),ge=new C({props:{title:"Normalisation",local:"normalisation",headingTag:"h2"}}),Te=new w({props:{code:"V2F2MlZlYzIlM0ElMjAlMjBIRSUyMFRFTExTJTIwVVMlMjBUSEFUJTIwQVQlMjBUSElTJTIwRkVTVElWRSUyMFNFQVNPTiUyME9GJTIwVEhFJTIwWUVBUiUyMFdJVEglMjBDSFJJU1RNQVVTJTIwQU5EJTIwUk9TRSUyMEJFRUYlMjBMT09NSU5HJTIwQkVGT1JFJTIwVVMlMjBTSU1BTFlJUyUyMERSQVdOJTIwRlJPTSUyMEVBVElORyUyMEFORCUyMElUUyUyMFJFU1VMVFMlMjBPQ0NVUiUyME1PU1QlMjBSRUFESUxZJTIwVE8lMjBUSEUlMjBNSU5EJTBBV2hpc3BlciUzQSUyMCUyMCUyMEhlJTIwdGVsbHMlMjB1cyUyMHRoYXQlMjBhdCUyMHRoaXMlMjBmZXN0aXZlJTIwc2Vhc29uJTIwb2YlMjB0aGUlMjB5ZWFyJTJDJTIwd2l0aCUyMENocmlzdG1hcyUyMGFuZCUyMHJvYXN0JTIwYmVlZiUyMGxvb21pbmclMjBiZWZvcmUlMjB1cyUyQyUyMHNpbWlsYXJseSUyMGlzJTIwZHJhd24lMjBmcm9tJTIwZWF0aW5nJTIwYW5kJTIwaXRzJTIwcmVzdWx0cyUyMG9jY3VyJTIwbW9zdCUyMHJlYWRpbHklMjB0byUyMHRoZSUyMG1pbmQu",highlighted:`<span class="hljs-symbol">Wav2Vec2:</span>  HE TELLS US THAT <span class="hljs-built_in">AT</span> THIS FESTIVE SEASON OF THE YEAR WITH CHRISTMAUS <span class="hljs-keyword">AND </span>ROSE <span class="hljs-keyword">BEEF </span>LOOMING <span class="hljs-keyword">BEFORE </span>US SIMALYIS DRAWN FROM EATING <span class="hljs-keyword">AND </span>ITS RESULTS OCCUR MOST READILY TO THE MIND
<span class="hljs-symbol">Whisper:</span>   He tells us that <span class="hljs-built_in">at</span> this festive season of the year, with Christmas <span class="hljs-keyword">and </span>roast <span class="hljs-keyword">beef </span>looming <span class="hljs-keyword">before </span>us, similarly is drawn from eating <span class="hljs-keyword">and </span>its results occur most readily to the mind.`,wrap:!1}}),xe=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5tb2RlbHMud2hpc3Blci5lbmdsaXNoX25vcm1hbGl6ZXIlMjBpbXBvcnQlMjBCYXNpY1RleHROb3JtYWxpemVyJTBBJTBBbm9ybWFsaXplciUyMCUzRCUyMEJhc2ljVGV4dE5vcm1hbGl6ZXIoKSUwQSUwQXByZWRpY3Rpb24lMjAlM0QlMjAlMjIlMjBIZSUyMHRlbGxzJTIwdXMlMjB0aGF0JTIwYXQlMjB0aGlzJTIwZmVzdGl2ZSUyMHNlYXNvbiUyMG9mJTIwdGhlJTIweWVhciUyQyUyMHdpdGglMjBDaHJpc3RtYXMlMjBhbmQlMjByb2FzdCUyMGJlZWYlMjBsb29taW5nJTIwYmVmb3JlJTIwdXMlMkMlMjBzaW1pbGFybHklMjBpcyUyMGRyYXduJTIwZnJvbSUyMGVhdGluZyUyMGFuZCUyMGl0cyUyMHJlc3VsdHMlMjBvY2N1ciUyMG1vc3QlMjByZWFkaWx5JTIwdG8lMjB0aGUlMjBtaW5kLiUyMiUwQW5vcm1hbGl6ZWRfcHJlZGljdGlvbiUyMCUzRCUyMG5vcm1hbGl6ZXIocHJlZGljdGlvbiklMEElMEFub3JtYWxpemVkX3ByZWRpY3Rpb24=",highlighted:`<span class="hljs-keyword">from</span> transformers.models.whisper.english_normalizer <span class="hljs-keyword">import</span> BasicTextNormalizer

normalizer = BasicTextNormalizer()

prediction = <span class="hljs-string">&quot; He tells us that at this festive season of the year, with Christmas and roast beef looming before us, similarly is drawn from eating and its results occur most readily to the mind.&quot;</span>
normalized_prediction = normalizer(prediction)

normalized_prediction`,wrap:!1}}),$e=new w({props:{code:"JyUyMGhlJTIwdGVsbHMlMjB1cyUyMHRoYXQlMjBhdCUyMHRoaXMlMjBmZXN0aXZlJTIwc2Vhc29uJTIwb2YlMjB0aGUlMjB5ZWFyJTIwd2l0aCUyMGNocmlzdG1hcyUyMGFuZCUyMHJvYXN0JTIwYmVlZiUyMGxvb21pbmclMjBiZWZvcmUlMjB1cyUyMHNpbWlsYXJseSUyMGlzJTIwZHJhd24lMjBmcm9tJTIwZWF0aW5nJTIwYW5kJTIwaXRzJTIwcmVzdWx0cyUyMG9jY3VyJTIwbW9zdCUyMHJlYWRpbHklMjB0byUyMHRoZSUyMG1pbmQlMjAn",highlighted:'&#x27; he tells us <span class="hljs-keyword">that</span> <span class="hljs-keyword">at</span> this festive season <span class="hljs-keyword">of</span> <span class="hljs-keyword">the</span> <span class="hljs-built_in">year</span> <span class="hljs-keyword">with</span> christmas <span class="hljs-keyword">and</span> roast beef looming <span class="hljs-keyword">before</span> us similarly <span class="hljs-keyword">is</span> drawn <span class="hljs-keyword">from</span> eating <span class="hljs-keyword">and</span> <span class="hljs-keyword">its</span> results occur most readily <span class="hljs-keyword">to</span> <span class="hljs-keyword">the</span> mind &#x27;',wrap:!1}}),We=new w({props:{code:"cmVmZXJlbmNlJTIwJTNEJTIwJTIySEUlMjBURUxMUyUyMFVTJTIwVEhBVCUyMEFUJTIwVEhJUyUyMEZFU1RJVkUlMjBTRUFTT04lMjBPRiUyMFRIRSUyMFlFQVIlMjBXSVRIJTIwQ0hSSVNUTUFTJTIwQU5EJTIwUk9BU1QlMjBCRUVGJTIwTE9PTUlORyUyMEJFRk9SRSUyMFVTJTIwU0lNSUxFUyUyMERSQVdOJTIwRlJPTSUyMEVBVElORyUyMEFORCUyMElUUyUyMFJFU1VMVFMlMjBPQ0NVUiUyME1PU1QlMjBSRUFESUxZJTIwVE8lMjBUSEUlMjBNSU5EJTIyJTBBbm9ybWFsaXplZF9yZWZlcmVjZSUyMCUzRCUyMG5vcm1hbGl6ZXIocmVmZXJlbmNlKSUwQSUwQXdlciUyMCUzRCUyMHdlcl9tZXRyaWMuY29tcHV0ZSglMEElMjAlMjAlMjAlMjByZWZlcmVuY2VzJTNEJTVCbm9ybWFsaXplZF9yZWZlcmVjZSU1RCUyQyUyMHByZWRpY3Rpb25zJTNEJTVCbm9ybWFsaXplZF9wcmVkaWN0aW9uJTVEJTBBKSUwQXdlcg==",highlighted:`reference = <span class="hljs-string">&quot;HE TELLS US THAT AT THIS FESTIVE SEASON OF THE YEAR WITH CHRISTMAS AND ROAST BEEF LOOMING BEFORE US SIMILES DRAWN FROM EATING AND ITS RESULTS OCCUR MOST READILY TO THE MIND&quot;</span>
normalized_referece = normalizer(reference)

wer = wer_metric.compute(
    references=[normalized_referece], predictions=[normalized_prediction]
)
wer`,wrap:!1}}),Ie=new w({props:{code:"MC4wNjI1",highlighted:'<span class="hljs-attribute">0</span>.<span class="hljs-number">0625</span>',wrap:!1}}),_e=new C({props:{title:"Putting it all together",local:"putting-it-all-together",headingTag:"h2"}}),He=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBaW1wb3J0JTIwdG9yY2glMEElMEFpZiUyMHRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCklM0ElMEElMjAlMjAlMjAlMjBkZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTNBMCUyMiUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTIwJTNEJTIwdG9yY2guZmxvYXQxNiUwQWVsc2UlM0ElMEElMjAlMjAlMjAlMjBkZXZpY2UlMjAlM0QlMjAlMjJjcHUlMjIlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUyMCUzRCUyMHRvcmNoLmZsb2F0MzIlMEElMEFwaXBlJTIwJTNEJTIwcGlwZWxpbmUoJTBBJTIwJTIwJTIwJTIwJTIyYXV0b21hdGljLXNwZWVjaC1yZWNvZ25pdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEJTIyb3BlbmFpJTJGd2hpc3Blci1zbWFsbCUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2hfZHR5cGUlMkMlMEElMjAlMjAlMjAlMjBkZXZpY2UlM0RkZXZpY2UlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
<span class="hljs-keyword">import</span> torch

<span class="hljs-keyword">if</span> torch.cuda.is_available():
    device = <span class="hljs-string">&quot;cuda:0&quot;</span>
    torch_dtype = torch.float16
<span class="hljs-keyword">else</span>:
    device = <span class="hljs-string">&quot;cpu&quot;</span>
    torch_dtype = torch.float32

pipe = pipeline(
    <span class="hljs-string">&quot;automatic-speech-recognition&quot;</span>,
    model=<span class="hljs-string">&quot;openai/whisper-small&quot;</span>,
    torch_dtype=torch_dtype,
    device=device,
)`,wrap:!1}}),Fe=new w({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,wrap:!1}}),Ne=new w({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBY29tbW9uX3ZvaWNlX3Rlc3QlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTBBJTIwJTIwJTIwJTIwJTIybW96aWxsYS1mb3VuZGF0aW9uJTJGY29tbW9uX3ZvaWNlXzEzXzAlMjIlMkMlMjAlMjJkdiUyMiUyQyUyMHNwbGl0JTNEJTIydGVzdCUyMiUwQSk=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

common_voice_test = load_dataset(
    <span class="hljs-string">&quot;mozilla-foundation/common_voice_13_0&quot;</span>, <span class="hljs-string">&quot;dv&quot;</span>, split=<span class="hljs-string">&quot;test&quot;</span>
)`,wrap:!1}}),J=new dn({props:{$$slots:{default:[jn]},$$scope:{ctx:lt}}}),Ae=new w({props:{code:"ZnJvbSUyMHRxZG0lMjBpbXBvcnQlMjB0cWRtJTBBZnJvbSUyMHRyYW5zZm9ybWVycy5waXBlbGluZXMucHRfdXRpbHMlMjBpbXBvcnQlMjBLZXlEYXRhc2V0JTBBJTBBYWxsX3ByZWRpY3Rpb25zJTIwJTNEJTIwJTVCJTVEJTBBJTBBJTIzJTIwcnVuJTIwc3RyZWFtZWQlMjBpbmZlcmVuY2UlMEFmb3IlMjBwcmVkaWN0aW9uJTIwaW4lMjB0cWRtKCUwQSUyMCUyMCUyMCUyMHBpcGUoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwS2V5RGF0YXNldChjb21tb25fdm9pY2VfdGVzdCUyQyUyMCUyMmF1ZGlvJTIyKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1heF9uZXdfdG9rZW5zJTNEMTI4JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ2VuZXJhdGVfa3dhcmdzJTNEJTdCJTIydGFzayUyMiUzQSUyMCUyMnRyYW5zY3JpYmUlMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNEMzIlMkMlMEElMjAlMjAlMjAlMjApJTJDJTBBJTIwJTIwJTIwJTIwdG90YWwlM0RsZW4oY29tbW9uX3ZvaWNlX3Rlc3QpJTJDJTBBKSUzQSUwQSUyMCUyMCUyMCUyMGFsbF9wcmVkaWN0aW9ucy5hcHBlbmQocHJlZGljdGlvbiU1QiUyMnRleHQlMjIlNUQp",highlighted:`<span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">from</span> transformers.pipelines.pt_utils <span class="hljs-keyword">import</span> KeyDataset

all_predictions = []

<span class="hljs-comment"># run streamed inference</span>
<span class="hljs-keyword">for</span> prediction <span class="hljs-keyword">in</span> tqdm(
    pipe(
        KeyDataset(common_voice_test, <span class="hljs-string">&quot;audio&quot;</span>),
        max_new_tokens=<span class="hljs-number">128</span>,
        generate_kwargs={<span class="hljs-string">&quot;task&quot;</span>: <span class="hljs-string">&quot;transcribe&quot;</span>},
        batch_size=<span class="hljs-number">32</span>,
    ),
    total=<span class="hljs-built_in">len</span>(common_voice_test),
):
    all_predictions.append(prediction[<span class="hljs-string">&quot;text&quot;</span>])`,wrap:!1}}),j=new dn({props:{$$slots:{default:[xn]},$$scope:{ctx:lt}}}),Qe=new w({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwbG9hZCUwQSUwQXdlcl9tZXRyaWMlMjAlM0QlMjBsb2FkKCUyMndlciUyMiklMEElMEF3ZXJfb3J0aG8lMjAlM0QlMjAxMDAlMjAqJTIwd2VyX21ldHJpYy5jb21wdXRlKCUwQSUyMCUyMCUyMCUyMHJlZmVyZW5jZXMlM0Rjb21tb25fdm9pY2VfdGVzdCU1QiUyMnNlbnRlbmNlJTIyJTVEJTJDJTIwcHJlZGljdGlvbnMlM0RhbGxfcHJlZGljdGlvbnMlMEEpJTBBd2VyX29ydGhv",highlighted:`<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> load

wer_metric = load(<span class="hljs-string">&quot;wer&quot;</span>)

wer_ortho = <span class="hljs-number">100</span> * wer_metric.compute(
    references=common_voice_test[<span class="hljs-string">&quot;sentence&quot;</span>], predictions=all_predictions
)
wer_ortho`,wrap:!1}}),Pe=new w({props:{code:"MTY3LjI5NTc3MjY4NjEyMDIy",highlighted:'<span class="hljs-attribute">167</span>.<span class="hljs-number">29577268612022</span>',wrap:!1}}),Oe=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5tb2RlbHMud2hpc3Blci5lbmdsaXNoX25vcm1hbGl6ZXIlMjBpbXBvcnQlMjBCYXNpY1RleHROb3JtYWxpemVyJTBBJTBBbm9ybWFsaXplciUyMCUzRCUyMEJhc2ljVGV4dE5vcm1hbGl6ZXIoKSUwQSUwQSUyMyUyMGNvbXB1dGUlMjBub3JtYWxpc2VkJTIwV0VSJTBBYWxsX3ByZWRpY3Rpb25zX25vcm0lMjAlM0QlMjAlNUJub3JtYWxpemVyKHByZWQpJTIwZm9yJTIwcHJlZCUyMGluJTIwYWxsX3ByZWRpY3Rpb25zJTVEJTBBYWxsX3JlZmVyZW5jZXNfbm9ybSUyMCUzRCUyMCU1Qm5vcm1hbGl6ZXIobGFiZWwpJTIwZm9yJTIwbGFiZWwlMjBpbiUyMGNvbW1vbl92b2ljZV90ZXN0JTVCJTIyc2VudGVuY2UlMjIlNUQlNUQlMEElMEElMjMlMjBmaWx0ZXJpbmclMjBzdGVwJTIwdG8lMjBvbmx5JTIwZXZhbHVhdGUlMjB0aGUlMjBzYW1wbGVzJTIwdGhhdCUyMGNvcnJlc3BvbmQlMjB0byUyMG5vbi16ZXJvJTIwcmVmZXJlbmNlcyUwQWFsbF9wcmVkaWN0aW9uc19ub3JtJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwYWxsX3ByZWRpY3Rpb25zX25vcm0lNUJpJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwaSUyMGluJTIwcmFuZ2UobGVuKGFsbF9wcmVkaWN0aW9uc19ub3JtKSklMEElMjAlMjAlMjAlMjBpZiUyMGxlbihhbGxfcmVmZXJlbmNlc19ub3JtJTVCaSU1RCklMjAlM0UlMjAwJTBBJTVEJTBBYWxsX3JlZmVyZW5jZXNfbm9ybSUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMGFsbF9yZWZlcmVuY2VzX25vcm0lNUJpJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwaSUyMGluJTIwcmFuZ2UobGVuKGFsbF9yZWZlcmVuY2VzX25vcm0pKSUwQSUyMCUyMCUyMCUyMGlmJTIwbGVuKGFsbF9yZWZlcmVuY2VzX25vcm0lNUJpJTVEKSUyMCUzRSUyMDAlMEElNUQlMEElMEF3ZXIlMjAlM0QlMjAxMDAlMjAqJTIwd2VyX21ldHJpYy5jb21wdXRlKCUwQSUyMCUyMCUyMCUyMHJlZmVyZW5jZXMlM0RhbGxfcmVmZXJlbmNlc19ub3JtJTJDJTIwcHJlZGljdGlvbnMlM0RhbGxfcHJlZGljdGlvbnNfbm9ybSUwQSklMEElMEF3ZXI=",highlighted:`<span class="hljs-keyword">from</span> transformers.models.whisper.english_normalizer <span class="hljs-keyword">import</span> BasicTextNormalizer

normalizer = BasicTextNormalizer()

<span class="hljs-comment"># compute normalised WER</span>
all_predictions_norm = [normalizer(pred) <span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> all_predictions]
all_references_norm = [normalizer(label) <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> common_voice_test[<span class="hljs-string">&quot;sentence&quot;</span>]]

<span class="hljs-comment"># filtering step to only evaluate the samples that correspond to non-zero references</span>
all_predictions_norm = [
    all_predictions_norm[i]
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(all_predictions_norm))
    <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(all_references_norm[i]) &gt; <span class="hljs-number">0</span>
]
all_references_norm = [
    all_references_norm[i]
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(all_references_norm))
    <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(all_references_norm[i]) &gt; <span class="hljs-number">0</span>
]

wer = <span class="hljs-number">100</span> * wer_metric.compute(
    references=all_references_norm, predictions=all_predictions_norm
)

wer`,wrap:!1}}),et=new w({props:{code:"MTI1LjY5ODA5MDg5OTYwNzA3",highlighted:'<span class="hljs-attribute">125</span>.<span class="hljs-number">69809089960707</span>',wrap:!1}}),at=new Jn({props:{source:"https://github.com/huggingface/audio-transformers-course/blob/main/chapters/en/chapter5/evaluation.mdx"}}),{c(){u=i("meta"),f=n(),v=i("p"),ht=n(),o($.$$.fragment),dt=n(),o(R.$$.fragment),yt=n(),W=i("p"),W.innerHTML=pa,ut=n(),E=i("p"),E.textContent=oa,wt=n(),I=i("ol"),I.innerHTML=ma,ft=n(),k=i("p"),k.innerHTML=ca,Mt=n(),B=i("p"),B.innerHTML=ha,gt=n(),o(_.$$.fragment),vt=n(),Z=i("p"),Z.textContent=da,bt=n(),o(G.$$.fragment),Tt=n(),H=i("p"),H.textContent=ya,Ut=n(),o(V.$$.fragment),Jt=n(),S=i("p"),S.innerHTML=ua,jt=n(),F=i("table"),F.innerHTML=wa,xt=n(),X=i("p"),X.textContent=fa,Ct=n(),N=i("ul"),N.innerHTML=Ma,$t=n(),z=i("p"),ea=T(`This gives 2 errors in total. To get our error rate, we divide the number of errors by the total number of words in our
reference (N), which for this example is 6:
`),Rt=new Os(!1),Wt=n(),A=i("p"),A.textContent=ga,Et=n(),Y=i("p"),Y.innerHTML=va,It=n(),Q=i("p"),Q.textContent=ba,kt=n(),o(L.$$.fragment),Bt=n(),P=i("p"),P.textContent=Ta,_t=n(),o(q.$$.fragment),Zt=n(),D=i("p"),D.innerHTML=Ua,Gt=n(),o(O.$$.fragment),Ht=n(),K=i("p"),K.textContent=Ja,Vt=n(),ee=i("p"),ee.textContent=ja,St=n(),o(te.$$.fragment),Ft=n(),b=i("p"),ta=T("While WER measures the accuracy of transcriptions, the "),it=i("em"),it.textContent=xa,sa=T(` measures the speed of an ASR system.
RTFx is the inverse ratio of processing time to audio duration:
`),Xt=new Os(!1),Nt=n(),se=i("p"),se.textContent=Ca,zt=n(),ae=i("p"),ae.textContent=$a,At=n(),ne=i("ul"),ne.innerHTML=Ra,Yt=n(),le=i("p"),le.textContent=Wa,Qt=n(),ie=i("ul"),ie.innerHTML=Ea,Lt=n(),re=i("p"),re.textContent=Ia,Pt=n(),o(pe.$$.fragment),qt=n(),M=i("p"),aa=T("We can flip the WER around to give us a metric where "),rt=i("em"),rt.textContent=ka,na=T(`. Rather than measuring the word error rate,
we can measure the `),pt=i("em"),pt.textContent=Ba,la=T(` of our system:
`),Dt=new Os(!1),Ot=n(),oe=i("p"),oe.textContent=_a,Kt=n(),o(me.$$.fragment),es=n(),ce=i("p"),ce.innerHTML=Za,ts=n(),he=i("table"),he.innerHTML=Ga,ss=n(),de=i("p"),de.textContent=Ha,as=n(),ye=i("p"),ia=T(`In our example, we have 1 character substitution, 0 insertions, and 4 deletions. In total, we have 22 characters. So, our CER is:
`),ns=new Os(!1),ls=n(),ue=i("p"),ue.textContent=Va,is=n(),o(we.$$.fragment),rs=n(),fe=i("p"),fe.textContent=Sa,ps=n(),Me=i("p"),Me.textContent=Fa,os=n(),o(ge.$$.fragment),ms=n(),ve=i("p"),ve.innerHTML=Xa,cs=n(),be=i("p"),be.innerHTML=Na,hs=n(),o(Te.$$.fragment),ds=n(),Ue=i("p"),Ue.textContent=za,ys=n(),Je=i("p"),Je.textContent=Aa,us=n(),je=i("p"),je.textContent=Ya,ws=n(),o(xe.$$.fragment),fs=n(),Ce=i("p"),Ce.innerHTML=Qa,Ms=n(),o($e.$$.fragment),gs=n(),Re=i("p"),Re.textContent=La,vs=n(),o(We.$$.fragment),bs=n(),Ee=i("p"),Ee.innerHTML=Pa,Ts=n(),o(Ie.$$.fragment),Us=n(),ke=i("p"),ke.textContent=qa,Js=n(),Be=i("p"),Be.textContent=Da,js=n(),o(_e.$$.fragment),xs=n(),Ze=i("p"),Ze.innerHTML=Oa,Cs=n(),Ge=i("p"),Ge.innerHTML=Ka,$s=n(),o(He.$$.fragment),Rs=n(),Ve=i("p"),Ve.innerHTML=en,Ws=n(),Se=i("p"),Se.innerHTML=tn,Es=n(),o(Fe.$$.fragment),Is=n(),Xe=i("p"),Xe.textContent=sn,ks=n(),o(Ne.$$.fragment),Bs=n(),o(J.$$.fragment),_s=n(),ze=i("p"),ze.innerHTML=an,Zs=n(),o(Ae.$$.fragment),Gs=n(),o(j.$$.fragment),Hs=n(),Ye=i("p"),Ye.textContent=nn,Vs=n(),o(Qe.$$.fragment),Ss=n(),Le=i("p"),Le.innerHTML=ln,Fs=n(),o(Pe.$$.fragment),Xs=n(),qe=i("p"),qe.textContent=rn,Ns=n(),De=i("p"),De.textContent=pn,zs=n(),o(Oe.$$.fragment),As=n(),Ke=i("p"),Ke.innerHTML=on,Ys=n(),o(et.$$.fragment),Qs=n(),tt=i("p"),tt.textContent=mn,Ls=n(),st=i("p"),st.textContent=cn,Ps=n(),o(at.$$.fragment),qs=n(),ct=i("p"),this.h()},l(e){const t=Tn("svelte-u9bgzb",document.head);u=r(t,"META",{name:!0,content:!0}),t.forEach(s),f=l(e),v=r(e,"P",{}),mt(v).forEach(s),ht=l(e),m($.$$.fragment,e),dt=l(e),m(R.$$.fragment,e),yt=l(e),W=r(e,"P",{"data-svelte-h":!0}),p(W)!=="svelte-bhh0s4"&&(W.innerHTML=pa),ut=l(e),E=r(e,"P",{"data-svelte-h":!0}),p(E)!=="svelte-14gi20s"&&(E.textContent=oa),wt=l(e),I=r(e,"OL",{"data-svelte-h":!0}),p(I)!=="svelte-1rakz49"&&(I.innerHTML=ma),ft=l(e),k=r(e,"P",{"data-svelte-h":!0}),p(k)!=="svelte-1u4b8et"&&(k.innerHTML=ca),Mt=l(e),B=r(e,"P",{"data-svelte-h":!0}),p(B)!=="svelte-1hnsgur"&&(B.innerHTML=ha),gt=l(e),m(_.$$.fragment,e),vt=l(e),Z=r(e,"P",{"data-svelte-h":!0}),p(Z)!=="svelte-1oavasl"&&(Z.textContent=da),bt=l(e),m(G.$$.fragment,e),Tt=l(e),H=r(e,"P",{"data-svelte-h":!0}),p(H)!=="svelte-meydoa"&&(H.textContent=ya),Ut=l(e),m(V.$$.fragment,e),Jt=l(e),S=r(e,"P",{"data-svelte-h":!0}),p(S)!=="svelte-etln9n"&&(S.innerHTML=ua),jt=l(e),F=r(e,"TABLE",{"data-svelte-h":!0}),p(F)!=="svelte-1p0gpk"&&(F.innerHTML=wa),xt=l(e),X=r(e,"P",{"data-svelte-h":!0}),p(X)!=="svelte-83gpva"&&(X.textContent=fa),Ct=l(e),N=r(e,"UL",{"data-svelte-h":!0}),p(N)!=="svelte-rr6mcc"&&(N.innerHTML=Ma),$t=l(e),z=r(e,"P",{});var ot=mt(z);ea=U(ot,`This gives 2 errors in total. To get our error rate, we divide the number of errors by the total number of words in our
reference (N), which for this example is 6:
`),Rt=Ks(ot,!1),ot.forEach(s),Wt=l(e),A=r(e,"P",{"data-svelte-h":!0}),p(A)!=="svelte-1x9kpft"&&(A.textContent=ga),Et=l(e),Y=r(e,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-1j0jh50"&&(Y.innerHTML=va),It=l(e),Q=r(e,"P",{"data-svelte-h":!0}),p(Q)!=="svelte-1ia5odq"&&(Q.textContent=ba),kt=l(e),m(L.$$.fragment,e),Bt=l(e),P=r(e,"P",{"data-svelte-h":!0}),p(P)!=="svelte-vl6c9u"&&(P.textContent=Ta),_t=l(e),m(q.$$.fragment,e),Zt=l(e),D=r(e,"P",{"data-svelte-h":!0}),p(D)!=="svelte-j2cy7r"&&(D.innerHTML=Ua),Gt=l(e),m(O.$$.fragment,e),Ht=l(e),K=r(e,"P",{"data-svelte-h":!0}),p(K)!=="svelte-ppsr8o"&&(K.textContent=Ja),Vt=l(e),ee=r(e,"P",{"data-svelte-h":!0}),p(ee)!=="svelte-uoepkq"&&(ee.textContent=ja),St=l(e),m(te.$$.fragment,e),Ft=l(e),b=r(e,"P",{});var x=mt(b);ta=U(x,"While WER measures the accuracy of transcriptions, the "),it=r(x,"EM",{"data-svelte-h":!0}),p(it)!=="svelte-1aavw8y"&&(it.textContent=xa),sa=U(x,` measures the speed of an ASR system.
RTFx is the inverse ratio of processing time to audio duration:
`),Xt=Ks(x,!1),x.forEach(s),Nt=l(e),se=r(e,"P",{"data-svelte-h":!0}),p(se)!=="svelte-1sxhqtn"&&(se.textContent=Ca),zt=l(e),ae=r(e,"P",{"data-svelte-h":!0}),p(ae)!=="svelte-omecrj"&&(ae.textContent=$a),At=l(e),ne=r(e,"UL",{"data-svelte-h":!0}),p(ne)!=="svelte-gxj3x1"&&(ne.innerHTML=Ra),Yt=l(e),le=r(e,"P",{"data-svelte-h":!0}),p(le)!=="svelte-1mi1zdc"&&(le.textContent=Wa),Qt=l(e),ie=r(e,"UL",{"data-svelte-h":!0}),p(ie)!=="svelte-17erncr"&&(ie.innerHTML=Ea),Lt=l(e),re=r(e,"P",{"data-svelte-h":!0}),p(re)!=="svelte-19pgx6v"&&(re.textContent=Ia),Pt=l(e),m(pe.$$.fragment,e),qt=l(e),M=r(e,"P",{});var nt=mt(M);aa=U(nt,"We can flip the WER around to give us a metric where "),rt=r(nt,"EM",{"data-svelte-h":!0}),p(rt)!=="svelte-efgwkj"&&(rt.textContent=ka),na=U(nt,`. Rather than measuring the word error rate,
we can measure the `),pt=r(nt,"EM",{"data-svelte-h":!0}),p(pt)!=="svelte-wk6fn0"&&(pt.textContent=Ba),la=U(nt,` of our system:
`),Dt=Ks(nt,!1),nt.forEach(s),Ot=l(e),oe=r(e,"P",{"data-svelte-h":!0}),p(oe)!=="svelte-14zvp5d"&&(oe.textContent=_a),Kt=l(e),m(me.$$.fragment,e),es=l(e),ce=r(e,"P",{"data-svelte-h":!0}),p(ce)!=="svelte-cssi9y"&&(ce.innerHTML=Za),ts=l(e),he=r(e,"TABLE",{"data-svelte-h":!0}),p(he)!=="svelte-q7muq3"&&(he.innerHTML=Ga),ss=l(e),de=r(e,"P",{"data-svelte-h":!0}),p(de)!=="svelte-jzeweo"&&(de.textContent=Ha),as=l(e),ye=r(e,"P",{});var ra=mt(ye);ia=U(ra,`In our example, we have 1 character substitution, 0 insertions, and 4 deletions. In total, we have 22 characters. So, our CER is:
`),ns=Ks(ra,!1),ra.forEach(s),ls=l(e),ue=r(e,"P",{"data-svelte-h":!0}),p(ue)!=="svelte-bl9y86"&&(ue.textContent=Va),is=l(e),m(we.$$.fragment,e),rs=l(e),fe=r(e,"P",{"data-svelte-h":!0}),p(fe)!=="svelte-1gur6wt"&&(fe.textContent=Sa),ps=l(e),Me=r(e,"P",{"data-svelte-h":!0}),p(Me)!=="svelte-sm17uh"&&(Me.textContent=Fa),os=l(e),m(ge.$$.fragment,e),ms=l(e),ve=r(e,"P",{"data-svelte-h":!0}),p(ve)!=="svelte-npx9p1"&&(ve.innerHTML=Xa),cs=l(e),be=r(e,"P",{"data-svelte-h":!0}),p(be)!=="svelte-8y51ri"&&(be.innerHTML=Na),hs=l(e),m(Te.$$.fragment,e),ds=l(e),Ue=r(e,"P",{"data-svelte-h":!0}),p(Ue)!=="svelte-inhmc"&&(Ue.textContent=za),ys=l(e),Je=r(e,"P",{"data-svelte-h":!0}),p(Je)!=="svelte-5l9h63"&&(Je.textContent=Aa),us=l(e),je=r(e,"P",{"data-svelte-h":!0}),p(je)!=="svelte-ht49w6"&&(je.textContent=Ya),ws=l(e),m(xe.$$.fragment,e),fs=l(e),Ce=r(e,"P",{"data-svelte-h":!0}),p(Ce)!=="svelte-1mvdyro"&&(Ce.innerHTML=Qa),Ms=l(e),m($e.$$.fragment,e),gs=l(e),Re=r(e,"P",{"data-svelte-h":!0}),p(Re)!=="svelte-kbg2hm"&&(Re.textContent=La),vs=l(e),m(We.$$.fragment,e),bs=l(e),Ee=r(e,"P",{"data-svelte-h":!0}),p(Ee)!=="svelte-1mvdyro"&&(Ee.innerHTML=Pa),Ts=l(e),m(Ie.$$.fragment,e),Us=l(e),ke=r(e,"P",{"data-svelte-h":!0}),p(ke)!=="svelte-1mk0tzt"&&(ke.textContent=qa),Js=l(e),Be=r(e,"P",{"data-svelte-h":!0}),p(Be)!=="svelte-pot7ah"&&(Be.textContent=Da),js=l(e),m(_e.$$.fragment,e),xs=l(e),Ze=r(e,"P",{"data-svelte-h":!0}),p(Ze)!=="svelte-a0jkyj"&&(Ze.innerHTML=Oa),Cs=l(e),Ge=r(e,"P",{"data-svelte-h":!0}),p(Ge)!=="svelte-657gq0"&&(Ge.innerHTML=Ka),$s=l(e),m(He.$$.fragment,e),Rs=l(e),Ve=r(e,"P",{"data-svelte-h":!0}),p(Ve)!=="svelte-1q149qh"&&(Ve.innerHTML=en),Ws=l(e),Se=r(e,"P",{"data-svelte-h":!0}),p(Se)!=="svelte-dngs6s"&&(Se.innerHTML=tn),Es=l(e),m(Fe.$$.fragment,e),Is=l(e),Xe=r(e,"P",{"data-svelte-h":!0}),p(Xe)!=="svelte-wi5uul"&&(Xe.textContent=sn),ks=l(e),m(Ne.$$.fragment,e),Bs=l(e),m(J.$$.fragment,e),_s=l(e),ze=r(e,"P",{"data-svelte-h":!0}),p(ze)!=="svelte-y5ayq9"&&(ze.innerHTML=an),Zs=l(e),m(Ae.$$.fragment,e),Gs=l(e),m(j.$$.fragment,e),Hs=l(e),Ye=r(e,"P",{"data-svelte-h":!0}),p(Ye)!=="svelte-13bou4y"&&(Ye.textContent=nn),Vs=l(e),m(Qe.$$.fragment,e),Ss=l(e),Le=r(e,"P",{"data-svelte-h":!0}),p(Le)!=="svelte-1mvdyro"&&(Le.innerHTML=ln),Fs=l(e),m(Pe.$$.fragment,e),Xs=l(e),qe=r(e,"P",{"data-svelte-h":!0}),p(qe)!=="svelte-dzd4kp"&&(qe.textContent=rn),Ns=l(e),De=r(e,"P",{"data-svelte-h":!0}),p(De)!=="svelte-ltuvl8"&&(De.textContent=pn),zs=l(e),m(Oe.$$.fragment,e),As=l(e),Ke=r(e,"P",{"data-svelte-h":!0}),p(Ke)!=="svelte-1mvdyro"&&(Ke.innerHTML=on),Ys=l(e),m(et.$$.fragment,e),Qs=l(e),tt=r(e,"P",{"data-svelte-h":!0}),p(tt)!=="svelte-sppy6n"&&(tt.textContent=mn),Ls=l(e),st=r(e,"P",{"data-svelte-h":!0}),p(st)!=="svelte-pm696u"&&(st.textContent=cn),Ps=l(e),m(at.$$.fragment,e),qs=l(e),ct=r(e,"P",{}),mt(ct).forEach(s),this.h()},h(){hn(u,"name","hf:doc:metadata"),hn(u,"content",$n),Rt.a=null,Xt.a=null,Dt.a=null,ns.a=null},m(e,t){g(document.head,u),a(e,f,t),a(e,v,t),a(e,ht,t),c($,e,t),a(e,dt,t),c(R,e,t),a(e,yt,t),a(e,W,t),a(e,ut,t),a(e,E,t),a(e,wt,t),a(e,I,t),a(e,ft,t),a(e,k,t),a(e,Mt,t),a(e,B,t),a(e,gt,t),c(_,e,t),a(e,vt,t),a(e,Z,t),a(e,bt,t),c(G,e,t),a(e,Tt,t),a(e,H,t),a(e,Ut,t),c(V,e,t),a(e,Jt,t),a(e,S,t),a(e,jt,t),a(e,F,t),a(e,xt,t),a(e,X,t),a(e,Ct,t),a(e,N,t),a(e,$t,t),a(e,z,t),g(z,ea),Rt.m(yn,z),a(e,Wt,t),a(e,A,t),a(e,Et,t),a(e,Y,t),a(e,It,t),a(e,Q,t),a(e,kt,t),c(L,e,t),a(e,Bt,t),a(e,P,t),a(e,_t,t),c(q,e,t),a(e,Zt,t),a(e,D,t),a(e,Gt,t),c(O,e,t),a(e,Ht,t),a(e,K,t),a(e,Vt,t),a(e,ee,t),a(e,St,t),c(te,e,t),a(e,Ft,t),a(e,b,t),g(b,ta),g(b,it),g(b,sa),Xt.m(un,b),a(e,Nt,t),a(e,se,t),a(e,zt,t),a(e,ae,t),a(e,At,t),a(e,ne,t),a(e,Yt,t),a(e,le,t),a(e,Qt,t),a(e,ie,t),a(e,Lt,t),a(e,re,t),a(e,Pt,t),c(pe,e,t),a(e,qt,t),a(e,M,t),g(M,aa),g(M,rt),g(M,na),g(M,pt),g(M,la),Dt.m(wn,M),a(e,Ot,t),a(e,oe,t),a(e,Kt,t),c(me,e,t),a(e,es,t),a(e,ce,t),a(e,ts,t),a(e,he,t),a(e,ss,t),a(e,de,t),a(e,as,t),a(e,ye,t),g(ye,ia),ns.m(fn,ye),a(e,ls,t),a(e,ue,t),a(e,is,t),c(we,e,t),a(e,rs,t),a(e,fe,t),a(e,ps,t),a(e,Me,t),a(e,os,t),c(ge,e,t),a(e,ms,t),a(e,ve,t),a(e,cs,t),a(e,be,t),a(e,hs,t),c(Te,e,t),a(e,ds,t),a(e,Ue,t),a(e,ys,t),a(e,Je,t),a(e,us,t),a(e,je,t),a(e,ws,t),c(xe,e,t),a(e,fs,t),a(e,Ce,t),a(e,Ms,t),c($e,e,t),a(e,gs,t),a(e,Re,t),a(e,vs,t),c(We,e,t),a(e,bs,t),a(e,Ee,t),a(e,Ts,t),c(Ie,e,t),a(e,Us,t),a(e,ke,t),a(e,Js,t),a(e,Be,t),a(e,js,t),c(_e,e,t),a(e,xs,t),a(e,Ze,t),a(e,Cs,t),a(e,Ge,t),a(e,$s,t),c(He,e,t),a(e,Rs,t),a(e,Ve,t),a(e,Ws,t),a(e,Se,t),a(e,Es,t),c(Fe,e,t),a(e,Is,t),a(e,Xe,t),a(e,ks,t),c(Ne,e,t),a(e,Bs,t),c(J,e,t),a(e,_s,t),a(e,ze,t),a(e,Zs,t),c(Ae,e,t),a(e,Gs,t),c(j,e,t),a(e,Hs,t),a(e,Ye,t),a(e,Vs,t),c(Qe,e,t),a(e,Ss,t),a(e,Le,t),a(e,Fs,t),c(Pe,e,t),a(e,Xs,t),a(e,qe,t),a(e,Ns,t),a(e,De,t),a(e,zs,t),c(Oe,e,t),a(e,As,t),a(e,Ke,t),a(e,Ys,t),c(et,e,t),a(e,Qs,t),a(e,tt,t),a(e,Ls,t),a(e,st,t),a(e,Ps,t),c(at,e,t),a(e,qs,t),a(e,ct,t),Ds=!0},p(e,[t]){const ot={};t&2&&(ot.$$scope={dirty:t,ctx:e}),J.$set(ot);const x={};t&2&&(x.$$scope={dirty:t,ctx:e}),j.$set(x)},i(e){Ds||(h($.$$.fragment,e),h(R.$$.fragment,e),h(_.$$.fragment,e),h(G.$$.fragment,e),h(V.$$.fragment,e),h(L.$$.fragment,e),h(q.$$.fragment,e),h(O.$$.fragment,e),h(te.$$.fragment,e),h(pe.$$.fragment,e),h(me.$$.fragment,e),h(we.$$.fragment,e),h(ge.$$.fragment,e),h(Te.$$.fragment,e),h(xe.$$.fragment,e),h($e.$$.fragment,e),h(We.$$.fragment,e),h(Ie.$$.fragment,e),h(_e.$$.fragment,e),h(He.$$.fragment,e),h(Fe.$$.fragment,e),h(Ne.$$.fragment,e),h(J.$$.fragment,e),h(Ae.$$.fragment,e),h(j.$$.fragment,e),h(Qe.$$.fragment,e),h(Pe.$$.fragment,e),h(Oe.$$.fragment,e),h(et.$$.fragment,e),h(at.$$.fragment,e),Ds=!0)},o(e){d($.$$.fragment,e),d(R.$$.fragment,e),d(_.$$.fragment,e),d(G.$$.fragment,e),d(V.$$.fragment,e),d(L.$$.fragment,e),d(q.$$.fragment,e),d(O.$$.fragment,e),d(te.$$.fragment,e),d(pe.$$.fragment,e),d(me.$$.fragment,e),d(we.$$.fragment,e),d(ge.$$.fragment,e),d(Te.$$.fragment,e),d(xe.$$.fragment,e),d($e.$$.fragment,e),d(We.$$.fragment,e),d(Ie.$$.fragment,e),d(_e.$$.fragment,e),d(He.$$.fragment,e),d(Fe.$$.fragment,e),d(Ne.$$.fragment,e),d(J.$$.fragment,e),d(Ae.$$.fragment,e),d(j.$$.fragment,e),d(Qe.$$.fragment,e),d(Pe.$$.fragment,e),d(Oe.$$.fragment,e),d(et.$$.fragment,e),d(at.$$.fragment,e),Ds=!1},d(e){e&&(s(f),s(v),s(ht),s(dt),s(yt),s(W),s(ut),s(E),s(wt),s(I),s(ft),s(k),s(Mt),s(B),s(gt),s(vt),s(Z),s(bt),s(Tt),s(H),s(Ut),s(Jt),s(S),s(jt),s(F),s(xt),s(X),s(Ct),s(N),s($t),s(z),s(Wt),s(A),s(Et),s(Y),s(It),s(Q),s(kt),s(Bt),s(P),s(_t),s(Zt),s(D),s(Gt),s(Ht),s(K),s(Vt),s(ee),s(St),s(Ft),s(b),s(Nt),s(se),s(zt),s(ae),s(At),s(ne),s(Yt),s(le),s(Qt),s(ie),s(Lt),s(re),s(Pt),s(qt),s(M),s(Ot),s(oe),s(Kt),s(es),s(ce),s(ts),s(he),s(ss),s(de),s(as),s(ye),s(ls),s(ue),s(is),s(rs),s(fe),s(ps),s(Me),s(os),s(ms),s(ve),s(cs),s(be),s(hs),s(ds),s(Ue),s(ys),s(Je),s(us),s(je),s(ws),s(fs),s(Ce),s(Ms),s(gs),s(Re),s(vs),s(bs),s(Ee),s(Ts),s(Us),s(ke),s(Js),s(Be),s(js),s(xs),s(Ze),s(Cs),s(Ge),s($s),s(Rs),s(Ve),s(Ws),s(Se),s(Es),s(Is),s(Xe),s(ks),s(Bs),s(_s),s(ze),s(Zs),s(Gs),s(Hs),s(Ye),s(Vs),s(Ss),s(Le),s(Fs),s(Xs),s(qe),s(Ns),s(De),s(zs),s(As),s(Ke),s(Ys),s(Qs),s(tt),s(Ls),s(st),s(Ps),s(qs),s(ct)),s(u),y($,e),y(R,e),y(_,e),y(G,e),y(V,e),y(L,e),y(q,e),y(O,e),y(te,e),y(pe,e),y(me,e),y(we,e),y(ge,e),y(Te,e),y(xe,e),y($e,e),y(We,e),y(Ie,e),y(_e,e),y(He,e),y(Fe,e),y(Ne,e),y(J,e),y(Ae,e),y(j,e),y(Qe,e),y(Pe,e),y(Oe,e),y(et,e),y(at,e)}}}const $n='{"title":"Evaluation metrics for ASR","local":"evaluation-metrics-for-asr","sections":[{"title":"Word Error Rate","local":"word-error-rate","sections":[],"depth":2},{"title":"Inverse Real-Time Factor (RTFx)","local":"inverse-real-time-factor-rtfx","sections":[],"depth":2},{"title":"Word Accuracy","local":"word-accuracy","sections":[],"depth":2},{"title":"Character Error Rate","local":"character-error-rate","sections":[],"depth":2},{"title":"Which metric should I use?","local":"which-metric-should-i-use","sections":[],"depth":2},{"title":"Normalisation","local":"normalisation","sections":[],"depth":2},{"title":"Putting it all together","local":"putting-it-all-together","sections":[],"depth":2}],"depth":1}';function Rn(lt){return gn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class _n extends vn{constructor(u){super(),bn(this,u,Rn,Cn,Mn,{})}}export{_n as component};
