1 /********************************************************************
2  *                                                                  *
3  * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE.   *
4  * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS     *
5  * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
6  * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING.       *
7  *                                                                  *
8  * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001             *
9  * by the Xiph.Org Foundation https://xiph.org/                     *
10  *                                                                  *
11  ********************************************************************
12 
13  function: function calls to collect codebook metrics
14 
15  ********************************************************************/
16 
17 
18 #include <stdlib.h>
19 #include <unistd.h>
20 #include <math.h>
21 #include "bookutil.h"
22 
23 /* collect the following metrics:
24 
25    mean and mean squared amplitude
26    mean and mean squared error
27    mean and mean squared error (per sample) by entry
28    worst case fit by entry
29    entry cell size
30    hits by entry
31    total bits
32    total samples
33    (average bits per sample)*/
34 
35 
36 /* set up metrics */
37 
38 float meanamplitude_acc=0.f;
39 float meanamplitudesq_acc=0.f;
40 float meanerror_acc=0.f;
41 float meanerrorsq_acc=0.f;
42 
43 float **histogram=NULL;
44 float **histogram_error=NULL;
45 float **histogram_errorsq=NULL;
46 float **histogram_hi=NULL;
47 float **histogram_lo=NULL;
48 float bits=0.f;
49 float count=0.f;
50 
_now(codebook * c,int i)51 static float *_now(codebook *c, int i){
52   return c->valuelist+i*c->c->dim;
53 }
54 
55 int books=0;
56 
process_preprocess(codebook ** bs,char * basename)57 void process_preprocess(codebook **bs,char *basename){
58   int i;
59   while(bs[books])books++;
60 
61   if(books){
62     histogram=_ogg_calloc(books,sizeof(float *));
63     histogram_error=_ogg_calloc(books,sizeof(float *));
64     histogram_errorsq=_ogg_calloc(books,sizeof(float *));
65     histogram_hi=_ogg_calloc(books,sizeof(float *));
66     histogram_lo=_ogg_calloc(books,sizeof(float *));
67   }else{
68     fprintf(stderr,"Specify at least one codebook\n");
69     exit(1);
70   }
71 
72   for(i=0;i<books;i++){
73     codebook *b=bs[i];
74     histogram[i]=_ogg_calloc(b->entries,sizeof(float));
75     histogram_error[i]=_ogg_calloc(b->entries*b->dim,sizeof(float));
76     histogram_errorsq[i]=_ogg_calloc(b->entries*b->dim,sizeof(float));
77     histogram_hi[i]=_ogg_calloc(b->entries*b->dim,sizeof(float));
78     histogram_lo[i]=_ogg_calloc(b->entries*b->dim,sizeof(float));
79   }
80 }
81 
_dist(int el,float * a,float * b)82 static float _dist(int el,float *a, float *b){
83   int i;
84   float acc=0.f;
85   for(i=0;i<el;i++){
86     float val=(a[i]-b[i]);
87     acc+=val*val;
88   }
89   return acc;
90 }
91 
cell_spacing(codebook * c)92 void cell_spacing(codebook *c){
93   int j,k;
94   float min=-1.f,max=-1.f,mean=0.f,meansq=0.f;
95   long total=0;
96 
97   /* minimum, maximum, mean, ms cell spacing */
98   for(j=0;j<c->c->entries;j++){
99     if(c->c->lengthlist[j]>0){
100       float localmin=-1.;
101       for(k=0;k<c->c->entries;k++){
102         if(c->c->lengthlist[k]>0){
103           float this=_dist(c->c->dim,_now(c,j),_now(c,k));
104           if(j!=k &&
105              (localmin==-1 || this<localmin))
106             localmin=this;
107         }
108       }
109 
110       if(min==-1 || localmin<min)min=localmin;
111       if(max==-1 || localmin>max)max=localmin;
112       mean+=sqrt(localmin);
113       meansq+=localmin;
114       total++;
115     }
116   }
117 
118   fprintf(stderr,"\tminimum cell spacing (closest side): %g\n",sqrt(min));
119   fprintf(stderr,"\tmaximum cell spacing (closest side): %g\n",sqrt(max));
120   fprintf(stderr,"\tmean closest side spacing: %g\n",mean/total);
121   fprintf(stderr,"\tmean sq closest side spacing: %g\n",sqrt(meansq/total));
122 }
123 
process_postprocess(codebook ** bs,char * basename)124 void process_postprocess(codebook **bs,char *basename){
125   int i,k,book;
126   char *buffer=alloca(strlen(basename)+80);
127 
128   fprintf(stderr,"Done.  Processed %ld data points:\n\n",
129           (long)count);
130 
131   fprintf(stderr,"Global statistics:******************\n\n");
132 
133   fprintf(stderr,"\ttotal samples: %ld\n",(long)count);
134   fprintf(stderr,"\ttotal bits required to code: %ld\n",(long)bits);
135   fprintf(stderr,"\taverage bits per sample: %g\n\n",bits/count);
136 
137   fprintf(stderr,"\tmean sample amplitude: %g\n",
138           meanamplitude_acc/count);
139   fprintf(stderr,"\tmean squared sample amplitude: %g\n\n",
140           sqrt(meanamplitudesq_acc/count));
141 
142   fprintf(stderr,"\tmean code error: %g\n",
143           meanerror_acc/count);
144   fprintf(stderr,"\tmean squared code error: %g\n\n",
145           sqrt(meanerrorsq_acc/count));
146 
147   for(book=0;book<books;book++){
148     FILE *out;
149     codebook *b=bs[book];
150     int n=b->c->entries;
151     int dim=b->c->dim;
152 
153     fprintf(stderr,"Book %d statistics:------------------\n",book);
154 
155     cell_spacing(b);
156 
157     sprintf(buffer,"%s-%d-mse.m",basename,book);
158     out=fopen(buffer,"w");
159     if(!out){
160       fprintf(stderr,"Could not open file %s for writing\n",buffer);
161       exit(1);
162     }
163 
164     for(i=0;i<n;i++){
165       for(k=0;k<dim;k++){
166         fprintf(out,"%d, %g, %g\n",
167                 i*dim+k,(b->valuelist+i*dim)[k],
168                 sqrt((histogram_errorsq[book]+i*dim)[k]/histogram[book][i]));
169       }
170     }
171     fclose(out);
172 
173     sprintf(buffer,"%s-%d-me.m",basename,book);
174     out=fopen(buffer,"w");
175     if(!out){
176       fprintf(stderr,"Could not open file %s for writing\n",buffer);
177       exit(1);
178     }
179 
180     for(i=0;i<n;i++){
181       for(k=0;k<dim;k++){
182         fprintf(out,"%d, %g, %g\n",
183                 i*dim+k,(b->valuelist+i*dim)[k],
184                 (histogram_error[book]+i*dim)[k]/histogram[book][i]);
185       }
186     }
187     fclose(out);
188 
189     sprintf(buffer,"%s-%d-worst.m",basename,book);
190     out=fopen(buffer,"w");
191     if(!out){
192       fprintf(stderr,"Could not open file %s for writing\n",buffer);
193       exit(1);
194     }
195 
196     for(i=0;i<n;i++){
197       for(k=0;k<dim;k++){
198         fprintf(out,"%d, %g, %g, %g\n",
199                 i*dim+k,(b->valuelist+i*dim)[k],
200                 (b->valuelist+i*dim)[k]+(histogram_lo[book]+i*dim)[k],
201                 (b->valuelist+i*dim)[k]+(histogram_hi[book]+i*dim)[k]);
202       }
203     }
204     fclose(out);
205   }
206 }
207 
process_one(codebook * b,int book,float * a,int dim,int step,int addmul,float base)208 float process_one(codebook *b,int book,float *a,int dim,int step,int addmul,
209                    float base){
210   int j,entry;
211   float amplitude=0.f;
212 
213   if(book==0){
214     float last=base;
215     for(j=0;j<dim;j++){
216       amplitude=a[j*step]-(b->c->q_sequencep?last:0);
217       meanamplitude_acc+=fabs(amplitude);
218       meanamplitudesq_acc+=amplitude*amplitude;
219       count++;
220       last=a[j*step];
221     }
222   }
223 
224   if(b->c->q_sequencep){
225     float temp;
226     for(j=0;j<dim;j++){
227       temp=a[j*step];
228       a[j*step]-=base;
229     }
230     base=temp;
231   }
232 
233   entry=vorbis_book_besterror(b,a,step,addmul);
234 
235   if(entry==-1){
236     fprintf(stderr,"Internal error: _best returned -1.\n");
237     exit(1);
238   }
239 
240   histogram[book][entry]++;
241   bits+=vorbis_book_codelen(b,entry);
242 
243   for(j=0;j<dim;j++){
244     float error=a[j*step];
245 
246     if(book==books-1){
247       meanerror_acc+=fabs(error);
248       meanerrorsq_acc+=error*error;
249     }
250     histogram_errorsq[book][entry*dim+j]+=error*error;
251     histogram_error[book][entry*dim+j]+=fabs(error);
252     if(histogram[book][entry]==0 || histogram_hi[book][entry*dim+j]<error)
253       histogram_hi[book][entry*dim+j]=error;
254     if(histogram[book][entry]==0 || histogram_lo[book][entry*dim+j]>error)
255       histogram_lo[book][entry*dim+j]=error;
256   }
257   return base;
258 }
259 
260 
process_vector(codebook ** bs,int * addmul,int inter,float * a,int n)261 void process_vector(codebook **bs,int *addmul,int inter,float *a,int n){
262   int bi;
263   int i;
264 
265   for(bi=0;bi<books;bi++){
266     codebook *b=bs[bi];
267     int dim=b->dim;
268     float base=0.f;
269 
270     if(inter){
271       for(i=0;i<n/dim;i++)
272         base=process_one(b,bi,a+i,dim,n/dim,addmul[bi],base);
273     }else{
274       for(i=0;i<=n-dim;i+=dim)
275         base=process_one(b,bi,a+i,dim,1,addmul[bi],base);
276     }
277   }
278 
279   if((long)(count)%100)spinnit("working.... samples: ",count);
280 }
281 
process_usage(void)282 void process_usage(void){
283   fprintf(stderr,
284           "usage: vqmetrics [-i] +|*<codebook>.vqh [ +|*<codebook.vqh> ]... \n"
285           "                 datafile.vqd [datafile.vqd]...\n\n"
286           "       data can be taken on stdin.  -i indicates interleaved coding.\n"
287           "       Output goes to output files:\n"
288           "       basename-me.m:       gnuplot: mean error by entry value\n"
289           "       basename-mse.m:      gnuplot: mean square error by entry value\n"
290           "       basename-worst.m:    gnuplot: worst error by entry value\n"
291           "       basename-distance.m: gnuplot file showing distance probability\n"
292           "\n");
293 
294 }
295