1 /**
2  * @file   wchmm.c
3  *
4  * <JA>
5  * @brief  �ڹ�¤������ι���
6  *
7  * �����Ǥϡ�Ϳ����줿ñ�켭��, HMM�������Ӹ����������ڹ�¤�������
8  * ���ۤ���ؿ����������Ƥ��ޤ�. �ڹ�¤������ϵ�ư���˹��ۤ��졤
9  * ��1�ѥ���ǧ�����Ѥ����ޤ�. �ڹ�¤������Ͼ���ñ�̤ǹ������졤
10  * �ƾ��֤�HMM���ϳ�Ψ���������¾�������õ���Τ�����͡��ʾ����ޤߤޤ�.
11  *
12  * ��ȯ�ηа޾塤��������Ǥ��ڹ�¤������� wchmm (word-conjunction HMM) ��
13  * ��ɽ������Ƥ��ޤ�.
14  *
15  * </JA>
16  *
17  * <EN>
18  * @brief  Construction of tree lexicon.
19  *
20  * Functions to build a tree lexicon (or called word-conjunction HMM here)
21  * from word dictionary, HMM and language models are defined here.  The
22  * constructed tree lexicon will be used for the recognition of the 1st pass.
23  * The lexicon is composed per HMM state unit, and various informations
24  * about output probabilities, arcs, language model constraints, and others
25  * are assembled in the lexicon.
26  *
27  * Note that the word "wchmm" in the source code is a synonim of
28  * "tree lexicon".
29  * </EN>
30  *
31  * @author Akinobu Lee
32  * @date   Mon Sep 19 23:39:15 2005
33  *
34  * $Revision: 1.7 $
35  *
36  */
37 /*
38  * Copyright (c) 1991-2007 Kawahara Lab., Kyoto University
39  * Copyright (c) 2000-2005 Shikano Lab., Nara Institute of Science and Technology
40  * Copyright (c) 2005-2007 Julius project team, Nagoya Institute of Technology
41  * All rights reserved
42  */
43 
44 /* wchmm = word conjunction HMM = lexicon tree */
45 
46 #include <julius/julius.h>
47 
48 
49 #define WCHMM_SIZE_CHECK		///< If defined, do wchmm size estimation (for debug only)
50 
51 /**************************************************************/
52 /*********** Initialization of tree lexicon *******************/
53 /**************************************************************/
54 
55 /**
56  * <JA>
57  * �ڹ�¤������¤�Τ����˳���դ���.
58  *
59  * @return �����˥����˳���դ���줿�ڹ�¤������¤�ΤؤΥݥ������֤�.
60  * </JA>
61  * <EN>
62  * Allocate a new tree lexicon structure.
63  *
64  * @return pointer to the newly allocated tree lexicon structure.
65  * </EN>
66  * @callgraph
67  * @callergraph
68  */
69 WCHMM_INFO *
wchmm_new()70 wchmm_new()
71 {
72   WCHMM_INFO *w;
73   w = (WCHMM_INFO *)mymalloc(sizeof(WCHMM_INFO));
74   w->lmtype = LM_UNDEF;
75   w->lmvar  = LM_UNDEF;
76   w->ngram = NULL;
77   w->dfa = NULL;
78   w->winfo = NULL;
79   w->malloc_root = NULL;
80 #ifdef PASS1_IWCD
81   w->lcdset_category_root = NULL;
82   w->lcdset_mroot = NULL;
83 #endif /* PASS1_IWCD */
84   w->wrk.out_from_len = 0;
85   /* reset user function entry point */
86   w->uni_prob_user = NULL;
87   w->bi_prob_user = NULL;
88   return w;
89 }
90 
91 /**
92  * <JA>
93  * �ڹ�¤����������Ƥ���������.
94  *
95  * @param wchmm [out] �ڹ�¤������ؤΥݥ���
96  * </JA>
97  * <EN>
98  * Initialize content of a lexicon tree.
99  *
100  * @param wchmm [out] pointer to the lexicon tree structure
101  * </EN>
102  */
103 static void
wchmm_init(WCHMM_INFO * wchmm)104 wchmm_init(WCHMM_INFO *wchmm)
105 {
106   /* the resulting tree size is typically half of total state num */
107   wchmm->maxwcn = wchmm->winfo->totalstatenum / 2;
108   wchmm->state = (WCHMM_STATE *)mymalloc(sizeof(WCHMM_STATE)*wchmm->maxwcn);
109   wchmm->self_a = (LOGPROB *)mymalloc(sizeof(LOGPROB)*wchmm->maxwcn);
110   wchmm->next_a = (LOGPROB *)mymalloc(sizeof(LOGPROB)*wchmm->maxwcn);
111   wchmm->ac = (A_CELL2 **)mymalloc(sizeof(A_CELL2 *)*wchmm->maxwcn);
112   wchmm->stend = (WORD_ID *)mymalloc(sizeof(WORD_ID)*wchmm->maxwcn);
113   wchmm->offset = (int **)mymalloc(sizeof(int *)*wchmm->winfo->num);
114   wchmm->wordend = (int *)mymalloc(sizeof(int)*wchmm->winfo->num);
115   wchmm->maxstartnum = STARTNODE_STEP;
116   wchmm->startnode = (int *)mymalloc(sizeof(int)*STARTNODE_STEP);
117   wchmm->startnum = 0;
118   if (wchmm->category_tree) {
119     wchmm->start2wid = (WORD_ID *)mymalloc(sizeof(WORD_ID)*STARTNODE_STEP);
120   }
121   if (wchmm->hmminfo->multipath) {
122     wchmm->wordbegin = (int *)mymalloc(sizeof(int)*wchmm->winfo->num);
123     wchmm->wrk.out_from = (int *)mymalloc(sizeof(int) * wchmm->winfo->maxwn);
124     wchmm->wrk.out_from_next = (int *)mymalloc(sizeof(int) * wchmm->winfo->maxwn);
125     wchmm->wrk.out_a = (LOGPROB *)mymalloc(sizeof(LOGPROB) * wchmm->winfo->maxwn);
126     wchmm->wrk.out_a_next = (LOGPROB *)mymalloc(sizeof(LOGPROB) * wchmm->winfo->maxwn);
127     wchmm->wrk.out_from_len = wchmm->winfo->maxwn;
128   } else {
129     wchmm->wordend_a = (LOGPROB *)mymalloc(sizeof(LOGPROB)*wchmm->winfo->num);
130   }
131 #ifdef PASS1_IWCD
132   wchmm->outstyle = (unsigned char *)mymalloc(sizeof(unsigned char)*wchmm->maxwcn);
133 #endif
134 #ifdef UNIGRAM_FACTORING
135   wchmm->start2isolate = NULL;
136   wchmm->isolatenum = 0;
137 #endif
138   if (!wchmm->category_tree) {
139     wchmm->sclist = NULL;
140     wchmm->sclist2node = NULL;
141 #ifdef UNIGRAM_FACTORING
142     wchmm->fscore = NULL;
143 #endif
144   }
145 
146   wchmm->n = 0;
147 }
148 
149 /**
150  * <JA>
151  * �ڹ�¤������ξ��ֳ�Ǽ�ΰ�� MAXWCNSTEP ʬ������Ĺ����.
152  *
153  * @param wchmm [i/o] �ڹ�¤������
154  * </JA>
155  * <EN>
156  * Expand state-related area in a tree lexicon by MAXWCNSTEP.
157  *
158  * @param wchmm [i/o] tree lexicon
159  * </EN>
160  */
161 static void
wchmm_extend(WCHMM_INFO * wchmm)162 wchmm_extend(WCHMM_INFO *wchmm)
163 {
164   /* practical value! */
165   wchmm->maxwcn += wchmm->winfo->totalstatenum / 6;
166   wchmm->state = (WCHMM_STATE *)myrealloc(wchmm->state, sizeof(WCHMM_STATE)*wchmm->maxwcn);
167   wchmm->self_a = (LOGPROB *)myrealloc(wchmm->self_a, sizeof(LOGPROB)*wchmm->maxwcn);
168   wchmm->next_a = (LOGPROB *)myrealloc(wchmm->next_a, sizeof(LOGPROB)*wchmm->maxwcn);
169   wchmm->ac = (A_CELL2 **)myrealloc(wchmm->ac, sizeof(A_CELL2 *)*wchmm->maxwcn);
170   wchmm->stend = (WORD_ID *)myrealloc(wchmm->stend, sizeof(WORD_ID)*wchmm->maxwcn);
171 #ifdef PASS1_IWCD
172   wchmm->outstyle = (unsigned char *)myrealloc(wchmm->outstyle, sizeof(unsigned char)*wchmm->maxwcn);
173 #endif
174 }
175 
176 /**
177  * <JA>
178  * �ڹ�¤�������ñ����Ƭ�Ρ��ɳ�Ǽ�ΰ�� STARTNODE_STEPʬ������Ĺ����.  (multipath)
179  *
180  * @param wchmm [i/o] �ڹ�¤������
181  * </JA>
182  * <EN>
183  * Expand word-start nodes area in a tree lexicon by STARTNODE_STEP. (multipath)
184  *
185  * @param wchmm [i/o] tree lexicon
186  * </EN>
187  */
188 static void
wchmm_extend_startnode(WCHMM_INFO * wchmm)189 wchmm_extend_startnode(WCHMM_INFO *wchmm)
190 {
191   wchmm->maxstartnum += STARTNODE_STEP;
192   wchmm->startnode = (int *)myrealloc(wchmm->startnode, sizeof(int) * wchmm->maxstartnum);
193   if (wchmm->category_tree) {
194     wchmm->start2wid = (WORD_ID *)myrealloc(wchmm->start2wid, sizeof(WORD_ID) * wchmm->maxstartnum);
195   }
196 }
197 
198 /**
199  * <JA>
200  * �ڹ�¤��������Ӥ��������γ��ե�������Ʋ�������.
201  *
202  * @param w [in] �ڹ�¤������
203  * </JA>
204  * <EN>
205  * Free all data in a tree lexicon.
206  *
207  * @param w [in] tree lexicon
208  * </EN>
209  * @callgraph
210  * @callergraph
211  */
212 void
wchmm_free(WCHMM_INFO * w)213 wchmm_free(WCHMM_INFO *w)
214 {
215   S_CELL *sc, *sctmp;
216   int i;
217   /* wchmm->state[i].ac malloced by mybmalloc2() */
218   /* wchmm->offset[][] malloced by mybmalloc2() */
219 #ifdef PASS1_IWCD
220   /* LRC_INFO, RC_INFO in wchmm->state[i].outsty malloced by mybmalloc2() */
221 #endif
222   /* they all will be freed by a single mybfree2() call */
223   mybfree2(&(w->malloc_root));
224   if (!w->category_tree) {
225     if (w->sclist != NULL) {
226       for(i=1;i<w->scnum;i++) {
227 	sc = w->sclist[i];
228 	while(sc) {
229 	  sctmp = sc->next;
230 	  free(sc);
231 	  sc = sctmp;
232 	}
233       }
234       free(w->sclist);
235     }
236     if (w->sclist2node != NULL) free(w->sclist2node);
237 #ifdef UNIGRAM_FACTORING
238     if (w->fscore != NULL) free(w->fscore);
239 #endif
240   }
241 #ifdef UNIGRAM_FACTORING
242   if (w->start2isolate != NULL) free(w->start2isolate);
243 #endif
244 #ifdef PASS1_IWCD
245   free(w->outstyle);
246 #endif
247   if (w->hmminfo->multipath) {
248     free(w->wordbegin);
249   } else {
250     free(w->wordend_a);
251   }
252   if (w->category_tree) free(w->start2wid);
253   free(w->startnode);
254   free(w->wordend);
255   free(w->offset);
256   free(w->stend);
257   free(w->ac);
258   free(w->next_a);
259   free(w->self_a);
260   free(w->state);
261 #ifdef PASS1_IWCD
262   if (w->category_tree) lcdset_remove_with_category_all(w);
263 #endif /* PASS1_IWCD */
264   if (w->wrk.out_from_len != 0) {
265     free(w->wrk.out_from);
266     free(w->wrk.out_from_next);
267     free(w->wrk.out_a);
268     free(w->wrk.out_a_next);
269     w->wrk.out_from_len = 0;
270   }
271   free(w);
272 }
273 
274 
275 /**************************************************************/
276 /*********** Word sort functions for tree construction ********/
277 /**************************************************************/
278 
279 /**
280  * <JA>
281  * ñ����ǤΤʤ�Ӥǥ����Ȥ���qsort_reentrant�ؿ�
282  *
283  * @param widx1 [in] ñ��ID 1 �ؤΥݥ���
284  * @param widx2 [in] ñ��ID 2 �ؤΥݥ���
285  *
286  * @return ñ��widx2��ñ��widx1�ΰ���������Ǥ���� 1, ñ��widx1��ñ��widx2�ΰ���������Ǥ���� -1, ����Ʊ�������¤ӤǤ���� 0 ���֤�.
287  * </JA>
288  * <EN>
289  * qsort_reentrant function to sort words by their phoneme sequence.
290  *
291  * @param widx1 [in] pointer to word id #1
292  * @param widx2 [in] pointer to wrod id #2
293  *
294  * @return 1 if word[widx2] is part of word[widx1], -1 if word[widx1] is part of word[widx2], or 0 if the two words are equal.
295  * </EN>
296  */
297 static int
compare_wseq(WORD_ID * widx1,WORD_ID * widx2,WORD_INFO * winfo)298 compare_wseq(WORD_ID *widx1, WORD_ID *widx2, WORD_INFO *winfo)
299 {
300   int len1, len2, n;
301   int p=0;
302 
303   len1 = winfo->wlen[*widx1];
304   len2 = winfo->wlen[*widx2];
305 
306   n=0;
307   /*  while (n < len1 && n < len2 && (p = (int)winfo->wseq[*widx1][n] - (int)winfo->wseq[*widx2][n]) == 0 ) n++;*/
308   while (n < len1 && n < len2 && (p = strcmp((winfo->wseq[*widx1][n])->name, (winfo->wseq[*widx2][n])->name)) == 0 ) n++;
309   if (n < len1) {
310     if (n < len2) {
311       /* differ */
312       return(p);
313     } else {
314       /* 2 is part of 1 */
315       return(1);
316     }
317   } else {
318     if (n < len2) {
319       /* 1 is part of 2 */
320       return(-1);
321     } else {
322       /* same */
323       return(0);
324     }
325   }
326 }
327 
328 /**
329  * <JA>
330  * ñ��ID�ν��� windex[bgn..bgn+len-1] ��ñ��β��Ǥʤ�Ӥǥ����Ȥ���.
331  *
332  * @param winfo [in] ñ�켭��
333  * @param windex [i/o] ñ��ID�Υ���ǥå�����������ǥ����Ȥ�����
334  * @param bgn [in] @a windex �Υ����ȳ�����
335  * @param len [in] @a windex �� @a bgn ����Υ����Ȥ������ǿ�
336  * </JA>
337  * <EN>
338  * Sort word IDs in windex[bgn..bgn+len-1] by their phoneme sequence order.
339  *
340  * @param winfo [in] word lexicon
341  * @param windex [i/o] index sequence of word IDs, (will be sorted in this function)
342  * @param bgn [in] start point to sort in @a windex
343  * @param len [in] length of indexes to be sorted from @a bgn
344  * </EN>
345  */
346 static void
wchmm_sort_idx_by_wseq(WORD_INFO * winfo,WORD_ID * windex,WORD_ID bgn,WORD_ID len)347 wchmm_sort_idx_by_wseq(WORD_INFO *winfo, WORD_ID *windex, WORD_ID bgn, WORD_ID len)
348 {
349   qsort_reentrant(&(windex[bgn]), len, sizeof(WORD_ID), (int (*)(const void *, const void *, void *))compare_wseq, winfo);
350 }
351 
352 /**
353  * <JA>
354  * ñ����ƥ���ID�ǥ����Ȥ���qsort�ؿ�.
355  *
356  * @param widx1 [in] ����1�ؤΥݥ���
357  * @param widx2 [in] ����2�ؤΥݥ���
358  *
359  * @return
360  * </JA>
361  * <EN>
362  * qsort function to sort words by their category ID.
363  *
364  * @param widx1 [in] pointer to element #1
365  * @param widx2 [in] pointer to element #2
366  *
367  * @return
368  * </EN>
369  */
370 static int
compare_category(WORD_ID * widx1,WORD_ID * widx2,WORD_INFO * winfo)371 compare_category(WORD_ID *widx1, WORD_ID *widx2, WORD_INFO *winfo)
372 {
373   int c1,c2;
374   c1 = winfo->wton[*widx1];
375   c2 = winfo->wton[*widx2];
376   return(c1 - c2);
377 }
378 
379 /**
380  * <JA>
381  * ñ��ID���� windex[0..len-1] ���ƥ���ID�ǥ����Ȥ���.
382  *
383  * @param winfo [in] ñ�켭��
384  * @param windex [i/o] ñ��ID�Υ���ǥå�����������ǥ����Ȥ�����
385  * @param len [in] @a windex �����ǿ�
386  * </JA>
387  * <EN>
388  * Sort word IDs in windex[0..len-1] by their category ID.
389  *
390  * @param winfo [in] tree lexicon
391  * @param windex [i/o] index sequence of word IDs, (will be sorted in this function)
392  * @param len [in] number of elements in @a windex
393  * </EN>
394  */
395 static void
wchmm_sort_idx_by_category(WORD_INFO * winfo,WORD_ID * windex,WORD_ID len)396 wchmm_sort_idx_by_category(WORD_INFO *winfo, WORD_ID *windex, WORD_ID len)
397 {
398   qsort_reentrant(windex, len, sizeof(WORD_ID), (int (*)(const void *, const void *, void *))compare_category, winfo);
399 }
400 
401 
402 /**********************************************************************/
403 /************** Subroutines to link part of words  ********************/
404 /**********************************************************************/
405 
406 /**
407  * <JA>
408  * ��ñ��֤ǡ�ñ�����Ƭ����Ʊ��Ƕ�ͭ��ǽ�ʲ��Ǥο���Ĵ�٤�.
409  *
410  * @param winfo [in] ñ�켭��
411  * @param i [in] ñ�죱
412  * @param j [in] ñ�죲
413  *
414  * @return ��ͭ��ǽ����Ƭ����β��ǿ����֤�.
415  * </JA>
416  * <EN>
417  * Compare two words from word head per phoneme to see how many phones
418  * can be shared among the two.
419  *
420  * @param winfo [in] word dictionary
421  * @param i [in] a word
422  * @param j [in] another word
423  *
424  * @return the number of phonemes to be shared from the head of the words.
425  * </EN>
426  */
427 static int
wchmm_check_match(WORD_INFO * winfo,int i,int j)428 wchmm_check_match(WORD_INFO *winfo, int i, int j)
429 {
430   int k,tmplen;
431 
432   for (tmplen=0,k=0;k<winfo->wlen[i];k++) {
433     if (k > winfo->wlen[j]-1)
434       break;
435     if (! (strmatch(winfo->wseq[i][k]->name, winfo->wseq[j][k]->name)))
436       break;
437     tmplen++;
438   }
439   return(tmplen);
440 }
441 
442 /**
443  * <EN>
444  * Initialize transition information on a node.
445  * </EN>
446  * <JA>
447  * �Ρ��ɤ����ܾ������������.
448  * </JA>
449  *
450  * @param wchmm [i/o] tree lexicon
451  * @param node [in] node id
452  *
453  */
454 static void
acc_init(WCHMM_INFO * wchmm,int node)455 acc_init(WCHMM_INFO *wchmm, int node)
456 {
457   wchmm->self_a[node] = LOG_ZERO;
458   wchmm->next_a[node] = LOG_ZERO;
459   wchmm->ac[node] = NULL;
460 }
461 
462 /**
463  * <EN>
464  * Add an arc to a node.
465  * This function is for transition other than self and next node.
466  * </EN>
467  * <JA>
468  * �Ρ��ɤ����ܤ��ɲä���.
469  * ���δؿ��ϼ������ܡ��٤ؤ����ܰʳ��ξ��˻��Ѥ����.
470  * </JA>
471  *
472  * @param wchmm [i/o] tree lexicon
473  * @param node [in] node id
474  * @param a [in] transition probability in log10
475  * @param arc [in] transition destination node id
476  *
477  */
478 static void
add_ac(WCHMM_INFO * wchmm,int node,LOGPROB a,int arc)479 add_ac(WCHMM_INFO *wchmm, int node, LOGPROB a, int arc)
480 {
481   A_CELL2 *ac2;
482 
483   for(ac2=wchmm->ac[node];ac2;ac2=ac2->next) {
484     if (ac2->n < A_CELL2_ALLOC_STEP) break;
485   }
486   if (ac2 == NULL) {
487     ac2 = (A_CELL2 *)mybmalloc2(sizeof(A_CELL2), &(wchmm->malloc_root));
488     ac2->n = 0;
489     ac2->next = wchmm->ac[node];
490     wchmm->ac[node] = ac2;
491   }
492   ac2->arc[ac2->n] = arc;
493   ac2->a[ac2->n]   = a;
494   ac2->n++;
495 }
496 
497 /**
498  * <JA>
499  * �ڹ�¤������Τ���Ρ��ɤˡ��̤ΥΡ��ɤؤ����ܤ��ɲä���
500  *
501  * @param wchmm [i/o] �ڹ�¤������
502  * @param node [in] �Ρ����ֹ�
503  * @param a [in] ���ܳ�Ψ���п���
504  * @param arc [in] ������ΥΡ����ֹ�
505  * </JA>
506  * <EN>
507  * Add a transition arc between two nodes on the tree lexicon
508  *
509  * @param wchmm [i/o] tree lexicon
510  * @param node [in] node number of source node
511  * @param a [in] transition probability in log scale
512  * @param arc [in] node number of destination node
513  * </EN>
514  */
515 static void
add_wacc(WCHMM_INFO * wchmm,int node,LOGPROB a,int arc)516 add_wacc(WCHMM_INFO *wchmm, int node, LOGPROB a, int arc)
517 {
518   if (arc == node) {
519     wchmm->self_a[node] = a;
520   } else if (arc == node + 1) {
521     wchmm->next_a[node] = a;
522   } else {
523     add_ac(wchmm, node, a, arc);
524   }
525 }
526 
527 /**
528  * <JA>
529  * ����ñ��Τ�����֤β��Ǥ���ñ����ü�γ��ؽФ����ܤΥꥹ�Ȥ�����. (multipath)
530  *
531  * @param wchmm [in] �ڹ�¤������
532  * @param w [in] ñ��ID
533  * @param pos [in] ���ǰ���
534  * @param node [out] ������Ρ�ñ����ü���ؤ����ܤ���ľ��֤Υꥹ��
535  * @param a [out] @a node �γ����Ǥ����ܳ�Ψ
536  * @param num [out] @a node �����ǿ�. ȯ�����������ä����.
537  * @param maxnum [in] @a node �γ�Ǽ��ǽ�ʺ����
538  * @param insert_sp [in] ñ�콪ü�Ǥ� sp ���߹��ߤ��θ����ʤ�TRUE
539  * </JA>
540  * <EN>
541  * Make outgoing transition list for given phone position of a word. (multipath)
542  *
543  * @param wchmm [in] tree lexicon
544  * @param w [in] word ID
545  * @param pos [in] location of target phone to be inspected in the word @a w
546  * @param node [out] list of wchmm states that possibly has outgoing transition
547  * @param a [out] transition probabilities of the outgoing transitions in @a node
548  * @param num [out] number of elements in @a out (found num will be added)
549  * @param maxnum [in] maximum number of elements that can be stored in @a node
550  * @param insert_sp [in] TRUE if consider short-pause insertion on word end
551  * </EN>
552  */
553 static void
get_outtrans_list(WCHMM_INFO * wchmm,WORD_ID w,int pos,int * node,LOGPROB * a,int * num,int maxnum,boolean insert_sp)554 get_outtrans_list(WCHMM_INFO *wchmm, WORD_ID w, int pos, int *node, LOGPROB *a, int *num, int maxnum, boolean insert_sp)
555 {
556   HMM_Logical *ltmp;
557   int states;
558   int k;
559   LOGPROB prob;
560   int oldnum;
561 
562   if (pos < 0) {
563 
564     /* set the word-beginning node, and return */
565     node[*num] = wchmm->wordbegin[w];
566     a[*num] = 0.0;
567     (*num)++;
568 
569   } else {
570 
571     ltmp = wchmm->winfo->wseq[w][pos];
572     states = hmm_logical_state_num(ltmp);
573 
574     /* check initial->final state */
575     if ((hmm_logical_trans(ltmp))->a[0][states-1] != LOG_ZERO) {
576       /* recursive call for previous phone */
577       oldnum = *num;
578       get_outtrans_list(wchmm, w, pos-1, node, a, num, maxnum, FALSE); /* previous phone should not be an sp-inserted phone */
579       /* add probability of the skip transition to all the previous ones */
580       for(k=oldnum;k<*num;k++) {
581 	a[k] += (hmm_logical_trans(ltmp))->a[0][states-1];
582       }
583     }
584     /* add to list the arcs from output state to final state */
585     for (k = 1; k < states - 1; k++) {
586       prob = (hmm_logical_trans(ltmp))->a[k][states-1];
587       if (prob != LOG_ZERO) {
588 	if (*num >= maxnum) {
589 	  j_internal_error("get_outtrans_list: maximum outtrans list num exceeded %d\n", maxnum);
590 	}
591 	node[*num] = wchmm->offset[w][pos] + k - 1;
592 	a[*num] = prob;
593 	(*num)++;
594       }
595     }
596     /* for -iwsp, add outgoing arc from the tail sp model
597        only if need_sp == TRUE.
598        need_sp should be TRUE only when the connecting [pos] phone is also an end phone of the to-be-added word (i.e. homophone word)
599      */
600     /*  */
601     if (insert_sp) {
602       /* consider sp */
603       for (k = 1; k < hmm_logical_state_num(wchmm->hmminfo->sp) - 1; k++) {
604 	prob = hmm_logical_trans(wchmm->hmminfo->sp)->a[k][hmm_logical_state_num(wchmm->hmminfo->sp)-1];
605 	if (prob != LOG_ZERO) {
606 	  if (*num >= maxnum) {
607 	    j_internal_error("get_outtrans_list: maximum outtrans list num exceeded %d\n", maxnum);
608 	  }
609 	  node[*num] = wchmm->offset[w][pos] + (states - 2) + k - 1;
610 	  a[*num] = prob;
611 	  (*num)++;
612 	}
613       }
614     }
615   }
616   /*printf("   %d(%s)-%d:\"%s\", num=%d\n", w, wchmm->winfo->woutput[w], pos,
617     (pos < 0) ? "BGN" : wchmm->winfo->wseq[w][pos]->name, *num);*/
618   return;
619 }
620 
621 /**
622  * <JA>
623  * ���벻�Ǥ������ξ��֤��顤���벻�Ǥ���Ƭ���֤ؤ����ܤ��ɲä���.
624  *
625  * @param wchmm [i/o] �ڹ�¤������
626  * @param from_node [in] ���벻�Ǥ������ξ���
627  * @param to_node [in] ���벻�Ǥ���Ƭ����
628  * @param tinfo [in] @a from_node ��°���벻��HMM�����ܳ�Ψ����
629  * </JA>
630  * <EN>
631  * Add a transition from end node of a phone to start node of another phone.
632  *
633  * @param wchmm [i/o] tree lexicon
634  * @param from_node [in] end node of a phone
635  * @param to_node [in] start node of a phone
636  * @param tinfo [in] transition prob. matrix of the @a from_node phone.
637  * </EN>
638  */
639 static void
wchmm_link_hmm(WCHMM_INFO * wchmm,int from_node,int to_node,HTK_HMM_Trans * tinfo)640 wchmm_link_hmm(WCHMM_INFO *wchmm, int from_node, int to_node, HTK_HMM_Trans *tinfo)
641 {
642   A_CELL2 *actmp;
643   LOGPROB a;
644   int i, j;
645   boolean tflag;
646 
647   /* get transition probability to outer state in tinfo */
648   for(i = tinfo->statenum - 2; i >= 0; i--) {
649     if ((a = tinfo->a[i][tinfo->statenum-1]) != LOG_ZERO) { /* found */
650       /* check if the arc already exist */
651       tflag = FALSE;
652       if (to_node == from_node && wchmm->self_a[from_node] == a) {
653 	tflag = TRUE;
654       } else if (to_node == from_node + 1 && wchmm->next_a[from_node] == a) {
655 	tflag = TRUE;
656       } else {
657 	for (actmp = wchmm->ac[from_node]; actmp; actmp = actmp->next) {
658 	  for(j=0;j<actmp->n;j++) {
659 	    if (actmp->arc[j] == to_node && actmp->a[j] == a) {
660 	      tflag = TRUE;
661 	      break;
662 	    }
663 	  }
664 	  if (tflag == TRUE) break;
665 	}
666       }
667       if (tflag) break;
668       /* add the arc to wchmm */
669       add_wacc(wchmm, from_node, a, to_node);
670       return;			/* exit function here */
671     }
672   }
673   j_internal_error("wchmm_link_hmm: No arc to endstate?\n");
674 }
675 
676 /**
677  * <JA>
678  * �ڹ�¤��������Σ�ñ����Τ��벻�Ǵ֤���³����.
679  *
680  * @param wchmm [i/o] �ڹ�¤������
681  * @param from_word [in] ���ܸ���ñ���ID
682  * @param from_seq [in] ���ܸ���ñ�������³���벻�Ǥΰ���
683  * @param to_word [in] �������ñ���ID
684  * @param to_seq [in] �������ñ�������³���벻�Ǥΰ���
685  * </JA>
686  * <EN>
687  * Connect two phonemes in tree lexicon.
688  *
689  * @param wchmm [i/o] tree lexicon
690  * @param from_word [in] source word ID
691  * @param from_seq [in] index of source phoneme in @a from_word from which the other will be connected
692  * @param to_word [in] destination word ID
693  * @param to_seq [in] index of destination phoneme in @a to_word to which the other will connect
694  * </EN>
695  */
696 static void
wchmm_link_subword(WCHMM_INFO * wchmm,int from_word,int from_seq,int to_word,int to_seq)697 wchmm_link_subword(WCHMM_INFO *wchmm, int from_word, int from_seq, int to_word, int to_seq)
698 {
699   HMM_Logical *last;
700   int lastp;
701 
702   last = wchmm->winfo->wseq[from_word][from_seq];
703   lastp = wchmm->offset[from_word][from_seq] + hmm_logical_state_num(last)-2 -1;
704   wchmm_link_hmm(wchmm, lastp, wchmm->offset[to_word][to_seq],
705 		 hmm_logical_trans(last));
706 }
707 
708 /**************************************************************/
709 /******** homophone processing: duplicating leaf nodes ********/
710 /**************************************************************/
711 
712 /**
713  * @note
714  * <JA>
715  * Ʊ�������:
716  * �ڹ�¤������ˤ����Ƥ��٤Ƥ�ñ�����Ω�����ǽ����֤����ɬ�פ����뤿�ᡤ
717  * Ʊ�������տ�������ɬ�פ�����. ���Τ��ᡤ�ǽ���ڹ�¤��������ۤ�����,
718  * �̤�ñ��ȴ����˶�ͭ���줿ñ��(Ʊ����), ���뤤���̤�ñ��ΰ����Ȥ���
719  * �����ޤ�Ƥ��ޤäƤ���ñ���ȯ������ȤȤ��, ���κǽ��Ρ��ɤ�
720  * ���ԡ����ƿ�����ñ�콪ü�Ρ��ɤ���ɬ�פ�����.
721  * </JA>
722  * <EN>
723  * Homophones:
724  * As all words need to have an uniq state as a final state in a lexicon tree,
725  * homophones should be handled carefully.  After primal tree has been made,
726  * we look through the tree to find the fully shared or embedded words
727  * (homophone or part of other word), and duplicate the last leaf node
728  * to have uniq end state.
729  * </EN>
730  */
731 
732 /**
733  * <JA>
734  * ñ�콪ü���֤���Ω����Ϳ����줿ñ��ν�ü�Ρ��ɤ��ԡ����ơ�
735  * �����ˤ���ñ��κǽ����֤Ȥ����������.
736  *
737  * @param wchmm [i/o] �ڹ�¤������
738  * @param node [in] Ʊ����ν�ü�Ρ����ֹ�
739  * @param word [in] ��������Ͽ����ñ��
740  * </JA>
741  * <EN>
742  * Isolation of word-end nodes for homophones: duplicate the word-end state,
743  * link as the same as original, and make it the new word-end node of the
744  * given new word.
745  *
746  * @param wchmm [i/o] tree lexicon
747  * @param node [in] the word end node of the already existing homophone
748  * @param word [in] word ID to be added to the tree
749  * </EN>
750  */
751 static void
wchmm_duplicate_state(WCHMM_INFO * wchmm,int node,int word)752 wchmm_duplicate_state(WCHMM_INFO *wchmm, int node, int word) /* source node, new word */
753 {
754   int j, n;
755   int n_src, n_prev;
756   A_CELL2	*ac;
757   HMM_Logical *lastphone;
758 
759   /* 1 state will newly created: expand tree if needed */
760   if (wchmm->n + 1 >= wchmm->maxwcn) {
761     wchmm_extend(wchmm);
762   }
763   /* n: the target new node to which 'node' is copied */
764   n = wchmm->n;
765 
766   n_src = node;
767 
768   /* copy output probability info */
769 #ifdef PASS1_IWCD
770   {
771     RC_INFO *rcnew;
772     LRC_INFO *lrcnew;
773     wchmm->outstyle[n] = wchmm->outstyle[n_src];
774     if (wchmm->outstyle[n] == AS_RSET) {
775       /* duplicate RC_INFO because it has its own cache */
776       rcnew = (RC_INFO *)mybmalloc2(sizeof(RC_INFO), &(wchmm->malloc_root));
777       memcpy(rcnew, wchmm->state[n_src].out.rset, sizeof(RC_INFO));
778       wchmm->state[n].out.rset = rcnew;
779     } else if (wchmm->outstyle[n] == AS_LRSET) {
780       /* duplicate LRC_INFO because it has its own cache */
781       lrcnew = (LRC_INFO *)mybmalloc2(sizeof(LRC_INFO), &(wchmm->malloc_root));
782       memcpy(lrcnew, wchmm->state[n_src].out.lrset, sizeof(LRC_INFO));
783       wchmm->state[n].out.lrset = lrcnew;
784     } else {
785       /* share same info, simply copy the pointer */
786       memcpy(&(wchmm->state[n].out), &(wchmm->state[n_src].out), sizeof(ACOUSTIC_SPEC));
787     }
788   }
789 #else  /* ~PASS1_IWCD */
790   memcpy(&(wchmm->state[n].out), &(wchmm->state[n_src].out), sizeof(HTK_HMM_State *));
791 #endif
792 
793   lastphone = wchmm->winfo->wseq[word][wchmm->winfo->wlen[word]-1];
794   acc_init(wchmm, n);
795 
796   /* add self transition arc */
797   wchmm->self_a[n] = wchmm->self_a[n_src];
798 
799   /* copy transition arcs whose destination is the source node to new node */
800   if (hmm_logical_state_num(lastphone) == 3) { /* = 1 state */
801     /* phone with only 1 state should be treated carefully */
802     if (wchmm->winfo->wlen[word] == 1) { /* word consists of only this phone */
803       /* no arcs need to be copied: this is also a start node of a word */
804       wchmm->offset[word][0] = n;
805       /* index the new word-beginning node as startnode (old ststart) */
806       if (wchmm->lmtype != LM_PROB || word != wchmm->winfo->head_silwid) {
807 	wchmm->startnode[wchmm->startnum] = n;
808 	if (wchmm->category_tree) wchmm->start2wid[wchmm->startnum] = word;
809 	/* expand data area if necessary */
810 	if (++wchmm->startnum >= wchmm->maxstartnum) wchmm_extend_startnode(wchmm);
811       }
812     } else {
813       /* copy arcs from the last state of the previous phone */
814       n_prev = wchmm->offset[word][wchmm->winfo->wlen[word]-2]
815 	+ hmm_logical_state_num(wchmm->winfo->wseq[word][wchmm->winfo->wlen[word]-2]) - 3;
816       if(n_src == n_prev + 1) {
817 	add_wacc(wchmm, n_prev, wchmm->next_a[n_prev], n);
818       } else {
819 	for(ac=wchmm->ac[n_prev];ac;ac=ac->next) {
820 	  for(j=0;j<ac->n;j++) {
821 	    if (ac->arc[j] == n_src) {
822 	      add_wacc(wchmm, n_prev, ac->a[j], n);
823 	    }
824 	  }
825 	}
826       }
827       /* also update the last offset (== wordend in this case) */
828       wchmm->offset[word][wchmm->winfo->wlen[word]-1] = n;
829     }
830   } else {			/* phone with more than 2 states */
831     /* copy arcs from/to the source node to new node */
832     for (n_prev = wchmm->offset[word][wchmm->winfo->wlen[word]-1]; n_prev < n_src; n_prev++) {
833       if (n_src == n_prev + 1) {
834 	add_wacc(wchmm, n_prev, wchmm->next_a[n_prev], n);
835       } else {
836 	for(ac=wchmm->ac[n_prev];ac;ac=ac->next) {
837 	  for(j=0;j<ac->n;j++) {
838 	    if (ac->arc[j] == n_src) {
839 	      add_wacc(wchmm, n_prev, ac->a[j], n);
840 	    }
841 	  }
842 	}
843       }
844       if (n_prev == n_src + 1) {
845 	add_wacc(wchmm, n, wchmm->next_a[n_src], n_prev);
846       } else {
847 	for(ac=wchmm->ac[n_src];ac;ac=ac->next) {
848 	  for(j=0;j<ac->n;j++) {
849 	    if (ac->arc[j] == n_prev) {
850 	      add_wacc(wchmm, n, ac->a[j], n_prev);
851 	    }
852 	  }
853 	}
854       }
855     }
856   }
857 
858   /* map word <-> node */
859   wchmm->stend[n]   = word;	/* 'n' is an end node of word 'word' */
860   wchmm->wordend[word] = n;	/* the word end node of 'word' is 'n' */
861 
862   /* new state has been created: increment the size */
863   wchmm->n++;
864 
865 }
866 
867 /**
868  * <JA>
869  * �ڹ�¤���������Τ��������ơ����٤Ƥ�Ʊ����ˤĤ���ñ�콪ü���֤���Ω��
870  * ��Ԥ�.
871  *
872  * @param wchmm [i/o] �ڹ�¤������
873  * </JA>
874  * <EN>
875  * Scan the whole lexicon tree to find already registered homophones, and
876  * make word-end nodes of the found homophones isolated from others.
877  *
878  * @param wchmm [i/o] tree lexicon
879  * </EN>
880  */
881 static int
wchmm_duplicate_leafnode(WCHMM_INFO * wchmm)882 wchmm_duplicate_leafnode(WCHMM_INFO *wchmm)
883 {
884   int w, nlast, n, narc, narc_model;
885   boolean *dupw;		/* node marker */
886   A_CELL2 *actmp;
887   int dupcount;
888 
889   dupcount = 0;
890 
891   nlast = wchmm->n;
892   dupw = (boolean *)mymalloc(sizeof(boolean) * nlast);
893   for(n=0;n<nlast;n++) dupw[n] = FALSE;	/* initialize all marker */
894 
895   for (w=0;w<wchmm->winfo->num;w++) {
896     n = wchmm->wordend[w];
897     if (dupw[n]) {		/* if already marked (2nd time or later */
898       wchmm_duplicate_state(wchmm, n, w); dupcount++; /* duplicate */
899     } else {			/* if not marked yet (1st time) */
900       /* try to find an arc outside the word */
901       {
902 	/* count number of model-internal arc from the last state */
903 	HMM_Logical *lastphone;
904 	HTK_HMM_Trans *tinfo;
905 	int laststate, i;
906 	lastphone = wchmm->winfo->wseq[w][wchmm->winfo->wlen[w]-1];
907 	laststate = hmm_logical_state_num(lastphone) - 2;
908 	tinfo = hmm_logical_trans(lastphone);
909 	narc_model=0;
910 	for(i=1;i<hmm_logical_state_num(lastphone)-1;i++) {
911 	  if (tinfo->a[laststate][i] != LOG_ZERO) narc_model++;
912 	}
913 	/* count number of actual arc from the last state in the tree */
914 	narc = 0;
915 	if (wchmm->self_a[n] != LOG_ZERO) narc++;
916 	if (wchmm->next_a[n] != LOG_ZERO) narc++;
917 	for(actmp=wchmm->ac[n];actmp;actmp=actmp->next) narc += actmp->n;
918       }
919       /* if both number does not match, it means it is not a single word tail */
920       if (narc_model != narc) {
921 	/* word 'w' is embedded as part of other words at this node 'n' */
922 	/* duplicate this node now */
923 	wchmm_duplicate_state(wchmm, n, w); dupcount++;
924 	/* as new node has been assigned as word end node of word 'w',
925 	   reset this source node as it is not the word end node */
926 	wchmm->stend[n] = WORD_INVALID;
927       } else {
928 	/* no arc to other node found, it means it is a single word tail */
929 	/* as this is first time, only make sure that this node is word end of [w] */
930 	wchmm->stend[n] = w;
931       }
932       /* mark node 'n' */
933       dupw[n] = TRUE;
934     }
935   }
936   free(dupw);
937 
938   return(dupcount);
939 }
940 
941 /**************************************************************/
942 /*************** add a word to wchmm lexicon tree *************/
943 /**************************************************************/
944 
945 /**
946  * <JA>
947  * �ڹ�¤������˿�����ñ����ɲä���. �ɲþ��ξ���Ȥ��ơ����ߤ��ڹ�¤��
948  * ������ǺǤ⤽��ñ�����Ƭ�����ɤ��ޥå�����ñ�졤����Ӥ��Υޥå�����Ĺ��
949  * ����ꤹ��.
950  *
951  * @param wchmm [i/o] �ڹ�¤������
952  * @param word [in] �ɲä��뼭��ñ���ID
953  * @param matchlen [in] @a word �� @a matchword ����Ƭ����ޥå����벻��Ĺ
954  * @param matchword [in] ��¸���ڹ�¤��������� @a word �ȺǤ�ޥå�����ñ��
955  * @param enable_iwsp [in] ñ��֥��硼�ȥݡ�����ǽ���ѻ�TRUE�����
956  * </JA>
957  * <EN>
958  * Add a new word to the lexicon tree.  The longest matched word in the current
959  * lexicon tree and the length of the matched phoneme from the word head should
960  * be specified to tell where to insert the new word to the tree.
961  *
962  * @param wchmm [i/o] tree lexicon
963  * @param word [in] word id to be added to the lexicon
964  * @param matchlen [in] phoneme match length between @a word and @a matchword.
965  * @param matchword [in] the longest matched word with @a word in the current lexicon tree
966  * @param enable_iwsp [in] should be TRUE when using inter-word short pause option
967  * </EN>
968  */
969 static boolean
wchmm_add_word(WCHMM_INFO * wchmm,int word,int matchlen,int matchword,boolean enable_iwsp)970 wchmm_add_word(WCHMM_INFO *wchmm, int word, int matchlen, int matchword, boolean enable_iwsp)
971 {
972   boolean ok_p;
973   int   j,k,n;
974   int   add_head, add_tail, add_to;
975   int   word_len, matchword_len;
976   HMM_Logical *ltmp;
977   int ato;
978   LOGPROB prob;
979   int ntmp;
980   int ltmp_state_num;
981 #ifdef PASS1_IWCD
982   CD_Set *lcd = NULL;
983 #endif
984   int *out_from;
985   int *out_from_next;
986   LOGPROB *out_a;
987   LOGPROB *out_a_next;
988 
989 
990   /* for multipath handling */
991   int out_num_prev, out_num_next;
992   int kkk;
993 
994   ok_p = TRUE;
995   if (wchmm->hmminfo->multipath) {
996     out_from = wchmm->wrk.out_from;
997     out_from_next = wchmm->wrk.out_from_next;
998     out_a = wchmm->wrk.out_a;
999     out_a_next = wchmm->wrk.out_a_next;
1000   }
1001 
1002 /*
1003  *   if (matchlen > 0) {
1004  *     printf("--\n");
1005  *     put_voca(stdout, wchmm->winfo, word);
1006  *     put_voca(stdout, wchmm->winfo, matchword);
1007  *     printf("matchlen=%d\n", matchlen);
1008  *   }
1009  */
1010 
1011   /* variable abbreviations */
1012   n = wchmm->n;
1013   word_len      = wchmm->winfo->wlen[word];
1014   matchword_len = wchmm->winfo->wlen[matchword];
1015 
1016   /* malloc phone offset area */
1017   wchmm->offset[word] = (int *)mybmalloc2(sizeof(int)*word_len, &(wchmm->malloc_root));
1018 
1019   /* allocate unshared (new) part */
1020   add_head = matchlen;
1021   add_tail = word_len - 1;
1022   add_to   = matchlen - 1;
1023 
1024   if (wchmm->hmminfo->multipath) {
1025     /* make word-beginning node if needed */
1026     if (matchlen == 0) {
1027       /* create word-beginning node */
1028       wchmm->wordbegin[word] = n;
1029       wchmm->stend[n] = WORD_INVALID;
1030       acc_init(wchmm, n);
1031       wchmm->state[n].out.state = NULL;
1032       /* index the new word-beginning node as startnode (old ststart) */
1033       wchmm->startnode[wchmm->startnum] = n;
1034       if (wchmm->category_tree) wchmm->start2wid[wchmm->startnum] = word;
1035       /* expand data area if necessary */
1036       if (++wchmm->startnum >= wchmm->maxstartnum) wchmm_extend_startnode(wchmm);
1037       if (++n >= wchmm->maxwcn) wchmm_extend(wchmm);
1038     } else {
1039       wchmm->wordbegin[word] = wchmm->wordbegin[matchword];
1040     }
1041 
1042     /* now n is at beginning of output state */
1043 
1044     /* store the initial outgoing arcs to out_from[] and out_a[] */
1045     out_num_prev = 0;
1046     if (matchlen == 0) {
1047       /* set the word-beginning node */
1048       out_from[0] = wchmm->wordbegin[word];
1049       out_a[0] = 0.0;
1050       out_num_prev = 1;
1051     } else {
1052       /*printf("%d(%s)\n", word, wchmm->winfo->woutput[word]);*/
1053       /* on -iwsp, trailing sp is needed only when no phone will be created */
1054       get_outtrans_list(wchmm, matchword, add_to, out_from, out_a, &out_num_prev, wchmm->winfo->maxwn, (enable_iwsp && add_tail - add_head + 1 <= 0) ? TRUE : FALSE);
1055       /*printf("NUM=%d\n", out_num_prev);*/
1056     }
1057   } else { /*  end of multipath block */
1058     if (matchlen == 0) {
1059       if (wchmm->lmtype != LM_PROB || word != wchmm->winfo->head_silwid) {
1060 	/* index the new word-beginning node as startnode (old ststart) */
1061 	wchmm->startnode[wchmm->startnum] = n;
1062 	if (wchmm->category_tree) wchmm->start2wid[wchmm->startnum] = word;
1063 	/* expand data area if necessary */
1064 	if (++wchmm->startnum >= wchmm->maxstartnum) wchmm_extend_startnode(wchmm);
1065       }
1066     }
1067   }
1068 
1069   if (add_tail - add_head + 1 > 0) { /* there are new phones to be created */
1070       ntmp = n;
1071       for (j=add_head; j <= add_tail; j++) { /* for each new phones */
1072 	ltmp = wchmm->winfo->wseq[word][j];
1073 	ltmp_state_num = hmm_logical_state_num(ltmp);
1074 #ifdef PASS1_IWCD
1075 	if (wchmm->ccd_flag) {
1076 	  /* in the triphone lexicon tree, the last phone of a word has
1077 	     left-context cdset */
1078 	  if (wchmm->winfo->wlen[word] > 1 && j == wchmm->winfo->wlen[word] - 1) {
1079 	    if (wchmm->category_tree) {
1080 #ifdef USE_OLD_IWCD
1081 	      lcd = lcdset_lookup_by_hmmname(wchmm->hmminfo, ltmp->name);
1082 #else
1083 	      lcd = lcdset_lookup_with_category(wchmm, ltmp, wchmm->winfo->wton[word]);
1084 	      if (lcd == NULL) {
1085 		/* no category-aware cdset found.  This is case when no word
1086 		   can follow this word grammatically.
1087 		   so fallback to normal state */
1088 		jlog("WARNING: wchmm: no lcdset found for [%s::%04d], fallback to [%s]\n", ltmp->name, wchmm->winfo->wton[word], ltmp->name);
1089 		lcd = lcdset_lookup_by_hmmname(wchmm->hmminfo, ltmp->name);
1090 	      }
1091 #endif
1092 	    } else {
1093 	      lcd = lcdset_lookup_by_hmmname(wchmm->hmminfo, ltmp->name);
1094 	    }
1095 	    if (lcd == NULL) {
1096 	      jlog("ERROR: wchmm: at word #%d: no lcdset found for [%s]\n", word, ltmp->name);
1097 	      ok_p = FALSE;
1098 	    }
1099 	  }
1100 	}
1101 #endif /* PASS1_IWCD */
1102 	for (k = 1; k < ltmp_state_num - 1; k++) { /* for each state in the phone */
1103 	  /* set state output prob info */
1104 #ifdef PASS1_IWCD
1105 	  if (wchmm->ccd_flag) {
1106 	    /* output info of triphones needs special handling */
1107 	    if (wchmm->winfo->wlen[word] == 1) { /* word with only 1 phone */
1108 	      wchmm->outstyle[ntmp] = AS_LRSET;
1109 	      wchmm->state[ntmp].out.lrset = (LRC_INFO *)mybmalloc2(sizeof(LRC_INFO), &(wchmm->malloc_root));
1110 	      (wchmm->state[ntmp].out.lrset)->hmm       = ltmp;
1111 	      (wchmm->state[ntmp].out.lrset)->state_loc = k;
1112 	      if (wchmm->category_tree) {
1113 		(wchmm->state[ntmp].out.lrset)->category  = wchmm->winfo->wton[word];
1114 	      }
1115 	    } else if (j == 0) {	/* head phone of a word */
1116 	      wchmm->outstyle[ntmp] = AS_RSET;
1117 	      wchmm->state[ntmp].out.rset = (RC_INFO *)mybmalloc2(sizeof(RC_INFO), &(wchmm->malloc_root));
1118 	      (wchmm->state[ntmp].out.rset)->hmm       = ltmp;
1119 	      (wchmm->state[ntmp].out.rset)->state_loc = k;
1120 	    } else if (j == wchmm->winfo->wlen[word] - 1) { /* last phone of a word */
1121 	      wchmm->outstyle[ntmp] = AS_LSET;
1122 	      wchmm->state[ntmp].out.lset = &(lcd->stateset[k]);
1123 	    } else {
1124 	      wchmm->outstyle[ntmp] = AS_STATE;
1125 	      if (ltmp->is_pseudo) {
1126 		jlog("WARNING: wchmm: word-internal phone should not be pseudo\n");
1127 		put_voca(stdout, wchmm->winfo, word);
1128 		ok_p = FALSE;
1129 	      }
1130 	      wchmm->state[ntmp].out.state = ltmp->body.defined->s[k];
1131 	    }
1132 	  } else {
1133 	    /* monophone */
1134 	    if (ltmp->is_pseudo) {
1135 	      j_internal_error("wchmm_add_word: CDSET phoneme exist in monophone?\n");
1136 	      put_voca(stdout, wchmm->winfo, word);
1137 	      ok_p = FALSE;
1138 	    }
1139 	    wchmm->outstyle[ntmp] = AS_STATE;
1140 	    wchmm->state[ntmp].out.state = ltmp->body.defined->s[k];
1141 	  }
1142 #else  /* ~PASS1_IWCD */
1143 	  if (ltmp->is_pseudo) {
1144 	    j_internal_error("wchmm_add_word: CDSET phoneme exist in monophone?\n");
1145 	    put_voca(stdout, wchmm->winfo, word);
1146 	    ok_p = FALSE;
1147 	  }
1148 	  wchmm->state[ntmp].out = ltmp->body.defined->s[k];
1149 #endif /* PASS1_IWCD */
1150 
1151  	  /* initialize other info */
1152 	  acc_init(wchmm, ntmp);
1153 	  wchmm->stend[ntmp] = WORD_INVALID;
1154 	  if (! wchmm->hmminfo->multipath) {
1155 	    /* make transition arc from HMM transition info */
1156 	    for (ato = 1; ato < ltmp_state_num; ato++) {
1157 	      prob = (hmm_logical_trans(ltmp))->a[k][ato];
1158 	      if (prob != LOG_ZERO) {
1159 		  if (j == add_tail && k == ltmp_state_num - 2 && ato == ltmp_state_num - 1) {
1160 		    /* arc outside new part will be handled later */
1161 		  } else {
1162 		    add_wacc(wchmm, ntmp, prob, ntmp + ato - k);
1163 		  }
1164 		}
1165 	    }
1166 	  }
1167 
1168 	  ntmp++;
1169 	  /* expand wchmm if neccesary */
1170 	  if (ntmp >= wchmm->maxwcn) wchmm_extend(wchmm);
1171         } /* end of state loop */
1172       }	/* end of phone loop */
1173 
1174       if (wchmm->hmminfo->multipath) {
1175 
1176 	/* On multipath version, the skip transition should be handled! */
1177 
1178 	/* make transition arc from HMM transition info */
1179 	ntmp = n;
1180 	for (j = add_head; j <= add_tail; j++) {
1181 	  ltmp = wchmm->winfo->wseq[word][j];
1182 	  ltmp_state_num = hmm_logical_state_num(ltmp);
1183 	  out_num_next = 0;
1184 	  /* arc from initial state ... need arc expansion from precious phone */
1185 	  for (ato = 1; ato < ltmp_state_num; ato++) {
1186 	    prob = (hmm_logical_trans(ltmp))->a[0][ato];
1187 	    if (prob != LOG_ZERO) {
1188 	      /* expand arc from previous HMM */
1189 	      if (ato == ltmp_state_num - 1) {
1190 		/* to final state ... just register states for next expansion */
1191 		for(kkk=0; kkk<out_num_prev; kkk++) {
1192 		  out_from_next[out_num_next] = out_from[kkk];
1193 		  out_a_next[out_num_next] = out_a[kkk] + prob;
1194 		  out_num_next++;
1195 		}
1196 	      } else {
1197 		for(kkk=0; kkk<out_num_prev; kkk++) {
1198 		  add_wacc(wchmm, out_from[kkk], out_a[kkk] + prob, ntmp + ato - 1);
1199 		}
1200 	      }
1201 	    }
1202 	  } /* end of state loop */
1203 	  /* from outprob state */
1204 	  for(k = 1; k < ltmp_state_num - 1; k++) {
1205 	    for (ato = 1; ato < ltmp_state_num; ato++) {
1206 	      prob = (hmm_logical_trans(ltmp))->a[k][ato];
1207 	      if (prob != LOG_ZERO) {
1208 		if (ato == ltmp_state_num - 1) {
1209 		  /* to final state ... register states for next expansion */
1210 		  out_from_next[out_num_next] = ntmp;
1211 		  out_a_next[out_num_next] = prob;
1212 		  out_num_next++;
1213 		} else {
1214 		  add_wacc(wchmm, ntmp, prob, ntmp + ato - k);
1215 		}
1216 	      }
1217 	    }
1218 	    ntmp++;
1219 	  } /* end of state loop */
1220 	  /* swap out list for next phone */
1221 	  for(kkk=0;kkk<out_num_next;kkk++) {
1222 	    out_from[kkk] = out_from_next[kkk];
1223 	    out_a[kkk] = out_a_next[kkk];
1224 	  }
1225 	  out_num_prev = out_num_next;
1226 	}	/* end of phone loop */
1227       }	/* end of multipath block */
1228 
1229   } /* new phone node creation loop for this word */
1230 
1231 
1232   /*************************************/
1233   /* Short Pause appending (multipath) */
1234   /*************************************/
1235 
1236   /* if -iwsp, add noise model to the end of word at ntmp */
1237   if (wchmm->hmminfo->multipath && enable_iwsp && add_tail - add_head + 1 > 0) { /* there are new phones to be created */
1238     int ntmp_bak;
1239 
1240     /* set short pause state info */
1241     ntmp_bak = ntmp;
1242     if (wchmm->hmminfo->sp->is_pseudo) {
1243       for(k = 1;k < hmm_logical_state_num(wchmm->hmminfo->sp) - 1; k++) {
1244 	wchmm->outstyle[ntmp] = AS_LSET;
1245 	wchmm->state[ntmp].out.lset = &(wchmm->hmminfo->sp->body.pseudo->stateset[k]);
1246 	acc_init(wchmm, ntmp);
1247 	wchmm->stend[ntmp] = WORD_INVALID;
1248 	ntmp++;
1249 	if (ntmp >= wchmm->maxwcn) wchmm_extend(wchmm);
1250       }
1251     } else {
1252       for(k = 1;k < hmm_logical_state_num(wchmm->hmminfo->sp) - 1; k++) {
1253 	wchmm->outstyle[ntmp] = AS_STATE;
1254 	wchmm->state[ntmp].out.state = wchmm->hmminfo->sp->body.defined->s[k];
1255 	acc_init(wchmm, ntmp);
1256 	wchmm->stend[ntmp] = WORD_INVALID;
1257 	ntmp++;
1258 	if (ntmp >= wchmm->maxwcn) wchmm_extend(wchmm);
1259       }
1260     }
1261     ntmp = ntmp_bak;
1262     /* connect incoming arcs from previous phone */
1263     out_num_next = 0;
1264     for (ato = 1; ato < hmm_logical_state_num(wchmm->hmminfo->sp); ato++) {
1265       prob = hmm_logical_trans(wchmm->hmminfo->sp)->a[0][ato];
1266       if (prob != LOG_ZERO) {
1267 	/* to control short pause insertion, transition probability toward
1268 	 the word-end short pause will be given a penalty */
1269 	prob += wchmm->hmminfo->iwsp_penalty;
1270 	if (ato == hmm_logical_state_num(wchmm->hmminfo->sp) - 1) {
1271 	  /* model has a model skip transition, just inherit them to next */
1272 	  for(kkk=0; kkk<out_num_prev; kkk++) {
1273 	    out_from_next[out_num_next] = out_from[kkk];
1274 	    out_a_next[out_num_next] = out_a[kkk] + prob;
1275 	    out_num_next++;
1276 	  }
1277 	} else {
1278 	  /* connect incoming arcs from previous phone to this phone */
1279 	  for(kkk=0; kkk<out_num_prev; kkk++) {
1280 	    add_wacc(wchmm, out_from[kkk], out_a[kkk] + prob, ntmp + ato - 1);
1281 	  }
1282 	}
1283       }
1284     }
1285     /* if short pause model doesn't have a model skip transition, also add it */
1286     if (hmm_logical_trans(wchmm->hmminfo->sp)->a[0][hmm_logical_state_num(wchmm->hmminfo->sp)-1] == LOG_ZERO) {
1287       /* to make insertion sp model to have no effect on the original path,
1288 	 the skip transition probability should be 0.0 (=100%) */
1289       prob = 0.0;
1290       for(kkk=0; kkk<out_num_prev; kkk++) {
1291 	out_from_next[out_num_next] = out_from[kkk];
1292 	out_a_next[out_num_next] = out_a[kkk] + prob;
1293 	out_num_next++;
1294       }
1295     }
1296     /* connect arcs within model, and store new outgoing arcs for wordend node */
1297     for (k = 1; k < hmm_logical_state_num(wchmm->hmminfo->sp) - 1; k++) {
1298       for (ato = 1; ato < hmm_logical_state_num(wchmm->hmminfo->sp); ato++) {
1299 	prob = hmm_logical_trans(wchmm->hmminfo->sp)->a[k][ato];
1300 	if (prob != LOG_ZERO) {
1301 	  if (ato == hmm_logical_state_num(wchmm->hmminfo->sp) - 1) {
1302 	    out_from_next[out_num_next] = ntmp;
1303 	    out_a_next[out_num_next] = prob;
1304 	    out_num_next++;
1305 	  } else {
1306 	    add_wacc(wchmm, ntmp, prob, ntmp + ato - k);
1307 	  }
1308 	}
1309       }
1310       ntmp++;
1311     }
1312     /* swap work area for next */
1313     for(kkk=0;kkk<out_num_next;kkk++) {
1314       out_from[kkk] = out_from_next[kkk];
1315       out_a[kkk] = out_a_next[kkk];
1316     }
1317     out_num_prev = out_num_next;
1318 
1319   } /* end of inter-word short pause appending block */
1320 
1321   /* make mapping: word <-> node on wchmm */
1322   for (j=0;j<word_len;j++) {
1323     if (j < add_head) {	/* shared part */
1324       wchmm->offset[word][j] = wchmm->offset[matchword][j];
1325     } else if (add_tail < j) { /* shared tail part (should not happen..) */
1326       wchmm->offset[word][j] = wchmm->offset[matchword][j+(matchword_len-word_len)];
1327     } else {			/* newly created part */
1328       wchmm->offset[word][j] = n;
1329       n += hmm_logical_state_num(wchmm->winfo->wseq[word][j]) - 2;
1330     }
1331   }
1332 
1333 
1334   if (wchmm->hmminfo->multipath) {
1335     /* create word-end node */
1336 
1337     /* paranoia check if the short-pause addition has been done well */
1338     if (enable_iwsp && add_tail - add_head + 1 > 0) {
1339       n += hmm_logical_state_num(wchmm->hmminfo->sp) - 2;
1340       if (n != ntmp) j_internal_error("wchmm_add_word: cannot match\n");
1341     }
1342 
1343     /* create word-end node */
1344     wchmm->wordend[word] = n;	/* tail node of 'word' is 'n' */
1345     wchmm->stend[n] = word;	/* node 'k' is a tail node of 'word' */
1346     acc_init(wchmm, n);
1347     wchmm->state[n].out.state = NULL;
1348 
1349     /* connect the final outgoing arcs in out_from[] to the word end node */
1350     for(k = 0; k < out_num_prev; k++) {
1351       add_wacc(wchmm, out_from[k], out_a[k], n);
1352     }
1353     n++;
1354     if (n >= wchmm->maxwcn) wchmm_extend(wchmm);
1355 
1356     if (matchlen == 0) {
1357       /* check if the new word has whole word-skipping transition */
1358       /* (use out_from and out_num_prev temporary) */
1359       out_num_prev = 0;
1360       get_outtrans_list(wchmm, word, word_len-1, out_from, out_a, &out_num_prev, wchmm->winfo->maxwn, enable_iwsp);
1361       for(k=0;k<out_num_prev;k++) {
1362 	if (out_from[k] == wchmm->wordbegin[word]) {
1363 	  jlog("ERROR: *** ERROR: WORD SKIPPING TRANSITION NOT ALLOWED ***\n");
1364 	  jlog("ERROR:   Word id=%d (%s[%s]) has \"word skipping transition\".\n", word, wchmm->winfo->wname[word], wchmm->winfo->woutput[word]);
1365 	  jlog("ERROR:   All HMMs in the word:\n    ");
1366 	  for(kkk=0;kkk<wchmm->winfo->wlen[word];kkk++) {
1367 	    jlog("%s ", wchmm->winfo->wseq[word][kkk]->name);
1368 	  }
1369 	  jlog("\n");
1370 	  jlog("ERROR:  has transitions from initial state to final state.\n");
1371 	  jlog("ERROR:  This type of word skipping is not supported.\n");
1372 	  ok_p = FALSE;
1373 	}
1374       }
1375     }
1376 
1377     wchmm->n = n;
1378 
1379   } else {
1380 
1381     wchmm->n = n;
1382     k = wchmm->offset[word][word_len-1] + hmm_logical_state_num(wchmm->winfo->wseq[word][word_len-1])-2 -1;
1383     wchmm->wordend[word] = k;	/* tail node of 'word' is 'k' */
1384     wchmm->stend[k] = word;	/* node 'k' is a tail node of 'word' */
1385 
1386     if (matchlen != 0 && add_tail - add_head + 1 > 0) {
1387       /* new part has been created in the above procedure: */
1388       /* now make link from shared part to the new part */
1389       wchmm_link_subword(wchmm, matchword,add_to,word,add_head);
1390     }
1391 
1392   }
1393 
1394   return(ok_p);
1395 
1396 }
1397 
1398 /*************************************************************/
1399 /**** parse whole structure (after wchmm has been built) *****/
1400 /*************************************************************/
1401 
1402 /**
1403  * <JA>
1404  * �ڹ�¤���������������ñ��ν�ü���֤��鳰�ؤμ����ܳ�Ψ�Υꥹ�Ȥ��������.
1405  * (non multipath)
1406  *
1407  * @param wchmm [i/o] �ڹ�¤������
1408  * </JA>
1409  * <EN>
1410  * Scan the lexicon tree to make list of emission probability from the word end
1411  * state. (non multipath)
1412  *
1413  * @param wchmm [i/o] tree lexicon
1414  * </EN>
1415  */
1416 static void
wchmm_calc_wordend_arc(WCHMM_INFO * wchmm)1417 wchmm_calc_wordend_arc(WCHMM_INFO *wchmm)
1418 {
1419   WORD_ID w;
1420   HTK_HMM_Trans *tr;
1421   LOGPROB a;
1422 
1423   for (w=0;w<wchmm->winfo->num;w++) {
1424     tr = hmm_logical_trans(wchmm->winfo->wseq[w][wchmm->winfo->wlen[w]-1]);
1425     a = tr->a[tr->statenum-2][tr->statenum-1];
1426     wchmm->wordend_a[w] = a;
1427   }
1428 }
1429 
1430 #ifdef SEPARATE_BY_UNIGRAM
1431 
1432 /********************************************************************/
1433 /****** for separation (linearization) of high-frequent words *******/
1434 /********************************************************************/
1435 
1436 /**
1437  * <JA>
1438  * unigram��Ψ�ǥ����Ȥ��뤿��� qsort ������Хå��ؿ�.
1439  *
1440  * @param a [in] ����1
1441  * @param b [in] ����2
1442  *
1443  * @return �黻�η�̤������֤�.
1444  * </JA>
1445  * <EN>
1446  * qsort callback function to sort unigram values.
1447  *
1448  * @param a [in] element #1
1449  * @param b [in] element #2
1450  *
1451  * @return the result of comparison.
1452  * </EN>
1453  */
1454 static int
compare_prob(LOGPROB * a,LOGPROB * b)1455 compare_prob(LOGPROB *a, LOGPROB *b)
1456 {
1457   if (*a < *b)  return (1);
1458   if (*a > *b)  return (-1);
1459   return(0);
1460 }
1461 
1462 /**
1463  * <JA>
1464  * 1-gram�������ξ�� N ���ܤ��ͤ����.
1465  *
1466  * @param winfo [in] ñ�켭��
1467  * @param n [in] ������
1468  *
1469  * @return ��� N ���ܤ� uni-gram ��Ψ���ͤ��֤�.
1470  * </JA>
1471  * <EN>
1472  * Get the Nth-best unigram probability from all words.
1473  *
1474  * @param winfo [in] word dictionary
1475  * @param n [in] required rank
1476  *
1477  * @return the Nth-best unigram probability.
1478  * </EN>
1479  */
1480 static LOGPROB
get_nbest_uniprob(WCHMM_INFO * wchmm,int n)1481 get_nbest_uniprob(WCHMM_INFO *wchmm, int n)
1482 {
1483   LOGPROB *u_p;
1484   WORD_ID w;
1485   LOGPROB x;
1486   WORD_INFO *winfo;
1487   NGRAM_INFO *ngram;
1488 
1489   winfo = wchmm->winfo;
1490   ngram = wchmm->ngram;
1491 
1492   if (n < 1) n = 1;
1493   if (n > winfo->num) n = winfo->num;
1494 
1495   /* store all unigram probability to u_p[] */
1496   u_p = (LOGPROB *)mymalloc(sizeof(LOGPROB) * winfo->num);
1497   for(w=0;w<winfo->num;w++) {
1498     if (ngram) {
1499       x = uni_prob(ngram, winfo->wton[w])
1500 #ifdef CLASS_NGRAM
1501 	+ winfo->cprob[w]
1502 #endif
1503 	;
1504     } else {
1505       x = LOG_ZERO;
1506     }
1507     if (wchmm->lmvar == LM_NGRAM_USER) {
1508       x = (*(wchmm->uni_prob_user))(wchmm->winfo, w, x);
1509     }
1510     u_p[w] = x;
1511   }
1512 
1513   /* sort them downward */
1514   qsort(u_p, winfo->num, sizeof(LOGPROB),
1515 	(int (*)(const void *,const void *))compare_prob);
1516 
1517   /* return the Nth value */
1518   x = u_p[n-1];
1519   free(u_p);
1520   return(x);
1521 }
1522 
1523 #endif
1524 
1525 /**********************************************************/
1526 /****** MAKE WCHMM (LEXICON TREE) --- main function *******/
1527 /**********************************************************/
1528 
1529 #define COUNT_STEP 500         ///< Word count step for debug progress output
1530 
1531 /**
1532  * <JA>
1533  * Ϳ����줿ñ�켭��ȸ����ǥ뤫���ڹ�¤��������ۤ���. ���δؿ���
1534  * �������٤���Julian��"-oldtree"���ץ���������Τ߻��Ѥ���ޤ�. ���ץ����
1535  * �����������Julius�Ǥ������ build_wchmm2() ���Ѥ����ޤ�.
1536  *
1537  * @param wchmm [i/o] �ڹ�¤������
1538  * @param lmconf [in] �����ǥ�(LM)����ѥ�᡼��
1539  * </JA>
1540  * <EN>
1541  * Build a tree lexicon from given word dictionary and language model.
1542  * This function is slow and only used when "-oldtree" option is specified
1543  * in Julian.  Julian without that option and Julius uses build_wchmm2()
1544  * instead of this.
1545  *
1546  * @param wchmm [i/o] lexicon tree
1547  * @param lmconf [in] language model (LM) configuration parameters
1548  * </EN>
1549  * @callgraph
1550  * @callergraph
1551  */
1552 boolean
build_wchmm(WCHMM_INFO * wchmm,JCONF_LM * lmconf)1553 build_wchmm(WCHMM_INFO *wchmm, JCONF_LM *lmconf)
1554 {
1555   int i,j;
1556   int matchword=0, sharelen=0, maxsharelen=0;
1557   int num_duplicated;
1558 #ifdef SEPARATE_BY_UNIGRAM
1559   LOGPROB separate_thres;
1560   LOGPROB p;
1561 #endif
1562   boolean ok_p;
1563 
1564   /* lingustic infos must be set before build_wchmm() is called */
1565   /* check if necessary lingustic info is already assigned (for debug) */
1566   if (wchmm->winfo == NULL
1567       || (wchmm->lmvar == LM_NGRAM && wchmm->ngram == NULL)
1568       || (wchmm->lmvar == LM_DFA_GRAMMAR && wchmm->dfa == NULL)
1569       ) {
1570     jlog("ERROR: wchmm: linguistic info not available!!\n");
1571     return FALSE;
1572   }
1573 
1574   ok_p = TRUE;
1575 
1576 #ifdef SEPARATE_BY_UNIGRAM
1577   /* ���[separate_wnum]���ܤ�1-gram����������� */
1578   /* 1-gram�������������Ͱʾ�Τ�Τ��ڤ���ʬ���� */
1579   separate_thres = get_nbest_uniprob(wchmm, lmconf->separate_wnum);
1580 #endif
1581 
1582 #ifdef PASS1_IWCD
1583 #ifndef USE_OLD_IWCD
1584   if (wchmm->category_tree) {
1585     if (wchmm->ccd_flag) {
1586       /* ���ƤΥ��ƥ���ID�դ� lcd_set ����� */
1587       lcdset_register_with_category_all(wchmm);
1588     }
1589   }
1590 #endif
1591 #endif /* PASS1_IWCD */
1592 
1593 
1594   /* wchmm������ */
1595   wchmm_init(wchmm);
1596 
1597   /* �������ꥻ�å� */
1598   wchmm->separated_word_count=0;
1599 
1600   jlog("STAT: wchmm: Building HMM lexicon tree (left-to-right)\n");
1601   for (i=0;i<wchmm->winfo->num;i++) {
1602 
1603     if (wchmm->lmtype == LM_PROB) {
1604       if (i == wchmm->winfo->head_silwid || i == wchmm->winfo->tail_silwid) {
1605 	/* ��Ƭ/������̵����ǥ���ڹ�¤��������
1606 	 * ��Ƭ��̵��ñ�����Ƭ�ؤ����ܡ�����ñ���������������ܤϺ��ʤ�*/
1607 	/* sharelen=0�Ǥ��Τޤ� */
1608 	if (wchmm_add_word(wchmm, i, 0, 0, lmconf->enable_iwsp) == FALSE) {
1609 	  jlog("ERROR: wchmm: failed to add word #%d to lexicon tree\n");
1610 	  ok_p = FALSE;
1611 	}
1612 	continue;
1613       }
1614 #ifndef NO_SEPARATE_SHORT_WORD
1615       if (wchmm->winfo->wlen[i] <= SHORT_WORD_LEN) {
1616 	/* Ĺ����û��ñ����ڹ�¤�����ʤ�(�����Ǥ�1����) */
1617 	/* sharelen=0�Ǥ��Τޤ� */
1618 	if (wchmm_add_word(wchmm, i, 0, 0, lmconf->enable_iwsp) == FALSE) {
1619 	  jlog("ERROR: wchmm: failed to add word #%d to lexicon tree\n");
1620 	  ok_p = FALSE;
1621 	}
1622 	wchmm->separated_word_count++;
1623 	continue;
1624       }
1625 #endif
1626 #ifdef SEPARATE_BY_UNIGRAM
1627       if (wchmm->ngram) {
1628 	p = uni_prob(wchmm->ngram, wchmm->winfo->wton[i])
1629 #ifdef CLASS_NGRAM
1630 	  + wchmm->winfo->cprob[i]
1631 #endif
1632 	  ;
1633       } else {
1634 	p = LOG_ZERO;
1635       }
1636       if (wchmm->lmvar == LM_NGRAM_USER) {
1637 	p = (*(wchmm->uni_prob_user))(wchmm->winfo, i, p);
1638       }
1639       if (p >= separate_thres && wchmm->separated_word_count < lmconf->separate_wnum) {
1640 	/* ���٤ι⤤ñ����ڹ�¤�����ʤ� */
1641 	/* separate_thres �Ͼ��separate_wnum���ܤΥ����� */
1642 	if (wchmm_add_word(wchmm, i, 0, 0, lmconf->enable_iwsp) == FALSE) {
1643 	  jlog("ERROR: wchmm: failed to add word #%d to lexicon tree\n");
1644 	  ok_p = FALSE;
1645 	}
1646 	wchmm->separated_word_count++;
1647 	continue;
1648       }
1649 #endif
1650     }
1651 
1652     /* �Ǥ�Ĺ�����Ǥ�ͭ�����ñ���õ�� */
1653     maxsharelen=0;
1654     for (j=0;j<i;j++) {
1655       if (wchmm->category_tree  && wchmm->lmtype == LM_DFA) {
1656 	if (wchmm->winfo->wton[i] != wchmm->winfo->wton[j]) continue;
1657       }
1658       sharelen = wchmm_check_match(wchmm->winfo, i, j);
1659       if (sharelen == wchmm->winfo->wlen[i] && sharelen == wchmm->winfo->wlen[j]) {
1660        /* word ��Ʊ���줬¸�ߤ��� */
1661        /* ɬ�������Ĺ���Ǥ��ꡤ��ʣ������Ȥ����뤿�ᤳ����ȴ���� */
1662        maxsharelen = sharelen;
1663        matchword = j;
1664        break;
1665       }
1666       if (sharelen > maxsharelen) {
1667        matchword = j;
1668        maxsharelen = sharelen;
1669       }
1670     }
1671     if (wchmm_add_word(wchmm, i, maxsharelen, matchword, lmconf->enable_iwsp) == FALSE) {
1672       jlog("ERROR: wchmm: failed to add word #%d to lexicon tree\n");
1673       ok_p = FALSE;
1674     }
1675   }
1676 
1677 #if 0
1678   /* �ڹ�¤����ʤ� */
1679   for (i=0;i<wchmm->winfo->num;i++) {
1680     if (wchmm_add_word(wchmm, i, 0, 0, lmconf->enable_iwsp) == FALSE) {
1681       jlog("ERROR: wchmm: failed to add word #%d to lexicon tree\n");
1682       ok_p = FALSE;
1683     }
1684   }
1685 #endif
1686   jlog("STAT:  %5d words ended     (%6d nodes)\n",i,wchmm->n);
1687 
1688   if (! wchmm->hmminfo->multipath) {
1689     /* Ʊ�첻�Ƿ�������ñ��Ʊ�Τ� leaf node ��2�Ų����ƶ��̤��� */
1690     num_duplicated = wchmm_duplicate_leafnode(wchmm);
1691     jlog("STAT:  %d leaf nodes are made unshared\n", num_duplicated);
1692 
1693     /* ñ��ν�ü���鳰�ؤ����ܳ�Ψ����Ƥ��� */
1694     wchmm_calc_wordend_arc(wchmm);
1695   }
1696 
1697   /* wchmm��������������å����� */
1698   check_wchmm(wchmm);
1699 
1700   /* factoring�Ѥ˳ƾ��֤˸�³ñ��Υꥹ�Ȥ��ղä��� */
1701   if (!wchmm->category_tree) {
1702 
1703 #ifdef UNIGRAM_FACTORING
1704     if (wchmm->lmtype == LM_PROB) {
1705       /* Ʊ��������ä�factoring�ͤ�׻� */
1706       make_successor_list_unigram_factoring(wchmm);
1707       jlog("STAT:  1-gram factoring values has been pre-computed\n");
1708     } else {
1709       make_successor_list(wchmm);
1710     }
1711 #else
1712     make_successor_list(wchmm);
1713 #endif /* UNIGRAM_FACTORING */
1714 
1715     if (wchmm->hmminfo->multipath) {
1716       /* ���ۤ��줿 factoring ��������å����ܤ����ʸƬʸˡ�Ρ��ɤ˥��ԡ� */
1717       adjust_sc_index(wchmm);
1718     }
1719 
1720 #ifdef UNIGRAM_FACTORING
1721     if (wchmm->lmtype == LM_PROB) {
1722       /* ñ���LM����å��夬ɬ�פʥΡ��ɤΥꥹ�Ȥ��� */
1723       make_iwcache_index(wchmm);
1724     }
1725 #endif /* UNIGRAM_FACTORING */
1726 
1727     /* sclist2node is no longer used */
1728     if (wchmm->sclist2node != NULL) {
1729       free(wchmm->sclist2node);
1730       wchmm->sclist2node = NULL;
1731     }
1732 
1733   }
1734 
1735   jlog("STAT: done\n");
1736 
1737   return ok_p;
1738 }
1739 
1740 /**
1741  * <JA>
1742  * Ϳ����줿ñ�켭��ȸ����ǥ뤫���ڹ�¤��������ۤ���.
1743  * ���δؿ��� bulid_wchmm() ��Ʊ��������Ԥ��ޤ�����
1744  * �ǽ��ñ�������ǥ����Ȥ��Ʋ�����λ������ñ����¤٤뤿�ᡤ
1745  * ����®���ڹ�¤����Ԥ����Ȥ��Ǥ���. �Ȥ��˥��ץ���������ʤ�
1746  * �¤ꡤJulius/Julian�ǤϤ����餬�Ѥ�����.
1747  *
1748  * @param wchmm [i/o] �ڹ�¤������
1749  * @param lmconf [in] �����ǥ�(LM)����ѥ�᡼��
1750  * </JA>
1751  * <EN>
1752  * Build a tree lexicon from given word dictionary and language model.
1753  * This function does the same job as build_wchmm(), but it is much
1754  * faster because finding of the longest matched word to an adding word
1755  * is done by first sorting all the words in the dictoinary by their phoneme
1756  * sequence order.  This function will be used instead of build_wchmm()
1757  * by default.
1758  *
1759  * @param wchmm [i/o] lexicon tree
1760  * @param lmconf [in] language model (LM) configuration parameters
1761  * </EN>
1762  * @callgraph
1763  * @callergraph
1764  */
1765 boolean
build_wchmm2(WCHMM_INFO * wchmm,JCONF_LM * lmconf)1766 build_wchmm2(WCHMM_INFO *wchmm, JCONF_LM *lmconf)
1767 {
1768   int i,j, last_i;
1769   int num_duplicated;
1770   WORD_ID *windex;
1771 #ifdef SEPARATE_BY_UNIGRAM
1772   LOGPROB separate_thres;
1773   LOGPROB p;
1774 #endif
1775   boolean ok_p;
1776   boolean ret;
1777 
1778   /* lingustic infos must be set before build_wchmm() is called */
1779   /* check if necessary lingustic info is already assigned (for debug) */
1780   if (wchmm->winfo == NULL
1781       || (wchmm->lmvar == LM_NGRAM && wchmm->ngram == NULL)
1782       || (wchmm->lmvar == LM_DFA_GRAMMAR && wchmm->dfa == NULL)
1783       ) {
1784     jlog("ERROR: wchmm: linguistic info not available!!\n");
1785     return FALSE;
1786   }
1787 
1788   ok_p = TRUE;
1789 
1790   wchmm->separated_word_count = 0;
1791 
1792   jlog("STAT: Building HMM lexicon tree\n");
1793 
1794   if (wchmm->lmtype == LM_PROB) {
1795 #ifdef SEPARATE_BY_UNIGRAM
1796     /* compute score threshold beforehand to separate words from tree */
1797     /* here we will separate best [separate_wnum] words from tree */
1798     separate_thres = get_nbest_uniprob(wchmm, lmconf->separate_wnum);
1799 #endif
1800   }
1801 
1802 #ifdef PASS1_IWCD
1803 #ifndef USE_OLD_IWCD
1804   if (wchmm->category_tree) {
1805     if (wchmm->ccd_flag) {
1806       /* when Julian mode (category-tree) and triphone is used,
1807 	 make all category-indexed context-dependent phone set (cdset) here */
1808       /* these will be assigned on the last phone of each word on tree */
1809       lcdset_register_with_category_all(wchmm);
1810     }
1811   }
1812 #endif
1813 #endif /* PASS1_IWCD */
1814 
1815  /* initialize wchmm */
1816   wchmm_init(wchmm);
1817 
1818   /* make sorted word index ordered by phone sequence */
1819   windex = (WORD_ID *)mymalloc(sizeof(WORD_ID) * wchmm->winfo->num);
1820   for(i=0;i<wchmm->winfo->num;i++) windex[i] = i;
1821 
1822   if (wchmm->category_tree && wchmm->lmtype == LM_DFA) {
1823 
1824     /* sort by category -> sort by word ID in each category */
1825     wchmm_sort_idx_by_category(wchmm->winfo, windex, wchmm->winfo->num);
1826     {
1827       int last_cate;
1828       last_i = 0;
1829       last_cate = wchmm->winfo->wton[windex[0]];
1830       for(i = 1;i<wchmm->winfo->num;i++) {
1831 	if (wchmm->winfo->wton[windex[i]] != last_cate) {
1832 	  wchmm_sort_idx_by_wseq(wchmm->winfo, windex, last_i, i - last_i);
1833 	  last_cate = wchmm->winfo->wton[windex[i]];
1834 	  last_i = i;
1835 	}
1836       }
1837       wchmm_sort_idx_by_wseq(wchmm->winfo, windex, last_i, wchmm->winfo->num - last_i);
1838     }
1839 
1840   } else {
1841 
1842     /* sort by word ID for whole vocabulary */
1843     wchmm_sort_idx_by_wseq(wchmm->winfo, windex, 0, wchmm->winfo->num);
1844 
1845   }
1846 
1847 /*
1848  *   {
1849  *     int i,w;
1850  *     for(i=0;i<wchmm->winfo->num;i++) {
1851  *	 w = windex[i];
1852  *	 printf("%d: cate=%4d wid=%4d %s\n",i, wchmm->winfo->wton[w], w, wchmm->winfo->woutput[w]);
1853  *     }
1854  *   }
1855  */
1856 
1857   /* incrementaly add words to lexicon tree */
1858   /* now for each word, the previous word (last_i) is always the most matched one */
1859   last_i = WORD_INVALID;
1860   for (j=0;j<wchmm->winfo->num;j++) {
1861     i = windex[j];
1862 
1863     if (wchmm->lmtype == LM_PROB) {
1864 
1865       /* start/end silence word should not be shared */
1866       if (i == wchmm->winfo->head_silwid || i == wchmm->winfo->tail_silwid) {
1867 	/* add whole word as new (sharelen=0) */
1868 	if (wchmm_add_word(wchmm, i, 0, 0, lmconf->enable_iwsp) == FALSE) {
1869 	  jlog("ERROR: wchmm: failed to add word #%d to lexicon tree\n");
1870 	  ok_p = FALSE;
1871 	}
1872 	continue;
1873       }
1874 #ifndef NO_SEPARATE_SHORT_WORD
1875       /* separate short words from tree */
1876       if (wchmm->winfo->wlen[i] <= SHORT_WORD_LEN) {
1877 	if (wchmm_add_word(wchmm, i, 0, 0, lmconf->enable_iwsp) == FALSE) {
1878 	  jlog("ERROR: wchmm: failed to add word #%d to lexicon tree\n");
1879 	  ok_p = FALSE;
1880 	}
1881 	wchmm->separated_word_count++;
1882 	continue;
1883       }
1884 #endif
1885 #ifdef SEPARATE_BY_UNIGRAM
1886       if (wchmm->ngram) {
1887 	p = uni_prob(wchmm->ngram, wchmm->winfo->wton[i])
1888 #ifdef CLASS_NGRAM
1889 	  + wchmm->winfo->cprob[i]
1890 #endif
1891 	  ;
1892       } else {
1893 	p = LOG_ZERO;
1894       }
1895       if (wchmm->lmvar == LM_NGRAM_USER) {
1896 	p = (*(wchmm->uni_prob_user))(wchmm->winfo, i, p);
1897       }
1898       /* separate high-frequent words from tree (threshold = separate_thres) */
1899       if (p >= separate_thres && wchmm->separated_word_count < lmconf->separate_wnum) {
1900 	if (wchmm_add_word(wchmm, i, 0, 0, lmconf->enable_iwsp) == FALSE) {
1901 	  jlog("ERROR: wchmm: failed to add word #%d to lexicon tree\n");
1902 	  ok_p = FALSE;
1903 	}
1904 	wchmm->separated_word_count++;
1905 	continue;
1906       }
1907 #endif
1908     }
1909 
1910     if (last_i == WORD_INVALID) { /* first word */
1911       ret = wchmm_add_word(wchmm, i, 0, 0, lmconf->enable_iwsp);
1912     } else {
1913       /* the previous word (last_i) is always the most matched one */
1914       if (wchmm->category_tree && wchmm->lmtype == LM_DFA) {
1915 	if (wchmm->winfo->wton[i] != wchmm->winfo->wton[last_i]) {
1916 	  ret = wchmm_add_word(wchmm, i, 0, 0, lmconf->enable_iwsp);
1917 	} else {
1918 	  ret = wchmm_add_word(wchmm, i, wchmm_check_match(wchmm->winfo, i, last_i), last_i, lmconf->enable_iwsp);
1919 	}
1920       } else {
1921 	ret = wchmm_add_word(wchmm, i, wchmm_check_match(wchmm->winfo, i, last_i), last_i, lmconf->enable_iwsp);
1922       }
1923     }
1924     if (ret == FALSE) {
1925       jlog("ERROR: wchmm: failed to add word #%d to lexicon tree\n");
1926       ok_p = FALSE;
1927     }
1928     last_i = i;
1929 
1930   } /* end of add word loop */
1931 
1932   /*j_printerr("\r %5d words ended     (%6d nodes)\n",j,wchmm->n);*/
1933 
1934   /* free work area */
1935   free(windex);
1936 
1937   if (wchmm->hmminfo->multipath) {
1938     jlog("STAT: lexicon size: %d nodes\n", wchmm->n);
1939   } else {
1940     /* duplicate leaf nodes of homophone/embedded words */
1941     jlog("STAT: lexicon size: %d", wchmm->n);
1942     num_duplicated = wchmm_duplicate_leafnode(wchmm);
1943     jlog("+%d=%d\n", num_duplicated, wchmm->n);
1944   }
1945 
1946   if (! wchmm->hmminfo->multipath) {
1947     /* calculate transition probability of word end node to outside */
1948     wchmm_calc_wordend_arc(wchmm);
1949   }
1950 
1951   /* check wchmm coherence (internal debug) */
1952   check_wchmm(wchmm);
1953 
1954   /* make successor list for all branch nodes for N-gram factoring */
1955   if (!wchmm->category_tree) {
1956 
1957 #ifdef UNIGRAM_FACTORING
1958     if (wchmm->lmtype == LM_PROB) {
1959       /* for 1-gram factoring, we can compute the values before search */
1960        make_successor_list_unigram_factoring(wchmm);
1961        jlog("STAT:  1-gram factoring values has been pre-computed\n");
1962     } else {
1963       make_successor_list(wchmm);
1964     }
1965 #else
1966     make_successor_list(wchmm);
1967 #endif /* UNIGRAM_FACTORING */
1968     if (wchmm->hmminfo->multipath) {
1969       /* Copy the factoring data according to the skip transitions and startword nodes */
1970       adjust_sc_index(wchmm);
1971     }
1972 #ifdef UNIGRAM_FACTORING
1973     if (wchmm->lmtype == LM_PROB) {
1974       /* make list of start nodes that needs inter-word LM cache */
1975       make_iwcache_index(wchmm);
1976     }
1977 #endif /* UNIGRAM_FACTORING */
1978 
1979     /* sclist2node is no longer used */
1980     if (wchmm->sclist2node != NULL) {
1981       free(wchmm->sclist2node);
1982       wchmm->sclist2node = NULL;
1983     }
1984 
1985   }
1986 
1987   //jlog("STAT: done\n");
1988 
1989 #ifdef WCHMM_SIZE_CHECK
1990   if (debug2_flag) {
1991     /* detailed check of lexicon tree size (inaccurate!) */
1992     jlog("STAT: --- memory size of word lexicon ---\n");
1993     jlog("STAT: wchmm: %d words, %d nodes\n", wchmm->winfo->num, wchmm->n);
1994     jlog("STAT: %9d bytes: wchmm->state[node] (exclude ac, sc)\n", sizeof(WCHMM_STATE) * wchmm->n);
1995     {
1996       int count1 = 0;
1997       int count2 = 0;
1998       int count3 = 0;
1999       for(i=0;i<wchmm->n;i++) {
2000 	if (wchmm->self_a[i] != LOG_ZERO) count1++;
2001 	if (wchmm->next_a[i] != LOG_ZERO) count2++;
2002 	if (wchmm->ac[i] != NULL) count3++;
2003       }
2004       jlog("STAT: %9d bytes: wchmm->self_a[node] (%4.1f%% filled)\n", sizeof(LOGPROB) * wchmm->n, 100.0 * count1 / (float)wchmm->n);
2005       jlog("STAT: %9d bytes: wchmm->next_a[node] (%4.1f%% filled)\n", sizeof(LOGPROB) * wchmm->n, 100.0 * count2 / (float)wchmm->n);
2006       jlog("STAT: %9d bytes: wchmm->ac[node] (%4.1f%% used)\n", sizeof(A_CELL2 *) * wchmm->n, 100.0 * count3 / (float)wchmm->n);
2007     }
2008     jlog("STAT: %9d bytes: wchmm->stend[node]\n", sizeof(WORD_ID) * wchmm->n);
2009     {
2010       int w,count;
2011       count = 0;
2012       for(w=0;w<wchmm->winfo->num;w++) {
2013 	count += wchmm->winfo->wlen[w] * sizeof(int) + sizeof(int *);
2014       }
2015       jlog("STAT: %9d bytes: wchmm->offset[w][]\n", count);
2016     }
2017     if (wchmm->hmminfo->multipath) {
2018       jlog("STAT: %9d bytes: wchmm->wordbegin[w]\n", wchmm->winfo->num * sizeof(int));
2019     }
2020     jlog("STAT: %9d bytes: wchmm->wordend[w]\n", wchmm->winfo->num * sizeof(int));
2021     jlog("STAT: %9d bytes: wchmm->startnode[]\n", wchmm->startnum * sizeof(int));
2022     if (wchmm->category_tree) {
2023       jlog("STAT: %9d bytes: wchmm->start2wid[]\n", wchmm->startnum * sizeof(WORD_ID));
2024     }
2025 #ifdef UNIGRAM_FACTORING
2026     if (wchmm->lmtype == LM_PROB) {
2027       jlog("STAT: %9d bytes: wchmm->start2isolate[]\n", wchmm->isolatenum * sizeof(int));
2028     }
2029 #endif
2030     if (!wchmm->hmminfo->multipath) {
2031       jlog("STAT: %9d bytes: wchmm->wordend_a[]\n", wchmm->winfo->num * sizeof(LOGPROB));
2032     }
2033 #ifdef PASS1_IWCD
2034     jlog("STAT: %9d bytes: wchmm->outstyle[]\n", wchmm->n * sizeof(unsigned char));
2035     {
2036       int c;
2037       c = 0;
2038       for(i=0;i<wchmm->n;i++) {
2039 	switch(wchmm->outstyle[i]) {
2040 	case AS_RSET:
2041 	  c += sizeof(RC_INFO);
2042 	  break;
2043 	case AS_LRSET:
2044 	  c += sizeof(LRC_INFO);
2045 	  break;
2046       }
2047       }
2048       if (c > 0) jlog("STAT: %9d bytes: wchmm->out (RC_INFO / LRC_INFO)\n", c);
2049     }
2050 #endif
2051     if (!wchmm->category_tree) {
2052       jlog("STAT: %9d bytes: wchmm->sclist[]\n", wchmm->scnum * sizeof(S_CELL *));
2053       jlog("STAT: %9d bytes: wchmm->sclist2node[]\n", wchmm->scnum * sizeof(int));
2054 #ifdef UNIGRAM_FACTORING
2055       if (wchmm->lmtype == LM_PROB) {
2056 	jlog("STAT: %9d bytes: wchmm->fscore[]\n", wchmm->fsnum * sizeof(LOGPROB));
2057       }
2058 #endif
2059     }
2060 
2061     {
2062       int count, n;
2063       A_CELL2 *ac;
2064       count = 0;
2065       for(n=0;n<wchmm->n;n++) {
2066 	for(ac=wchmm->ac[n];ac;ac=ac->next) {
2067 	  count += sizeof(A_CELL2);
2068 	}
2069       }
2070       jlog("STAT: %9d bytes: A_CELL2\n", count);
2071     }
2072     if (!wchmm->category_tree) {
2073       jlog("STAT: %9d bytes: sclist\n", wchmm->scnum * sizeof(S_CELL *));
2074       jlog("STAT: %9d bytes: sclist2node\n", wchmm->scnum * sizeof(int));
2075     }
2076 
2077   }
2078 
2079 #endif /* WCHMM_SIZE_CHECK */
2080 
2081 
2082   return ok_p;
2083 
2084 }
2085 
2086 
2087 /**
2088  * <JA>
2089  * �ڹ�¤������Υ������ʤɤξ����ɸ����Ϥ˽��Ϥ���.
2090  *
2091  * @param wchmm [in] �ڹ�¤������
2092  * </JA>
2093  * <EN>
2094  * Output some specifications of the tree lexicon (size etc.) to stdout.
2095  *
2096  * @param wchmm [in] tree lexicon already built
2097  * </EN>
2098  * @callgraph
2099  * @callergraph
2100  */
2101 void
print_wchmm_info(WCHMM_INFO * wchmm)2102 print_wchmm_info(WCHMM_INFO *wchmm)
2103 {
2104   int n,i, rootnum;
2105 
2106   if (wchmm->hmminfo->multipath) {
2107     rootnum = wchmm->startnum;
2108   } else {
2109     if (wchmm->lmtype == LM_PROB) {
2110       rootnum = wchmm->startnum + 1;	/* including winfo->head_silwid */
2111     } else if (wchmm->lmtype == LM_DFA) {
2112       rootnum = wchmm->startnum;
2113     }
2114   }
2115 
2116   jlog(" Lexicon tree:\n");
2117   jlog("\t total node num = %6d\n", wchmm->n);
2118   if (wchmm->lmtype == LM_PROB) {
2119     jlog("\t  root node num = %6d\n", rootnum);
2120 #ifdef NO_SEPARATE_SHORT_WORD
2121 #ifdef SEPARATE_BY_UNIGRAM
2122     jlog("\t(%d hi-freq. words are separated from tree lexicon)\n", wchmm->separated_word_count);
2123 #else
2124     jlog(" (no words are separated from tree)\n");
2125 #endif /* SEPARATE_BY_UNIGRAM */
2126 #else
2127     jlog(" (%d short words (<= %d phonemes) are separated from tree)\n", wchmm->separated_word_count, SHORT_WORD_LEN);
2128 #endif /* NO_SEPARATE_SHORT_WORD */
2129   }
2130   if (wchmm->lmtype == LM_DFA) {
2131     jlog("\t  root node num = %6d\n", rootnum);
2132   }
2133   for(n=0,i=0;i<wchmm->n;i++) {
2134     if (wchmm->stend[i] != WORD_INVALID) n++;
2135   }
2136   jlog("\t  leaf node num = %6d\n", n);
2137   if (!wchmm->category_tree) {
2138     jlog("\t fact. node num = %6d\n", wchmm->scnum - 1);
2139   }
2140 }
2141 
2142 /* end of file */
2143