1"""
2ldap.schema.tokenizer - Low-level parsing functions for schema element strings
3written by Michael Stroeder <michael@stroeder.com>
4
5See http://python-ldap.sourceforge.net for details.
6
7\$Id: tokenizer.py,v 1.8 2003/11/22 20:15:44 stroeder Exp $
8"""
9
10
11def split_tokens(s):
12  """
13  Returns list of syntax elements with quotes and spaces
14  stripped.
15  """
16  result = []
17  result_append = result.append
18  s_len = len(s)
19  i = 0
20  while i<s_len:
21    start = i
22    while i<s_len and s[i]!="'":
23      if s[i]=="(" or s[i]==")":
24        if i>start:
25          result_append(s[start:i])
26        result_append(s[i])
27        i +=1 # Consume parentheses
28        start = i
29      elif s[i]==" ":
30        if i>start:
31          result_append(s[start:i])
32        # Consume space chars
33        while i<s_len and s[i]==" ":
34          i +=1
35        start = i
36      else:
37        i +=1
38    if i>start:
39      result_append(s[start:i])
40    i +=1
41    if i>=s_len:
42      break
43    start = i
44    while i<s_len and s[i]!="'":
45      i +=1
46    if i>=start:
47      result_append(s[start:i])
48    i +=1
49  return result # split_tokens()
50
51
52def extract_tokens(l,known_tokens):
53  """
54  Returns dictionary of known tokens with all values
55  """
56  assert l[0].strip()=="(" and l[-1].strip()==")",ValueError(l)
57  result = {}
58  result_has_key = result.has_key
59  result.update(known_tokens)
60  i = 0
61  l_len = len(l)
62  while i<l_len:
63    if result_has_key(l[i]):
64      token = l[i]
65      i += 1 # Consume token
66      if i<l_len:
67        if result_has_key(l[i]):
68          # non-valued
69          result[token] = (())
70        elif l[i]=="(":
71          # multi-valued
72          i += 1 # Consume left parentheses
73          start = i
74          while i<l_len and l[i]!=")":
75            i += 1
76          result[token] = tuple(filter(lambda v:v!='$',l[start:i]))
77          i += 1 # Consume right parentheses
78        else:
79          # single-valued
80          result[token] = l[i],
81          i += 1 # Consume single value
82    else:
83      i += 1 # Consume unrecognized item
84  return result
85
86