RPA Toolkit
Work on documentation.
[rpatk.git] / tests / testrexcc / tokenjs.rexcc
1 /*
2  * To build:
3  * # rexcc tokenjs.rexcc -o tokenjs.c
4  * # gcc -I/usr/include/rpatk/rex -o tokenjs tokenjs.c
5  *
6  * To run:
7  * # echo "function add(a,b) { var c = a + b; return c; }" | ./tokenjs
8  */
9
10 #include <stdio.h>
11 #include <wchar.h>
12 #include <locale.h>
13
14 #define TOKEN_SELF 256
15 #define TOKEN_IDENTIFIER 257
16 #define TOKEN_SPACE 258
17 #define TOKEN_KEYWORD 259
18 #define TOKEN_OPERATOR 260
19 #define TOKEN_STRING 261
20 #define TOKEN_DECIMAL 262
21
22
23 %%
24 TOKEN_KEYWORD           instanceof | typeof | break | do | new | var | case | else | \
25                                         return | void | catch | finally | continue | for | \
26                                         switch | while | this | with |debugger | function | throw | default | \
27                                         if | try | delete | in | class | enum | extends | import | const | export | \
28                                         implements | let | private | public | static | interface | package | protected
29
30 TOKEN_IDENTIFIER        ([#0x0041-#0x005A] | [#0x00C0-#0x00DE] | [#0x0100-#0x0232] | [#0x0061-#0x007A] | \
31                                         [#0x00C0-#0x00DE] | $ | _ )([#0x0041-#0x005A] | [#0x00C0-#0x00DE] | \
32                                         [#0x0100-#0x0232] | [#0x0061-#0x007A] | [#0x00C0-#0x00DE] | $ | _ | [0-9] | [#0x0660-#0x0669])*
33
34 TOKEN_OPERATOR          === | !== | >= | <= | == | != | << | >>> | >> | & | ^= | ^ | ! | ~ | && | [|][|] | [?] | : | \
35                                         >>= | >>>= | &= | [|]= | = | [+]= | -= | [*]= | %= | <<= | [.] | ; | , | < | > | [|] | \
36                                         [+] | - | [*] | % | [+][+] | -- | / | /=
37
38 TOKEN_DECIMAL           [1-9][0-9]*
39
40 TOKEN_STRING            '[^']*'|"[^"]*"
41
42 TOKEN_SPACE                     [\t\r\n ]+
43
44 TOKEN_SELF                      [^\t\r\n+'" ]
45
46
47 %%
48
49
50 rexdfa_t *dfa = &ccdfa;
51
52
53 int get_token(wint_t *buffer, int size)
54 {
55         rexdfss_t *acc_ss = NULL;
56         rexuint_t nstate = REX_DFA_STARTSTATE;
57         int ret = -1, i = 0;
58         wint_t wc;
59         
60         while ((wc = fgetwc(stdin)) != WEOF) {
61                 if ((nstate = REX_DFA_NEXT(dfa, nstate, wc)) == REX_DFA_DEADSTATE) {
62                         ungetc(wc, stdin);
63                         break;
64                 }
65                 if (i + 1 < size) {
66                         buffer[i++] = wc;
67                 }
68                 if (REX_DFA_STATE(dfa, nstate)->type == REX_STATETYPE_ACCEPT) {
69                         /*
70                          * The DFA is in accepting state, lets find out what exactly is
71                          * being accepted.
72                          * The token ID is recorder in the substate's userdata
73                          *
74                          * Note: There are may be more than one accepting substate,
75                          * but we only check the first one (at offset 0). A real implementation
76                          * might need to check the rest of the accepting substates(and userdata)
77                          * to decide which one to use.
78                          *
79                          * Note: Some of the conflicts might be resolved simply be reordering
80                          * the regular expressions. For example TOKEN_KEYWORD such as 
81                          * 'while', 'if', etc. also match TOKEN_IDENTIFIER, but because
82                          * TOKEN_KEYWORD appears before TOKEN_IDENTIFIER it is placed first.
83                          *
84                          * Note: We will not break out of the loop here. We will keep going
85                          * in order to find the longest match.
86                          */
87                         acc_ss = REX_DFA_ACCSUBSTATE(dfa, nstate, 0);
88                         ret = (int) acc_ss->userdata;
89                         if (ret == TOKEN_SELF)
90                                 ret = wc;
91                 }
92         }
93         buffer[i++] = '\0';
94         return ret;
95 }
96
97 int main(int argc, char *argv[])
98 {
99         wint_t buffer[4000];
100         int token;
101         
102         if (!setlocale(LC_CTYPE, "")) {
103                 printf("Can not set the specified locale, please check LANG, LC_CTYPE, LC_ALL.\n");
104                 return 1;
105         }
106         while ((token = get_token(buffer, sizeof(buffer)/sizeof(buffer[0]))) > 0) {
107                 if (token != TOKEN_SPACE)
108                         fwprintf(stdout, L"token(%3d): %ls\n", token, buffer);
109         }
110         return 0;
111 }