RPA Toolkit
Fixed REX_DFA_NEXT macro.
authorMartin Stoilov <martin@rpasearch.com>
Wed, 22 Feb 2012 07:21:01 +0000 (23:21 -0800)
committerMartin Stoilov <martin@rpasearch.com>
Wed, 22 Feb 2012 07:21:01 +0000 (23:21 -0800)
rex/doc/example/tokenjs.rexcc
rex/rexdfa.c
rex/rexdfa.h
rexcc/rexcc.c
rexgrep/rexgrep.c
tests/testrexcc/tokenjs.rexcc [deleted file]

index 1c2d7c8..66a8a74 100644 (file)
@@ -58,7 +58,8 @@ int get_token(wint_t *buffer, int size)
        wint_t wc;
        
        while ((wc = fgetwc(stdin)) != WEOF) {
-               if ((nstate = REX_DFA_NEXT(dfa, nstate, wc)) == REX_DFA_DEADSTATE) {
+               REX_DFA_NEXT(dfa, nstate, wc, &nstate);
+               if (nstate == REX_DFA_DEADSTATE) {
                        ungetc(wc, stdin);
                        break;
                }
index 6de0644..c419667 100644 (file)
@@ -106,19 +106,9 @@ rexdfss_t *rex_dfa_accsubstate(rexdfa_t *dfa, rexuint_t nstate, rexuint_t naccsu
 
 rexuint_t rex_dfa_next(rexdfa_t *dfa, rexuint_t nstate, rexchar_t input)
 {
-       rexdft_t *t;
-       long mid, min = 0, max = min + REX_DFA_STATE(dfa, nstate)->ntrans;
-       while (max > min) {
-               mid = (min + max)/2;
-               t = REX_DFA_TRANSITION(dfa, nstate, mid);
-               if (input >= t->lowin) {
-                       min = mid + 1;
-               } else {
-                       max = mid;
-               }
-       }
-       t = REX_DFA_TRANSITION(dfa, nstate, min-1);
-       return t->state;
+       rexuint_t newstate = 0;
+       REX_DFA_NEXT(dfa, nstate, input, &newstate);
+       return newstate;
 }
 
 
index 0a2830b..ccb79b8 100644 (file)
@@ -178,19 +178,21 @@ typedef enum {
 
 
 /**
- * @def REX_DFA_NEXT(__dfa__, __nstate__, __input__)
+ * @def REX_DFA_NEXT(__dfa__, __nstate__, __input__, __nextptr__)
  *
  * Get the next state ID in the DFA for the specified input. The macro will
  * search through the transitions of the current state to find the next
- * state of the DFA for the specified input.
+ * state of the DFA for the specified input. The next state will be assigned
+ * to *(__nextptr__).
  *
  * @param __dfa__ Pointer to @ref rexdfa_t object
- * @param __nstate__ Current state of the DFA
+ * @param __nstate__ Current state of the DFA.
  * @param __input__ Current input
- * @return The next state of the DFA for the specified input
+ * @param __nextptr__ Output the next state in here
+ * @return The next state of the DFA for the specified input is written in __nextptr__
  */
-#define REX_DFA_NEXT(__dfa__, __nstate__, __input__) \
-               ({ \
+#define REX_DFA_NEXT(__dfa__, __nstate__, __input__, __nextptr__) \
+               do { \
                        rexdft_t *t; \
                        rexuint_t mid, min = 0, max = REX_DFA_STATE(__dfa__, __nstate__)->ntrans; \
                        while (max > min) { \
@@ -203,8 +205,8 @@ typedef enum {
                                } \
                        } \
                        t = REX_DFA_TRANSITION(__dfa__, __nstate__, min-1); \
-                       (t->state); \
-               })
+                       *(__nextptr__) = t->state; \
+               } while (0)
 
 rexdfa_t *rex_dfa_create(rexuint_t nstates, rexuint_t ntrans, rexuint_t naccsubstates, rexuint_t nsubstates);
 void rex_dfa_destroy(rexdfa_t *dfa);
index 0915ae1..655e173 100644 (file)
@@ -377,7 +377,8 @@ int rex_cc_gettoken(rexcc_t *pCC)
                return -1;
        }
        while ((inc = r_utf8_mbtowc(&wc, (const unsigned char*)input, (const unsigned char*)pCC->end)) > 0) {
-               if ((nstate = REX_DFA_NEXT(dfa, nstate, wc)) <= 0)
+               REX_DFA_NEXT(dfa, nstate, wc, &nstate);
+               if (nstate == 0)
                        break;
                input += inc;
                s = REX_DFA_STATE(dfa, nstate);
index 184fe6f..12b6206 100644 (file)
@@ -100,7 +100,8 @@ int rex_grep_match(rexgrep_t *pGrep, const char* input, const char *end)
                rexdfs_t *s;
 
                while ((inc = r_utf8_mbtowc(&wc, (const unsigned char*)input, (const unsigned char*)end)) > 0) {
-                       if ((nstate = rex_dfa_next(dfa, nstate, wc)) == 0)
+                       REX_DFA_NEXT(dfa, nstate, wc, &nstate);
+                       if (nstate == 0)
                                break;
                        input += inc;
                        s = REX_DFA_STATE(dfa, nstate);
diff --git a/tests/testrexcc/tokenjs.rexcc b/tests/testrexcc/tokenjs.rexcc
deleted file mode 100644 (file)
index 1c2d7c8..0000000
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * To build:
- * # rexcc tokenjs.rexcc -o tokenjs.c
- * # gcc -I/usr/include/rpatk/rex -o tokenjs tokenjs.c
- *
- * To run:
- * # echo "function add(a,b) { var c = a + b; return c; }" | ./tokenjs
- */
-
-#include <stdio.h>
-#include <wchar.h>
-#include <locale.h>
-
-#define TOKEN_SELF 256
-#define TOKEN_IDENTIFIER 257
-#define TOKEN_SPACE 258
-#define TOKEN_KEYWORD 259
-#define TOKEN_OPERATOR 260
-#define TOKEN_STRING 261
-#define TOKEN_DECIMAL 262
-
-
-%%
-TOKEN_KEYWORD          instanceof | typeof | break | do | new | var | case | else | \
-                                       return | void | catch | finally | continue | for | \
-                                       switch | while | this | with |debugger | function | throw | default | \
-                                       if | try | delete | in | class | enum | extends | import | const | export | \
-                                       implements | let | private | public | static | interface | package | protected
-
-TOKEN_IDENTIFIER       ([#0x0041-#0x005A] | [#0x00C0-#0x00DE] | [#0x0100-#0x0232] | [#0x0061-#0x007A] | \
-                                       [#0x00C0-#0x00DE] | $ | _ )([#0x0041-#0x005A] | [#0x00C0-#0x00DE] | \
-                                       [#0x0100-#0x0232] | [#0x0061-#0x007A] | [#0x00C0-#0x00DE] | $ | _ | [0-9] | [#0x0660-#0x0669])*
-
-TOKEN_OPERATOR         === | !== | >= | <= | == | != | << | >>> | >> | & | ^= | ^ | ! | ~ | && | [|][|] | [?] | : | \
-                                       >>= | >>>= | &= | [|]= | = | [+]= | -= | [*]= | %= | <<= | [.] | ; | , | < | > | [|] | \
-                                       [+] | - | [*] | % | [+][+] | -- | / | /=
-
-TOKEN_DECIMAL          [1-9][0-9]*
-
-TOKEN_STRING           '[^']*'|"[^"]*"
-
-TOKEN_SPACE                    [\t\r\n ]+
-
-TOKEN_SELF                     [^\t\r\n+'" ]
-
-
-%%
-
-
-rexdfa_t *dfa = &ccdfa;
-
-
-int get_token(wint_t *buffer, int size)
-{
-       rexdfss_t *acc_ss = NULL;
-       rexuint_t nstate = REX_DFA_STARTSTATE;
-       int ret = -1, i = 0;
-       wint_t wc;
-       
-       while ((wc = fgetwc(stdin)) != WEOF) {
-               if ((nstate = REX_DFA_NEXT(dfa, nstate, wc)) == REX_DFA_DEADSTATE) {
-                       ungetc(wc, stdin);
-                       break;
-               }
-               if (i + 1 < size) {
-                       buffer[i++] = wc;
-               }
-               if (REX_DFA_STATE(dfa, nstate)->type == REX_STATETYPE_ACCEPT) {
-                       /*
-                        * The DFA is in accepting state, lets find out what exactly is
-                        * being accepted.
-                        * The token ID is recorder in the substate's userdata
-                        *
-                        * Note: There are may be more than one accepting substate,
-                        * but we only check the first one (at offset 0). A real implementation
-                        * might need to check the rest of the accepting substates(and userdata)
-                        * to decide which one to use.
-                        *
-                        * Note: Some of the conflicts might be resolved simply be reordering
-                        * the regular expressions. For example TOKEN_KEYWORD such as 
-                        * 'while', 'if', etc. also match TOKEN_IDENTIFIER, but because
-                        * TOKEN_KEYWORD appears before TOKEN_IDENTIFIER it is placed first.
-                        *
-                        * Note: We will not break out of the loop here. We will keep going
-                        * in order to find the longest match.
-                        */
-                       acc_ss = REX_DFA_ACCSUBSTATE(dfa, nstate, 0);
-                       ret = (int) acc_ss->userdata;
-                       if (ret == TOKEN_SELF)
-                               ret = wc;
-               }
-       }
-       buffer[i++] = '\0';
-       return ret;
-}
-
-int main(int argc, char *argv[])
-{
-       wint_t buffer[4000];
-       int token;
-       
-       if (!setlocale(LC_CTYPE, "")) {
-               printf("Can not set the specified locale, please check LANG, LC_CTYPE, LC_ALL.\n");
-               return 1;
-       }
-       while ((token = get_token(buffer, sizeof(buffer)/sizeof(buffer[0]))) > 0) {
-               if (token != TOKEN_SPACE)
-                       fwprintf(stdout, L"token(%3d): %ls\n", token, buffer);
-       }
-       return 0;
-}