1 files changed, 43 insertions, 13 deletions

diff --git a/tools/d2c/tokens.l b/tools/d2c/tokens.l
index 220e5d6..60dd257 100644
--- a/tools/d2c/tokens.l
+++ b/tools/d2c/tokens.l
@@ -14,7 +14,6 @@
 
 
 %option noyywrap
-%option nounput
 %option yylineno
 %option stack
 %option noyy_top_state
@@ -23,11 +22,12 @@
 %x comments
 
 %x ins_name try_details ins_details
-%x encoding encoding_type encoding_content
+%x encoding encoding_type encoding_content syntax_content
 
 %x encoding_bits encoding_bits_size
 
-%x raw_line
+%x top_brace
+%x raw_line raw_block
 
 
 %%
@@ -52,45 +52,75 @@
 <ins_details>[^\n]*             { yylvalp->cstring = yytext; return INS_DETAILS; }
 <ins_details>[\n]               { BEGIN(INITIAL); }
 
+"@id"                           { yy_push_state(raw_line); return ID; }
 
-"@desc"                         { yy_push_state(raw_line); return DESC; }
+"@desc"                         { yy_push_state(raw_block); return DESC; }
 
 
-"@encoding"                     { BEGIN(encoding); return ENCODING; }
+"@encoding"                     { yy_push_state(encoding); return ENCODING; }
 
 <encoding>[ ]                   { }
-<encoding>"("                   { BEGIN(encoding_type); }
+<encoding>"("                   { yy_push_state(encoding_type); }
 
 <encoding_type>[A-Za-z]         { yylvalp->string = strdup(yytext); return TYPE; }
 <encoding_type>[0-9]+           { yylvalp->integer = atoi(yytext); return NUMBER; }
-<encoding_type>")"              { BEGIN(encoding); }
+<encoding_type>")"              { yy_pop_state(); }
+
+<encoding>"{"                   { yy_push_state(encoding_content); }
 
-<encoding>"{"                   { BEGIN(encoding_content); }
 <encoding_content>[ \t\n]+      { }
-<encoding_content>"}"           { BEGIN(INITIAL); }
+<encoding_content>"}"           { yy_pop_state(); yy_pop_state(); }
 
 
 <encoding_content>"@format"     { yy_push_state(raw_line); return FORMAT; }
+<encoding_content>"@unused"     { yy_push_state(raw_line); return UNUSED; }
 
 <encoding_content>"@half"       { yy_push_state(raw_line); return HALF; }
 <encoding_content>"@word"       { yy_push_state(raw_line); return WORD; }
 
-<encoding_content>"@syntax"     { yy_push_state(raw_line); return SYNTAX; }
+<encoding_content>"@syntax"     { yy_push_state(syntax_content); yy_push_state(top_brace); return SYNTAX; }
+
+<encoding_content>"@hooks"      { yy_push_state(raw_block); return HOOKS; }
+
+
+<syntax_content>[ \t\n]+        { }
+
+<syntax_content>"@assert"       { yy_push_state(raw_block); return ASSERT; }
+
+<syntax_content>"@conv"         { yy_push_state(raw_block); return CONV; }
+
+<syntax_content>"@asm"          { yy_push_state(raw_line); return ASM; }
 
-<encoding_content>"@conv"       { return CONV; }
+<syntax_content>"@rules"        { yy_push_state(raw_block); return RULES; }
 
-<encoding_content>"@hooks"      { return HOOKS; }
+<syntax_content>"}"             { yy_pop_state(); }
 
-<encoding_content>"@rules"      { return RULES; }
 
+<top_brace>[ \t\n]+             { }
+<top_brace>"{"                  { yy_pop_state(); }
 
 <raw_line>[^\n]+                { yylvalp->cstring = yytext; return RAW_LINE; }
 <raw_line>"\n"                  { yy_pop_state(); }
 
+<raw_block>[ \t\n]+             { }
+<raw_block>"{"                  {
+                                    read_block(temp);
+                                    yylvalp->cstring = temp; return RAW_BLOCK;
+                                }
+<raw_block>"}"                  { yy_pop_state(); }
+
+
 <encoding_content>"{"           {
                                     read_block(temp);
                                     yylvalp->cstring = temp; return RAW_BLOCK;
                                 }
 
+<*>.                            {
+                                    char *msg;
+                                    asprintf(&msg, "Unhandled token in d2c definition: '%s'", yytext);
+                                    YY_FATAL_ERROR(msg);
+                                    free(msg);
+                                }
+
 
 %%