#include "parse.h"
// Static table of recognized lexemes in the language
-struct Lexeme {
+static const struct Lexeme {
char* str;
int tok;
} LEXEMES[] = {
{"and", TOK_AND},
{"or", TOK_OR},
{"!", TOK_NOT},
+ {"&", TOK_BIT_AND},
+ {"|", TOK_BIT_OR},
+ {"^", TOK_BIT_XOR},
{"(", TOK_LPAR},
{")", TOK_RPAR},
{"[", TOK_LBRA},
{"*=", TOK_MULEQ},
{"/=", TOK_DIVEQ},
{"~=", TOK_CATEQ},
+ {"&=", TOK_BIT_ANDEQ},
+ {"|=", TOK_BIT_OREQ},
+ {"^=", TOK_BIT_XOREQ},
{"forindex", TOK_FORINDEX},
};
static void newToken(struct Parser* p, int pos, int type,
char* str, int slen, double num)
{
- struct Token* tok;
+ struct Token *tok, *last = p->tree.lastChild;
+
+ /* Adjacent string literals get concatenated */
+ if(type == TOK_LITERAL && str) {
+ if(last && last->type == TOK_LITERAL) {
+ int i, len1 = last->strlen;
+ char* str2 = naParseAlloc(p, len1 + slen);
+ for(i=0; i<len1; i++) str2[i] = last->str[i];
+ for(i=0; i<slen; i++) str2[i+len1] = str[i];
+ last->str = str2;
+ last->strlen += slen;
+ return;
+ }
+ }
tok = naParseAlloc(p, sizeof(struct Token));
tok->type = type;
tok->str = str;
tok->strlen = slen;
tok->num = num;
- tok->parent = &p->tree;
tok->next = 0;
- tok->prev = p->tree.lastChild;
+ tok->prev = last;
tok->children = 0;
tok->lastChild = 0;
-
- // Context sensitivity hack: a "-" following a binary operator of
- // higher precedence (MUL and DIV, basically) must be a unary
- // negation. Needed to get precedence right in the parser for
- // expressiong like "a * -2"
- if(type == TOK_MINUS && tok->prev)
- if(tok->prev->type == TOK_MUL || tok->prev->type == TOK_DIV)
- tok->type = type = TOK_NEG;
+ tok->rule = 0;
+
+ // Context sensitivity hack: a "-" or "~" following a binary operator of
+ // equal or higher precedence must be a unary negation. Needed to
+ // get precedence right in the parser for expressiong like "a * -2"
+ if((type == TOK_MINUS || type == TOK_CAT) && tok->prev) {
+ int pt = tok->prev->type;
+ if( pt==TOK_PLUS||pt==TOK_MINUS||pt==TOK_CAT||pt==TOK_MUL||pt==TOK_DIV
+ || pt==TOK_BIT_AND||pt==TOK_BIT_OR||pt==TOK_BIT_XOR )
+ tok->type = type = (type == TOK_MINUS ? TOK_NEG : TOK_BIT_NEG);
+ }
if(!p->tree.children) p->tree.children = tok;
if(p->tree.lastChild) p->tree.lastChild->next = tok;
}
// Ditto, but more complicated for double quotes.
+/* FIXME: need to handle \b (8), \f (12), and \uXXXX for JSON compliance */
static void dqEscape(char* buf, int len, int index, struct Parser* p,
char* cOut, int* eatenOut)
{
case 'n': *cOut = '\n'; break;
case 't': *cOut = '\t'; break;
case '\\': *cOut = '\\'; break;
+ case '`': *cOut = '`'; break;
case 'x':
if(len < 4) error(p, "unterminated string", index);
*cOut = (char)((hexc(buf[2], p, index)<<4) | hexc(buf[3], p, index));
}
}
-// FIXME: should handle UTF8 too
static void charLiteral(struct Parser* p, int index, char* s, int len)
{
- if(len != 1) error(p, "character constant not single character", index);
- newToken(p, index, TOK_LITERAL, 0, 0, *s);
+ int n, c;
+ c = naLexUtf8C(s, len, &n);
+ if(c < 0 || n != len) error(p, "invalid utf8 character constant", index);
+ newToken(p, index, TOK_LITERAL, 0, 0, c);
}
// Read in a string literal
return i+1;
}
-static int lexHexLiteral(struct Parser* p, int index)
+static int lexIntLiteral(struct Parser* p, int index, int base)
{
int nib, i = index;
double d = 0;
- while(i < p->len && (nib = hex(p->buf[i])) >= 0) {
- d = d*16 + nib;
+ while(i < p->len && (nib = hex(p->buf[i])) >= 0 && nib < base) {
+ d = d * base + nib;
i++;
}
newToken(p, index, TOK_LITERAL, 0, 0, d);
return i;
}
+#define ISNUM(c) ((c) >= '0' && (c) <= '9')
+#define ISHEX(c) (ISNUM(c) || ((c)>='a' && (c)<='f') || ((c)>='A' && (c)<='F'))
+#define NUMSTART(c) (ISNUM(c) || (c) == '+' || (c) == '-')
static int lexNumLiteral(struct Parser* p, int index)
{
int len = p->len, i = index;
unsigned char* buf = (unsigned char*)p->buf;
double d;
- if(i+1<len && buf[i+1] == 'x') return lexHexLiteral(p, index+2);
+ if( buf[i] == '0' && i + 2 < len ) {
+ if( buf[i+1] == 'x' && ISHEX(buf[i+2]) )
+ return lexIntLiteral(p, index+2, 16);
+ if( buf[i+1] == 'o' && ISNUM(buf[i+2]) )
+ return lexIntLiteral(p, index+2, 8);
+ }
- while(i<len && buf[i] >= '0' && buf[i] <= '9') i++;
+ while(i<len && ISNUM(buf[i])) i++;
if(i<len && buf[i] == '.') {
i++;
- while(i<len && buf[i] >= '0' && buf[i] <= '9') i++;
+ while(i<len && ISNUM(buf[i])) i++;
}
- if(i<len && (buf[i] == 'e' || buf[i] == 'E')) {
+ if(i+1<len && (buf[i] == 'e' || buf[i] == 'E') && NUMSTART(buf[i+1])) {
i++;
- if(i<len
- && (buf[i] == '-' || buf[i] == '+')
- && (i+1<len && buf[i+1] >= '0' && buf[i+1] <= '9')) i++;
- while(i<len && buf[i] >= '0' && buf[i] <= '9') i++;
+ if(buf[i] == '-' || buf[i] == '+') i++;
+ while(i<len && ISNUM(buf[i])) i++;
}
naStr_parsenum(p->buf + index, i - index, &d);
newToken(p, index, TOK_LITERAL, 0, 0, d);
i = lexStringLiteral(p, i, c);
break;
default:
- if(c >= '0' && c <= '9') i = lexNumLiteral(p, i);
+ if(ISNUM(c) || (c == '.' && (i+1)<p->len && ISNUM(p->buf[i+1])))
+ i = lexNumLiteral(p, i);
else handled = 0;
}