have literals

tests/test_parser.py

@@ -14,9 +14,22 @@ from ply.lex import LexToken
 
 from xonsh.parser import Parser
 
+PARSER = None
+DEBUG_LEVEL = 0
+
+def setup():
+    # only setup one parser
+    global PARSER
+    PARSER = Parser(lexer_optimize=False, yacc_optimize=False, yacc_debug=True)
+
 def nodes_equal(x, y):
     if type(x) != type(y):
         return False
+    if isinstance(x, ast.Expr):
+        if x.lineno != y.lineno:
+            return False
+        if x.col_offset != y.col_offset:
+            return False
     for (xname, xval), (yname, yval) in zip(ast.iter_fields(x), 
                                             ast.iter_fields(y)):
         if xname != yname:
@@ -35,15 +48,28 @@ def assert_nodes_equal(x, y):
     assert_equal(ast.dump(x), ast.dump(y))
 
 def check_ast(input):
+    # expect a Python AST
     exp = ast.parse(input)
-    p = Parser(lexer_optimize=False, yacc_optimize=False, yacc_debug=True)
-    obs = p.parse(input, debug_level=100)
+    # observe something from xonsh
+    obs = PARSER.parse(input, debug_level=DEBUG_LEVEL)
+    # Check that they are equal
     assert_nodes_equal(exp, obs)
+    # round trip by running xonsh AST via Python
+    exec(compile(obs, '<test>', 'exec'))
 
 
 def test_int_literal():
     yield check_ast, '42'
 
+def test_float_literal():
+    yield check_ast, '42.0'
+
+def test_str_literal():
+    yield check_ast, '"hello"'
+
+def test_bytes_literal():
+    yield check_ast, 'b"hello"'
+
 
 if __name__ == '__main__':
     nose.runmodule()

xonsh/ast.py

@@ -1,3 +1,3 @@
 """The xonsh abstract syntax tree node."""
 from __future__ import unicode_literals, print_function
-from ast import Module, Num, Expr
+from ast import Module, Num, Expr, Str, Bytes

xonsh/lexer.py

@@ -35,6 +35,7 @@ class Lexer(object):
     def build(self, **kwargs):
         """Part of the PLY lexer API."""
         self.lexer = lex.lex(object=self, **kwargs)
+        self.lexer.lineno = 1
 
     @property
     def lineno(self):

xonsh/parser.py

@@ -169,10 +169,9 @@ class Parser(object):
         """
         self.lexer.fname = filename
         self.lexer.lineno = 0
-        self._scope_stack = [dict()]
         self._last_yielded_token = None
         tree = self.parser.parse(input=s, lexer=self.lexer,
-                                  debug=debug_level)
+                                 debug=debug_level)
         return tree
 
     def _lexer_errfunc(self, msg, line, column):
@@ -209,6 +208,26 @@ class Parser(object):
         return Location(fname=self.lexer.fname, lineno=lineno,
                         column=column)
 
+    def expr(self, p):
+        """Creates an expression for a token."""
+        return ast.Expr(value=p, lineno=p.lineno, 
+                        col_offset=p.col_offset)
+
+    def token_col(self, t):
+        """Gets ths token column"""
+        return self.lexer.token_col(t)
+
+    @property
+    def lineno(self):
+        return self.lexer.lineno
+
+    @property
+    def col(self):
+        t = self._yacc_lookahead_token()
+        if t is not None:
+            return self.token_col(t)
+        return 1
+
     def _parse_error(self, msg, loc):
         raise SyntaxError('{0}: {1}'.format(loc, msg))
 
@@ -834,7 +853,9 @@ class Parser(object):
                           | UNICODE_LITERAL
                           | BYTES_LITERAL
         """
-        p[0] = p[1]
+        s = eval(p[1])
+        cls = ast.Bytes if p[1].startswith('b') else ast.Str
+        p[0] = cls(s=s, lineno=self.lineno, col_offset=self.col)
 
     def p_number(self, p):
         """number : INT_LITERAL
@@ -843,7 +864,7 @@ class Parser(object):
                   | BIN_LITERAL
                   | FLOAT_LITERAL
         """
-        p[0] = p[1]
+        p[0] = ast.Num(n=p[1], lineno=self.lineno, col_offset=self.col)
 
     def p_testlist_comp(self, p):
         """testlist_comp : test_or_star_expr comp_for 
@@ -892,7 +913,7 @@ class Parser(object):
 
     def p_testlist(self, p):
         """testlist : test comma_test_list_opt comma_opt"""
-        p[0] = p[1]
+        p[0] = [self.expr(p[1])]
         if p[2] is not None:
             p[0] += p[2]
         if p[3] is not None: