Merge pull request #121 from MoeOrganization/prakashk/quotesque

regexes with brackets as delimiters and double-quoted string interpolation
MoeOrganization · Jun 8, 2013 · 03a4f4d · 03a4f4d
2 parents e14ce31 + e924567
commit 03a4f4d
Show file tree

Hide file tree

Showing 18 changed files with 374 additions and 28 deletions.
diff --git a/spec/syntax-examples/001-literals/004-string-literals/001.ast b/spec/syntax-examples/001-literals/004-string-literals/001.ast
@@ -1 +1 @@
-{"StatementsNode" : [{"StringLiteralNode" : "hello world"}]}
+{"StatementsNode" : [{"StringSequenceNode" : {"stringParts" : [{"StringLiteralNode" : "hello world"}]}}]}
diff --git a/spec/syntax-examples/001-literals/004-string-literals/003.ast b/spec/syntax-examples/001-literals/004-string-literals/003.ast
@@ -0,0 +1 @@
+ {"StatementsNode" : [{"StringSequenceNode" : {"stringParts" : [{"StringLiteralNode" : "foo\\tbar\\n"}]}}]}
diff --git a/spec/syntax-examples/001-literals/004-string-literals/004.ast b/spec/syntax-examples/001-literals/004-string-literals/004.ast
@@ -1 +1 @@
-{"StatementsNode" : [{"StringLiteralNode" : "foo\"bar\""}]}
+{"StatementsNode" : [{"StringSequenceNode" : {"stringParts" : [{"StringLiteralNode" : "foo\"bar\""}]}}]}
diff --git a/spec/syntax-examples/001-literals/004-string-literals/006.ast b/spec/syntax-examples/001-literals/004-string-literals/006.ast
@@ -1 +1 @@
-{"StatementsNode" : [{"StringLiteralNode" : "\\x{03a3}"}]}
+{"StatementsNode" : [{"StringSequenceNode" : {"stringParts" : [{"StringLiteralNode" : "\\x{03a3}"}]}}]}
diff --git a/spec/syntax-examples/001-literals/006-array-literal/003.ast b/spec/syntax-examples/001-literals/006-array-literal/003.ast
@@ -1 +1 @@
-{"StatementsNode" : [{"ArrayLiteralNode" : [{"IntLiteralNode" : 10}, {"StringLiteralNode" : "twenty"}, {"FloatLiteralNode" : 30.5}, {"BooleanLiteralNode" : "false"}]}]}
+{"StatementsNode" : [{"ArrayLiteralNode" : [{"IntLiteralNode" : 10}, {"StringSequenceNode" : {"stringParts" : [{"StringLiteralNode" : "twenty"}]}}, {"FloatLiteralNode" : 30.5}, {"BooleanLiteralNode" : "false"}]}]}
diff --git a/spec/syntax-examples/001-literals/006-array-literal/004.ast b/spec/syntax-examples/001-literals/006-array-literal/004.ast
@@ -1 +1 @@
-{"StatementsNode" : [{"ArrayLiteralNode" : [{"IntLiteralNode" : 10}, {"ArrayLiteralNode" : [{"IntLiteralNode" : 20}, {"StringLiteralNode" : "thirty"}]}, {"IntLiteralNode" : 40}, {"ArrayLiteralNode" : [{"BooleanLiteralNode" : "true"}]}]}]}
+{"StatementsNode" : [{"ArrayLiteralNode" : [{"IntLiteralNode" : 10}, {"ArrayLiteralNode" : [{"IntLiteralNode" : 20}, {"StringSequenceNode" : {"stringParts" : [{"StringLiteralNode" : "thirty"}]}}]}, {"IntLiteralNode" : 40}, {"ArrayLiteralNode" : [{"BooleanLiteralNode" : "true"}]}]}]}
diff --git a/spec/syntax-examples/001-literals/007-hash-literal/003.ast b/spec/syntax-examples/001-literals/007-hash-literal/003.ast
@@ -1 +1 @@
- {"StatementsNode" : [{"HashLiteralNode" : [{"PairLiteralNode" : {"key" : {"StringLiteralNode" : "one"}, "value" : {"IntLiteralNode" : 10}}}, {"PairLiteralNode" : {"key" : {"StringLiteralNode" : "two"}, "value" : {"IntLiteralNode" : 20}}}, {"PairLiteralNode" : {"key" : {"StringLiteralNode" : "three"}, "value" : {"IntLiteralNode" : 30}}}]}]}
+ {"StatementsNode" : [{"HashLiteralNode" : [{"PairLiteralNode" : {"key" : {"StringSequenceNode" : {"stringParts" : [{"StringLiteralNode" : "one"}]}}, "value" : {"IntLiteralNode" : 10}}}, {"PairLiteralNode" : {"key" : {"StringSequenceNode" : {"stringParts" : [{"StringLiteralNode" : "two"}]}}, "value" : {"IntLiteralNode" : 20}}}, {"PairLiteralNode" : {"key" : {"StringSequenceNode" : {"stringParts" : [{"StringLiteralNode" : "three"}]}}, "value" : {"IntLiteralNode" : 30}}}]}]}
diff --git a/src/main/scala/org/moe/ast/AST.scala b/src/main/scala/org/moe/ast/AST.scala
@@ -148,3 +148,6 @@ case class MatchExpressionNode(pattern: AST, flags: AST) extends AST
 case class SubstExpressionNode(pattern: AST, replacement: AST, flags: AST) extends AST
 case class RegexMatchNode(target: AST, pattern: AST, flags: AST) extends AST
 case class RegexSubstNode(target: AST, pattern: AST, replacement: AST, flags: AST) extends AST
+
+case class StringSequenceNode(exprs: List[AST]) extends AST
+case class EvalExpressionNode(expr: String) extends AST
diff --git a/src/main/scala/org/moe/ast/Serializer.scala b/src/main/scala/org/moe/ast/Serializer.scala
@@ -531,6 +531,18 @@ object Serializer {
       )
     )
 
+    case StringSequenceNode(strParts) => JSONObject(
+      Map(
+        "StringSequenceNode" -> JSONObject(
+          Map(
+            "stringParts" -> JSONArray(strParts.map(toJSON(_)))
+          )
+        )
+      )
+    )
+
+    case EvalExpressionNode(value)  => JSONObject(Map("EvalExpressionNode" -> value))
+
     case x => "stub: " + x
   }
 

diff --git a/src/main/scala/org/moe/interpreter/guts/Literals.scala b/src/main/scala/org/moe/interpreter/guts/Literals.scala
@@ -4,6 +4,7 @@ import org.moe.interpreter._
 import org.moe.runtime._
 import org.moe.runtime.nativeobjects._
 import org.moe.ast._
+import org.moe.parser._
 
 object Literals extends Utils {
 
@@ -93,6 +94,32 @@ object Literals extends Utils {
         case (RegexLiteralNode(p), StringLiteralNode(r_), StringLiteralNode(f)) =>
          r.NativeObjects.getArray(List(r.NativeObjects.getRegex(p, f), r.NativeObjects.getStr(r_)) : _*)
       }
+
+    case (env, StringSequenceNode(parts: List[AST])) =>
+      r.NativeObjects.getStr(
+        ParserUtils.formatStr(
+          parts.map(
+            {
+              case StringLiteralNode(s) => s
+              case expr                 => {
+                val v = i.evaluate(env, expr)
+                // MoeStrObject.toString returns a double-quoted
+                // string literal, so call toString on all but
+                // MoeStrObjects
+                v match {
+                  case s: MoeStrObject => s.getNativeValue
+                  case x               => x.toString
+                }
+              }
+            }
+          ).mkString
+        )
+      )
+
+    case (env, EvalExpressionNode(expr)) => {
+      val body = MoeParser.parseStuff(expr)
+      i.eval(r, env, body)
+    }
   }
 
 }
diff --git a/src/main/scala/org/moe/parser/MoeLiterals.scala b/src/main/scala/org/moe/parser/MoeLiterals.scala
@@ -1,11 +1,12 @@
 package org.moe.parser
 
 import ParserUtils._
+// import MoeQuoteParser._
 
 import scala.util.parsing.combinator._
 import org.moe.ast._
 
-trait MoeLiterals extends JavaTokenParsers {
+trait MoeLiterals extends JavaTokenParsers with MoeQuoteParser {
 
   // treat comments as whitespace
   override val whiteSpace = """(#[^\n\r]*[\n\r]|\s)+""".r
@@ -49,17 +50,26 @@ trait MoeLiterals extends JavaTokenParsers {
   // things a little)
   // - SL
 
-  val doubleQuoteStringPattern = """"((?:[^"\p{Cntrl}\\]|\\[\\'"bfnrt]|\\x\{[a-fA-F0-9]{4}\})*)"""".r
-  def doubleQuoteString: Parser[StringLiteralNode] = doubleQuoteStringPattern ^^ {
-    case doubleQuoteStringPattern(s) => StringLiteralNode(formatStr(s))
- }
+  // val doubleQuoteStringPattern = """"((?:[^"\p{Cntrl}\\]|\\[\\'"bfnrt]|\\x\{[a-fA-F0-9]{4}\})*)"""".r
+  // def doubleQuoteString: Parser[StringLiteralNode] = doubleQuoteStringPattern ^^ {
+  //   case doubleQuoteStringPattern(s) => StringLiteralNode(formatStr(s))
+  // }
 
-  val singleQuoteStringPattern = """'((?:[^'\p{Cntrl}\\]|\\[\\'"bfnrt]|\\x\{[a-fA-F0-9]{4}\})*)'""".r
-  def singleQuoteString: Parser[StringLiteralNode] = singleQuoteStringPattern ^^ {
-    case singleQuoteStringPattern(s) => StringLiteralNode(s)
+  // val singleQuoteStringPattern = """'((?:[^'\p{Cntrl}\\]|\\[\\'"bfnrt]|\\x\{[a-fA-F0-9]{4}\})*)'""".r
+  // def singleQuoteString: Parser[StringLiteralNode] = singleQuoteStringPattern ^^ {
+  //   case singleQuoteStringPattern(s) => StringLiteralNode(s)
+  // }
+
+  def quotedString = quoted("""[/{\[(<]""".r)
+  def bracketedString = quoted("""[{\[(<]""".r) // only matching brackets as delimiters
+
+  def doubleQuoteString: Parser[StringSequenceNode] = (quoted('"') | ("qq" ~> quotedString)) ^^ {
+    s => MoeStringParser.interpolateStr(s)
   }
 
-  def string: Parser[StringLiteralNode] = doubleQuoteString | singleQuoteString
+  def singleQuoteString: Parser[StringLiteralNode] = quoted('\'') ^^ {
+    s => StringLiteralNode(s)
+  }
 
   // Self Literal
 
@@ -68,10 +78,10 @@ trait MoeLiterals extends JavaTokenParsers {
 
   // Regex Literal
 
-  def regexString = """(\\.|[^/])*""".r
+  // def regexString = """(\\.|[^/])*""".r
 
   // TODO: support for other delimiters
-  def regexLiteral: Parser[RegexLiteralNode] = "/" ~> regexString <~ "/" ^^ { rx => RegexLiteralNode(rx) }
+  def regexLiteral: Parser[RegexLiteralNode] = quoted('/') ^^ { rx => RegexLiteralNode(rx) }
 
   def literalValue: Parser[AST] = (
       floatNumber
@@ -83,7 +93,8 @@ trait MoeLiterals extends JavaTokenParsers {
     | constTrue
     | constFalse
     | constUndef
-    | string
+    | doubleQuoteString
+    | singleQuoteString
     | selfLiteral
     | superLiteral
     | regexLiteral

diff --git a/src/main/scala/org/moe/parser/MoeProductions.scala b/src/main/scala/org/moe/parser/MoeProductions.scala
@@ -303,20 +303,24 @@ trait MoeProductions extends MoeLiterals with JavaTokenParsers with PackratParse
 
   lazy val regexModifiers: Parser[AST] = """[igsmx]*""".r ^^ { flags => StringLiteralNode(flags) }
 
-  def matchExpression: Parser[AST] =
-    ("m".? ~> regexLiteral) ~ opt(regexModifiers) ^^ {
-      case pattern ~ None        => MatchExpressionNode(pattern, StringLiteralNode(""))
-      case pattern ~ Some(flags) => MatchExpressionNode(pattern, flags)
+  def matchExpression: Parser[AST] = (("m" ~> quotedString) | quoted('/')) ~ opt(regexModifiers) ^^ {
+      case pattern ~ None        => MatchExpressionNode(RegexLiteralNode(pattern), StringLiteralNode(""))
+      case pattern ~ Some(flags) => MatchExpressionNode(RegexLiteralNode(pattern), flags)
     }
 
-  def substExpression: Parser[AST] = ("s" ~> regexLiteral) ~ ("""(\\.|[^/])*""".r <~ "/") ~ opt(regexModifiers) ^^ {
-    case pattern ~ replacement ~ None        => SubstExpressionNode(pattern, StringLiteralNode(replacement), StringLiteralNode(""))
-    case pattern ~ replacement ~ Some(flags) => SubstExpressionNode(pattern, StringLiteralNode(replacement), flags)
+  def substExpression_1: Parser[AST] = ("s" ~> quotedPair('/')) ~ opt(regexModifiers) ^^ {
+    case (pattern, replacement) ~ None        => SubstExpressionNode(RegexLiteralNode(pattern), StringLiteralNode(replacement), StringLiteralNode(""))
+    case (pattern, replacement) ~ Some(flags) => SubstExpressionNode(RegexLiteralNode(pattern), StringLiteralNode(replacement), flags)
+  }
+
+  def substExpression_2: Parser[AST] = ("s" ~> bracketedString) ~ bracketedString ~ opt(regexModifiers) ^^ {
+    case pattern ~ replacement ~ None        => SubstExpressionNode(RegexLiteralNode(pattern), StringLiteralNode(replacement), StringLiteralNode(""))
+    case pattern ~ replacement ~ Some(flags) => SubstExpressionNode(RegexLiteralNode(pattern), StringLiteralNode(replacement), flags)
   }
 
   // TODO: tr (transliteration) operator
 
-  def regexExpression = (substExpression | matchExpression)
+  def regexExpression = (substExpression_2 | substExpression_1 | matchExpression)
 
   def matchOp = simpleExpression ~ "=~" ~ expression ^^ {
     case left ~ op ~ right => BinaryOpNode(left, op, right)

diff --git a/src/main/scala/org/moe/parser/MoeQuoteParser.scala b/src/main/scala/org/moe/parser/MoeQuoteParser.scala
@@ -0,0 +1,134 @@
+package org.moe.parser
+
+import scala.util.parsing.combinator._
+import scala.util.matching.Regex
+
+trait MoeQuoteParser extends JavaTokenParsers {
+  val quotePairMap = Map(
+    '(' -> ')',
+    '[' -> ']',
+    '{' -> '}',
+    '<' -> '>'
+  )
+
+  private def uptoEndDelim(in: Input, beginDelim: Char, endDelim: Char) = {
+    val source = in.source
+    val offset = in.offset
+    var j = offset + 1
+    // beginDelim found; copy input upto endDelim
+    val buf = new StringBuilder
+    val pairedDelim = beginDelim != endDelim
+
+    var nestingLevel = 0;
+    var done = false;
+    while (j < source.length && !done) {
+      source.charAt(j) match {
+        case '\\' if source.charAt(j+1) == beginDelim || source.charAt(j+1) == endDelim=> {
+          j += 1
+          buf += source.charAt(j)
+        }
+        case `beginDelim` => {
+          if (pairedDelim) {
+            nestingLevel += 1
+            buf += source.charAt(j)
+          }
+          else {
+            done = true
+          }
+        }
+        case `endDelim` => {
+          if (nestingLevel == 0) {
+            done = true
+          }
+          else {
+            nestingLevel -= 1
+            buf += source.charAt(j)
+          }
+        }
+        case _ => {
+          buf += source.charAt(j)
+        }
+      }
+      j += 1
+    }
+
+    if (done) {
+      (Some(buf.toString), in.drop(j - offset - 1))
+    }
+    else {
+      (None, in.drop(j - offset - 1))
+    }
+  }
+
+  def quoted(q: Char): Parser[String] = new Parser[String] {
+    def apply(in: Input) = {
+      val source = in.source
+      val offset = in.offset
+      val start = handleWhiteSpace(source, offset)
+      // println(s"source = |$source|, offset = |$offset|, start = |$start|")
+
+      var j = start
+      val beginDelim = q
+      val endDelim = quotePairMap.getOrElse(beginDelim, beginDelim)
+
+      if (j < source.length && beginDelim != source.charAt(j)) {
+        Failure("`" + q + "' expected but not found", in.drop(start - offset))
+      }
+      else {
+        uptoEndDelim(in.drop(start - offset), beginDelim, endDelim) match {
+          case (Some(matched), rest_in) => Success(matched, rest_in.drop(1))    // drop endDelim from input
+          case (None,          rest_in) => Failure("`" + endDelim + "' expected but not found.", rest_in)
+        }
+      }
+    }
+  }
+
+  def quoted(q: Regex): Parser[String] = new Parser[String] {
+    def apply(in: Input) = {
+      val source = in.source
+      val offset = in.offset
+      val start = handleWhiteSpace(source, offset)
+
+      (q findPrefixMatchOf(source.subSequence(start, source.length))) match {
+        case Some(matched) => {
+          val beginDelim = source.subSequence(start, start + matched.end).charAt(0)
+          val endDelim = quotePairMap.getOrElse(beginDelim, beginDelim)
+          uptoEndDelim(in.drop(start - offset), beginDelim, endDelim) match {
+            case (Some(matched), rest_in) => Success(matched, rest_in.drop(1))    // drop endDelim from input
+            case (None,          rest_in) => Failure("`" + endDelim + "' expected but not found.", rest_in)
+          }
+        }
+        case None =>
+          Failure("`string matching " + q + "' expected but not found", in.drop(start - offset))
+      }
+    }
+  }
+
+  def quotedPair(q: Char): Parser[(String, String)] = new Parser[(String, String)] {
+    def apply(in: Input) = {
+      val source = in.source
+      val offset = in.offset
+      val start = handleWhiteSpace(source, offset)
+      // println(s"source = |$source|, offset = |$offset|, start = |$start|")
+
+      var j = start
+      val beginDelim = q
+      val endDelim = quotePairMap.getOrElse(beginDelim, beginDelim)
+
+      if (j < source.length && beginDelim != source.charAt(j)) {
+        Failure("`" + q + "' expected but not found", in.drop(start - offset))
+      }
+      else {
+        uptoEndDelim(in.drop(start - offset), beginDelim, endDelim) match {
+          case (Some(matched_1), rest_in) => {
+            uptoEndDelim(rest_in, beginDelim, endDelim) match {
+              case (Some(matched_2), rest_in_2) => Success((matched_1, matched_2), rest_in_2.drop(1))    // drop endDelim from input
+              case (None,            rest_in_2) => Failure("`" + endDelim + "' expected but not found.", rest_in_2)
+            }
+          }
+          case (None,          rest_in) => Failure("`" + endDelim + "' expected but not found.", rest_in)
+        }
+      }
+    }
+  }
+}