NixOS · Apr 5, 2021 · Apr 5, 2021 · Apr 5, 2021
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,40 @@
 # Changelog
 
 ## [Unreleased]
+
+This release features a complete rewrite of the parser and lexer within
+the plugin.
+
+### Added
+
+- Support for the full syntax of Nix 2.3
+
+### Changed
+
+- Error detection and recovery has been overhauled.
+- The following words are no longer treated as keywords to make the
+  implementation consistent with Nix 2.3:
+
+  - `import`
+  - `imports`
+  - `require`
+  - `requires`
+  - `true`
+  - `false`
+
+  As a result, these words are no longer highlighted. We might bring
+  back the special highlighting in a future release by using a different
+  implementation for the highlighter.
+- Messages for syntax errors no longer contain the *“NixTokenType.”*
+  prefix for every expected token. This should make the messages much
+  easier to read.
+
+### Fixed
+
+- Various parsing errors (including but not limited to #8 and #13)
+- Incorrect reset of parser state when modifying a file
+
+## [0.3.0.5]
 ### Added
 - Support line comment and block comment IDEA actions
 ## [0.3.0.4]

diff --git a/gradle.properties b/gradle.properties
@@ -3,7 +3,7 @@
 
 pluginGroup = org.nixos.idea
 pluginName = NixIDEA
-pluginVersion = 0.3.0.5
+pluginVersion = 0.4.0.0
 pluginSinceBuild = 201
 pluginUntilBuild = 203.*
 # Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl

diff --git a/src/main/java/org/nixos/idea/lang/NixBraceMatcher.java b/src/main/java/org/nixos/idea/lang/NixBraceMatcher.java
@@ -2,35 +2,52 @@
 
 import com.intellij.lang.BracePair;
 import com.intellij.lang.PairedBraceMatcher;
+import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiFile;
 import com.intellij.psi.tree.IElementType;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.nixos.idea.psi.NixTypes;
 
 public class NixBraceMatcher implements PairedBraceMatcher {
+    // Grammar-Kit uses the first pair of this array to guide the error recovery
+    // (even when structural is set to false). Since the lexer tracks curly
+    // braces for its state transitions, the curly braces must be on top to keep
+    // the state of parser and lexer consistent. See
+    // https://intellij-support.jetbrains.com/hc/en-us/community/posts/360010379000
     public static final BracePair[] PAIRS = new BracePair[] {
+      new BracePair(NixTypes.LCURLY,NixTypes.RCURLY,true),
       new BracePair(NixTypes.LBRAC,NixTypes.RBRAC,false),
       new BracePair(NixTypes.LPAREN,NixTypes.RPAREN,false),
-      new BracePair(NixTypes.LCURLY,NixTypes.RCURLY,false),
-      new BracePair(NixTypes.DOLLAR_CURLY,NixTypes.RCURLY,true),
-      new BracePair(NixTypes.IND_STRING_OPEN,NixTypes.IND_STRING_CLOSE,true),
+      new BracePair(NixTypes.IND_STRING_OPEN,NixTypes.IND_STRING_CLOSE,false),
       new BracePair(NixTypes.STRING_OPEN,NixTypes.STRING_CLOSE,false)
     };
 
     @Override
-    public BracePair[] getPairs() {
+    public BracePair @NotNull [] getPairs() {
         return PAIRS;
     }
 
     @Override
-    public boolean isPairedBracesAllowedBeforeType(@NotNull IElementType iElementType, @Nullable IElementType iElementType1) {
+    public boolean isPairedBracesAllowedBeforeType(@NotNull IElementType lbraceType, @Nullable IElementType contextType) {
         return false;
     }
 
     @Override
-    public int getCodeConstructStart(PsiFile psiFile, int i) {
-        return 0;
+    public int getCodeConstructStart(PsiFile file, int openingBraceOffset) {
+        PsiElement openingBrace = file.findElementAt(openingBraceOffset);
+        if (openingBrace != null && openingBrace.getNode().getElementType() == NixTypes.LCURLY) {
+            PsiElement previousToken = openingBrace.getPrevSibling();
+            if (previousToken != null && previousToken.getNode().getElementType() == NixTypes.DOLLAR) {
+                return openingBraceOffset - 1;
+            }
+            else {
+                return openingBraceOffset;
+            }
+        }
+        else {
+            return openingBraceOffset;
+        }
     }
 }
 

diff --git a/src/main/java/org/nixos/idea/lang/NixParserDefinition.java b/src/main/java/org/nixos/idea/lang/NixParserDefinition.java
@@ -12,8 +12,11 @@
 import com.intellij.psi.TokenType;
 import com.intellij.psi.tree.IFileElementType;
 import com.intellij.psi.tree.TokenSet;
+import org.fest.util.Objects;
 import org.jetbrains.annotations.NotNull;
 import org.nixos.idea.file.NixFile;
+import org.nixos.idea.psi.NixTokenType;
+import org.nixos.idea.psi.NixTypeUtil;
 import org.nixos.idea.psi.NixTypes;
 
 public class NixParserDefinition implements ParserDefinition {
@@ -65,7 +68,21 @@ public PsiFile createFile(FileViewProvider viewProvider) {
 
     @Override
     public SpaceRequirements spaceExistenceTypeBetweenTokens(ASTNode left, ASTNode right) {
-        return SpaceRequirements.MAY;
+        NixTokenType leftType = Objects.castIfBelongsToType(left.getElementType(), NixTokenType.class);
+        NixTokenType rightType = Objects.castIfBelongsToType(right.getElementType(), NixTokenType.class);
+        if (leftType == NixTypes.SCOMMENT) {
+            return SpaceRequirements.MUST_LINE_BREAK;
+        }
+        if (leftType == NixTypes.DOLLAR && rightType == NixTypes.LCURLY) {
+            return SpaceRequirements.MUST_NOT;
+        }
+        else if (NixTypeUtil.MIGHT_COLLAPSE_WITH_ID.contains(leftType) &&
+                 NixTypeUtil.MIGHT_COLLAPSE_WITH_ID.contains(rightType)) {
+            return SpaceRequirements.MUST;
+        }
+        else {
+            return SpaceRequirements.MAY;
+        }
     }
 
     @NotNull

diff --git a/src/main/java/org/nixos/idea/lang/NixSyntaxHighlighter.java b/src/main/java/org/nixos/idea/lang/NixSyntaxHighlighter.java
@@ -148,7 +148,7 @@ public class NixSyntaxHighlighter extends SyntaxHighlighterBase {
                     tokenType == NixTypes.RPAREN ||
                     tokenType == NixTypes.LPAREN ||
                     tokenType == NixTypes.RCURLY ||
-                    tokenType == NixTypes.DOLLAR_CURLY ||
+                    tokenType == NixTypes.DOLLAR ||
                     tokenType == NixTypes.LCURLY) {
                 return PAREN_KEYS;
             }

diff --git a/src/main/java/org/nixos/idea/psi/NixTokenType.java b/src/main/java/org/nixos/idea/psi/NixTokenType.java
@@ -13,16 +13,14 @@ public NixTokenType(@NotNull @NonNls String debugName) {
 
     @Override
     public String toString() {
-        String debugName = super.toString();
-        int firstCodePoint = debugName.codePointAt(0);
-        if (Character.isUnicodeIdentifierStart(firstCodePoint) && Character.isLowerCase(firstCodePoint)) {
+        if (NixTypeUtil.KEYWORDS.contains(this)) {
             // The character U+2060 (Word Joiner) is used as a workaround to
             // make Grammar-Kit put quotation marks around keywords. See
             // https://github.com/JetBrains/Grammar-Kit/issues/262
-            return "\u2060" + debugName;
+            return "\u2060" + super.toString();
         }
         else {
-            return debugName;
+            return super.toString();
         }
     }
 }
diff --git a/src/main/java/org/nixos/idea/psi/NixTypeUtil.java b/src/main/java/org/nixos/idea/psi/NixTypeUtil.java
@@ -0,0 +1,33 @@
+package org.nixos.idea.psi;
+
+import com.intellij.psi.tree.TokenSet;
+
+public final class NixTypeUtil {
+
+    /** All token types which represent a keyword. */
+    public static final TokenSet KEYWORDS = TokenSet.create(
+            NixTypes.IF,
+            NixTypes.THEN,
+            NixTypes.ELSE,
+            NixTypes.ASSERT,
+            NixTypes.WITH,
+            NixTypes.LET,
+            NixTypes.IN,
+            NixTypes.REC,
+            NixTypes.INHERIT,
+            NixTypes.OR_KW);
+
+    /** Tokens would collapse if they were not separated by whitespace. */
+    public static final TokenSet MIGHT_COLLAPSE_WITH_ID = TokenSet.orSet(
+            KEYWORDS,
+            TokenSet.create(
+                    NixTypes.ID,
+                    NixTypes.INT,
+                    NixTypes.FLOAT,
+                    NixTypes.PATH,
+                    NixTypes.HPATH,
+                    NixTypes.SPATH,
+                    NixTypes.URI));
+
+    private NixTypeUtil() {} // Cannot be instantiated
+}
diff --git a/src/main/lang/Nix.bnf b/src/main/lang/Nix.bnf
@@ -45,7 +45,7 @@
     RCURLY = '}'
     LBRAC = '['
     RBRAC = ']'
-    DOLLAR_CURLY = '${'
+    DOLLAR = '$'
 
     DOT = '.'
     HAS = '?'
@@ -186,9 +186,9 @@ ind_string ::= IND_STRING_OPEN string_part* IND_STRING_CLOSE { pin=1 }
 ;{ extends("string_text|antiquotation")=string_part }
 string_part ::= string_text | antiquotation { recoverWhile=string_part_recover }
 string_text ::= STR | IND_STR
-antiquotation ::= DOLLAR_CURLY expr recover_antiquotation RCURLY { pin=1 }
+antiquotation ::= DOLLAR LCURLY expr recover_antiquotation RCURLY { pin=1 }
 private recover_antiquotation ::= { recoverWhile=curly_recover }
-private string_part_recover ::= !(STR | IND_STR | DOLLAR_CURLY | STRING_CLOSE | IND_STRING_CLOSE)
+private string_part_recover ::= !(STR | IND_STR | DOLLAR | STRING_CLOSE | IND_STRING_CLOSE)
 
 ;{ extends("bind_attr|bind_inherit")=bind }
 bind ::= bind_attr | bind_inherit
@@ -207,7 +207,11 @@ string_attr ::= std_string | antiquotation
 attr_path ::= attr ( DOT attr )*
 
 
+// The lexer uses curly braces to determine its state. To avoid inconsistencies
+// between the parser and lexer (i.e. the lexer sees a string where the parser
+// wants to read an expression), the error recovery for curly braces is very
+// strict while no other error recovery will consume curly braces.
 private braces_recover ::= !(RPAREN | RCURLY | RBRAC)
-private paren_recover ::= !RPAREN
+private paren_recover ::= !(RPAREN | RCURLY)
 private curly_recover ::= !RCURLY
-private brac_recover ::= !RBRAC
+private brac_recover ::= !(RBRAC | RCURLY)
diff --git a/src/main/lang/Nix.flex b/src/main/lang/Nix.flex
@@ -55,7 +55,7 @@ import static org.nixos.idea.psi.NixTypes.*;
 %function advance
 %type IElementType
 %unicode
-%xstate STRING IND_STRING
+%xstate STRING IND_STRING ANTIQUOTATION_START
 
 ANY=[^]
 ID=[a-zA-Z_][a-zA-Z0-9_'-]*
@@ -96,7 +96,9 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/
   "}"                   { popState(); return RCURLY; }
   "["                   { return LBRAC; }
   "]"                   { return RBRAC; }
-  "${"                  { pushState(YYINITIAL); return DOLLAR_CURLY; }
+  // '$' and '{' must be two separate tokens to make NixBraceMatcher work
+  // correctly with Grammar-Kit.
+  "$"/"{"               { return DOLLAR; }
 
   "."                   { return DOT; }
   "?"                   { return HAS; }
@@ -140,7 +142,7 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/
   ([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" { return STR; }
   ([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ |
   \$|\\|\$\\            { return STR; }
-  "${"                  { pushState(YYINITIAL); return DOLLAR_CURLY; }
+  "$"/"{"               { pushState(ANTIQUOTATION_START); return DOLLAR; }
   \"                    { popState(); return STRING_CLOSE; }
 }
 
@@ -150,7 +152,13 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/
   \$ |
   "'''" |
   "''"\\{ANY}           { return IND_STR; }
-  "${"                  { pushState(YYINITIAL); return DOLLAR_CURLY; }
+  "$"/"{"               { pushState(ANTIQUOTATION_START); return DOLLAR; }
   "''"                  { popState(); return IND_STRING_CLOSE; }
   "'"                   { return IND_STR; }
 }
+
+<ANTIQUOTATION_START> {
+  // '$' and '{' must be two separate tokens to make NixBraceMatcher work
+  // correctly with Grammar-Kit.
+  "{"                   { popState(); pushState(YYINITIAL); return LCURLY; }
+}
diff --git a/src/test/java/org/nixos/idea/lang/ParsingFailTest.java b/src/test/java/org/nixos/idea/lang/ParsingFailTest.java
@@ -1,8 +1,8 @@
 package org.nixos.idea.lang;
 
+import com.intellij.lang.LanguageBraceMatching;
 import com.intellij.testFramework.ParsingTestCase;
 import com.intellij.testFramework.TestDataPath;
-import org.nixos.idea.lang.NixParserDefinition;
 
 @TestDataPath("$PROJECT_ROOT/src/test/testData/ParsingFailTest")
 public final class ParsingFailTest extends ParsingTestCase {
@@ -15,6 +15,16 @@ public ParsingFailTest() {
   //  https://nixos.org/manual/nix/stable/#ch-expression-language
   //  https://github.com/NixOS/nix/blob/master/src/libexpr/parser.y
 
+  public void testAntiquotationStateSynchronization1() {
+    // See https://intellij-support.jetbrains.com/hc/en-us/community/posts/360010379000
+    doTest(true);
+  }
+
+  public void testAntiquotationStateSynchronization2() {
+    // See https://intellij-support.jetbrains.com/hc/en-us/community/posts/360010379000
+    doTest(true);
+  }
+
   public void testComment() {
     doTest(true);
   }
@@ -71,10 +81,6 @@ public void testRecoverFromSetBinding() {
     doTest(true);
   }
 
-  public void testRecoverFromString() {
-    doTest(true);
-  }
-
   public void testRecoverFromWith() {
     doTest(true);
   }
@@ -83,6 +89,14 @@ public void testRecWithoutSet() {
     doTest(true);
   }
 
+  @Override
+  protected void setUp() throws Exception {
+    super.setUp();
+    // Test environment of ParsingTestCase does not detect the brace matcher on its own. The brace matcher effects the
+    // error recovery of Grammar-Kit and must therefore be registered to get correct test results.
+    addExplicitExtension(LanguageBraceMatching.INSTANCE, NixLanguage.INSTANCE, new NixBraceMatcher());
+  }
+
   @Override
   protected String getTestDataPath() {
     return "src/test/testData";

diff --git a/src/test/testData/ParsingFailTest/AntiquotationStateSynchronization1.nix b/src/test/testData/ParsingFailTest/AntiquotationStateSynchronization1.nix
@@ -0,0 +1 @@
+"${ {{${{ }}}} "" }" end
diff --git a/src/test/testData/ParsingFailTest/AntiquotationStateSynchronization1.txt b/src/test/testData/ParsingFailTest/AntiquotationStateSynchronization1.txt
@@ -0,0 +1,35 @@
+Nix File(0,24)
+  NixExprAppImpl(EXPR_APP)(0,24)
+    NixStdStringImpl(STD_STRING)(0,20)
+      PsiElement(STRING_OPEN)('"')(0,1)
+      NixAntiquotationImpl(ANTIQUOTATION)(1,19)
+        PsiElement($)('$')(1,2)
+        PsiElement({)('{')(2,3)
+        PsiWhiteSpace(' ')(3,4)
+        NixExprAppImpl(EXPR_APP)(4,17)
+          NixSetImpl(SET)(4,14)
+            PsiElement({)('{')(4,5)
+            PsiElement(DUMMY_BLOCK)(5,13)
+              PsiErrorElement:'{' unexpected(5,6)
+                PsiElement({)('{')(5,6)
+              PsiElement(DUMMY_BLOCK)(6,7)
+                PsiElement($)('$')(6,7)
+              PsiElement(DUMMY_BLOCK)(7,12)
+                PsiElement({)('{')(7,8)
+                PsiElement(DUMMY_BLOCK)(8,11)
+                  PsiElement({)('{')(8,9)
+                  PsiWhiteSpace(' ')(9,10)
+                  PsiElement(})('}')(10,11)
+                PsiElement(})('}')(11,12)
+              PsiElement(})('}')(12,13)
+            PsiElement(})('}')(13,14)
+          PsiWhiteSpace(' ')(14,15)
+          NixStdStringImpl(STD_STRING)(15,17)
+            PsiElement(STRING_OPEN)('"')(15,16)
+            PsiElement(STRING_CLOSE)('"')(16,17)
+        PsiWhiteSpace(' ')(17,18)
+        PsiElement(})('}')(18,19)
+      PsiElement(STRING_CLOSE)('"')(19,20)
+    PsiWhiteSpace(' ')(20,21)
+    NixIdentifierImpl(IDENTIFIER)(21,24)
+      PsiElement(ID)('end')(21,24)
diff --git a/src/test/testData/ParsingFailTest/AntiquotationStateSynchronization2.nix b/src/test/testData/ParsingFailTest/AntiquotationStateSynchronization2.nix
@@ -0,0 +1 @@
+"${ ([ } ])" end