NixOS · Jan 25, 2021 · Feb 20, 2021 · Feb 20, 2021 · Apr 5, 2021 · Apr 5, 2021
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -34,6 +34,12 @@ repositories {
     mavenCentral()
 }
 
+dependencies {
+    testImplementation(platform("org.junit:junit-bom:5.7.0"))
+    testImplementation("org.junit.jupiter:junit-jupiter")
+    testRuntimeOnly("org.junit.vintage:junit-vintage-engine")
+}
+
 // Configure gradle-intellij-plugin plugin.
 // Read more: https://github.com/JetBrains/gradle-intellij-plugin
 intellij {
@@ -70,7 +76,7 @@ tasks {
     task<GenerateLexer>("generateNixLexer") {
         source = "src/main/lang/Nix.flex"
         targetDir = "src/gen/java/org/nixos/idea/lang"
-        targetClass = "NixLexer"
+        targetClass = "_NixLexer"
         purgeOldFiles = true
     }
 
@@ -101,7 +107,10 @@ tasks {
                 }
             }
         }
+    }
 
+    test {
+        useJUnitPlatform()
     }
 
     patchPluginXml {

diff --git a/src/main/java/org/nixos/idea/lang/NixBraceMatcher.java b/src/main/java/org/nixos/idea/lang/NixBraceMatcher.java
@@ -2,35 +2,52 @@
 
 import com.intellij.lang.BracePair;
 import com.intellij.lang.PairedBraceMatcher;
+import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiFile;
 import com.intellij.psi.tree.IElementType;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.nixos.idea.psi.NixTypes;
 
 public class NixBraceMatcher implements PairedBraceMatcher {
+    // Grammar-Kit uses the first pair of this array to guide the error recovery
+    // (even when structural is set to false). Since the lexer tracks curly
+    // braces for its state transitions, the curly braces must be on top to keep
+    // the state of parser and lexer consistent. See
+    // https://intellij-support.jetbrains.com/hc/en-us/community/posts/360010379000
     public static final BracePair[] PAIRS = new BracePair[] {
+      new BracePair(NixTypes.LCURLY,NixTypes.RCURLY,true),
       new BracePair(NixTypes.LBRAC,NixTypes.RBRAC,false),
       new BracePair(NixTypes.LPAREN,NixTypes.RPAREN,false),
-      new BracePair(NixTypes.LCURLY,NixTypes.RCURLY,false),
-      new BracePair(NixTypes.DOLLAR_CURLY,NixTypes.RCURLY,true),
-      new BracePair(NixTypes.IND_STRING_OPEN,NixTypes.IND_STRING_CLOSE,true),
-      new BracePair(NixTypes.FNUTT_OPEN,NixTypes.FNUTT_CLOSE,false)
+      new BracePair(NixTypes.IND_STRING_OPEN,NixTypes.IND_STRING_CLOSE,false),
+      new BracePair(NixTypes.STRING_OPEN,NixTypes.STRING_CLOSE,false)
     };
 
     @Override
-    public BracePair[] getPairs() {
+    public BracePair @NotNull [] getPairs() {
         return PAIRS;
     }
 
     @Override
-    public boolean isPairedBracesAllowedBeforeType(@NotNull IElementType iElementType, @Nullable IElementType iElementType1) {
+    public boolean isPairedBracesAllowedBeforeType(@NotNull IElementType lbraceType, @Nullable IElementType contextType) {
         return false;
     }
 
     @Override
-    public int getCodeConstructStart(PsiFile psiFile, int i) {
-        return 0;
+    public int getCodeConstructStart(PsiFile file, int openingBraceOffset) {
+        PsiElement openingBrace = file.findElementAt(openingBraceOffset);
+        if (openingBrace != null && openingBrace.getNode().getElementType() == NixTypes.LCURLY) {
+            PsiElement previousToken = openingBrace.getPrevSibling();
+            if (previousToken != null && previousToken.getNode().getElementType() == NixTypes.DOLLAR) {
+                return openingBraceOffset - 1;
+            }
+            else {
+                return openingBraceOffset;
+            }
+        }
+        else {
+            return openingBraceOffset;
+        }
     }
 }
 

diff --git a/src/main/java/org/nixos/idea/lang/NixLexer.java b/src/main/java/org/nixos/idea/lang/NixLexer.java
@@ -0,0 +1,52 @@
+package org.nixos.idea.lang;
+
+import com.intellij.lexer.FlexAdapter;
+import com.intellij.lexer.FlexLexer;
+import com.intellij.psi.tree.IElementType;
+
+import java.io.IOException;
+
+public class NixLexer extends FlexAdapter {
+    // todo: Implement RestartableLexer when it becomes non-experimental. See
+    //  https://intellij-support.jetbrains.com/hc/en-us/community/posts/360010305800/comments/360002861979
+
+    public NixLexer() {
+        // We need this anonymous wrapper class to map between internal states
+        // of Flex and external states for IntelliJ. See
+        // https://intellij-support.jetbrains.com/hc/en-us/community/posts/360010305800
+        super(new FlexLexer() {
+            private final _NixLexer lexer = new _NixLexer(null);
+
+            @Override
+            public void yybegin(int state) {
+                lexer.restoreState(state);
+            }
+
+            @Override
+            public int yystate() {
+                return lexer.getStateIndex();
+            }
+
+            @Override
+            public int getTokenStart() {
+                return lexer.getTokenStart();
+            }
+
+            @Override
+            public int getTokenEnd() {
+                return lexer.getTokenEnd();
+            }
+
+            @Override
+            public IElementType advance() throws IOException {
+                return lexer.advance();
+            }
+
+            @Override
+            public void reset(CharSequence buf, int start, int end, int initialState) {
+                lexer.reset(buf, start, end, _NixLexer.YYINITIAL);
+                lexer.restoreState(initialState);
+            }
+        });
+    }
+}
diff --git a/src/main/java/org/nixos/idea/lang/NixLexerAdapter.java b/src/main/java/org/nixos/idea/lang/NixLexerAdapter.java
diff --git a/src/main/java/org/nixos/idea/lang/NixParserDefinition.java b/src/main/java/org/nixos/idea/lang/NixParserDefinition.java
@@ -4,7 +4,6 @@
 import com.intellij.lang.Language;
 import com.intellij.lang.ParserDefinition;
 import com.intellij.lang.PsiParser;
-import com.intellij.lexer.FlexAdapter;
 import com.intellij.lexer.Lexer;
 import com.intellij.openapi.project.Project;
 import com.intellij.psi.FileViewProvider;
@@ -13,24 +12,23 @@
 import com.intellij.psi.TokenType;
 import com.intellij.psi.tree.IFileElementType;
 import com.intellij.psi.tree.TokenSet;
+import org.fest.util.Objects;
 import org.jetbrains.annotations.NotNull;
 import org.nixos.idea.file.NixFile;
+import org.nixos.idea.psi.NixTokenType;
 import org.nixos.idea.psi.NixTypes;
 
-import java.io.Reader;
-
 public class NixParserDefinition implements ParserDefinition {
     public static final TokenSet WHITE_SPACES = TokenSet.create(TokenType.WHITE_SPACE);
     public static final TokenSet COMMENTS = TokenSet.create(NixTypes.SCOMMENT,NixTypes.MCOMMENT);
-    public static final TokenSet STRING_LITERALS = TokenSet.create(NixTypes.LITERAL_SIMPLE_STRING, NixTypes.DOC_STRING);
+    public static final TokenSet STRING_LITERALS = TokenSet.create(NixTypes.STD_STRING, NixTypes.IND_STRING);
 
     public static final IFileElementType FILE = new IFileElementType(Language.<NixLanguage>findInstance(NixLanguage.class));
 
     @NotNull
     @Override
     public Lexer createLexer(Project project) {
-        NixLexer lxr = new NixLexer((Reader) null);
-        return new FlexAdapter(lxr);
+        return new NixLexer();
     }
 
     @NotNull
@@ -69,7 +67,21 @@ public PsiFile createFile(FileViewProvider viewProvider) {
 
     @Override
     public SpaceRequirements spaceExistenceTypeBetweenTokens(ASTNode left, ASTNode right) {
-        return SpaceRequirements.MAY;
+        NixTokenType leftType = Objects.castIfBelongsToType(left.getElementType(), NixTokenType.class);
+        NixTokenType rightType = Objects.castIfBelongsToType(right.getElementType(), NixTokenType.class);
+        if (leftType == NixTypes.SCOMMENT) {
+            return SpaceRequirements.MUST_LINE_BREAK;
+        }
+        if (leftType == NixTypes.DOLLAR && rightType == NixTypes.LCURLY) {
+            return SpaceRequirements.MUST_NOT;
+        }
+        else if (NixTokenType.MIGHT_COLLAPSE_WITH_ID.contains(leftType) &&
+                 NixTokenType.MIGHT_COLLAPSE_WITH_ID.contains(rightType)) {
+            return SpaceRequirements.MUST;
+        }
+        else {
+            return SpaceRequirements.MAY;
+        }
     }
 
     @NotNull

diff --git a/src/main/java/org/nixos/idea/lang/NixSyntaxHighlighter.java b/src/main/java/org/nixos/idea/lang/NixSyntaxHighlighter.java
@@ -28,7 +28,6 @@ public class NixSyntaxHighlighter extends SyntaxHighlighterBase {
 
     public static final TextAttributesKey ID = createTextAttributesKey("ID", DefaultLanguageHighlighterColors.IDENTIFIER);
     public static final TextAttributesKey INT = createTextAttributesKey("INT", DefaultLanguageHighlighterColors.NUMBER);
-    public static final TextAttributesKey BOOL = createTextAttributesKey("BOOL", DefaultLanguageHighlighterColors.PREDEFINED_SYMBOL);
 
     public static final TextAttributesKey ASSERT = createTextAttributesKey("ASSERT", DefaultLanguageHighlighterColors.KEYWORD);
     public static final TextAttributesKey IF = createTextAttributesKey("IF", DefaultLanguageHighlighterColors.KEYWORD);
@@ -40,10 +39,6 @@ public class NixSyntaxHighlighter extends SyntaxHighlighterBase {
     public static final TextAttributesKey WITH = createTextAttributesKey("WITH", DefaultLanguageHighlighterColors.KEYWORD);
     public static final TextAttributesKey INHERIT = createTextAttributesKey("INHERIT", DefaultLanguageHighlighterColors.KEYWORD);
     public static final TextAttributesKey OR_KW = createTextAttributesKey("OR_KW", DefaultLanguageHighlighterColors.KEYWORD);
-    public static final TextAttributesKey IMPORT = createTextAttributesKey("IMPORT", DefaultLanguageHighlighterColors.KEYWORD);
-    public static final TextAttributesKey IMPORTS = createTextAttributesKey("IMPORTS", DefaultLanguageHighlighterColors.KEYWORD);
-    public static final TextAttributesKey REQUIRE = createTextAttributesKey("REQUIRE", DefaultLanguageHighlighterColors.KEYWORD);
-    public static final TextAttributesKey REQUIRES = createTextAttributesKey("REQUIRES", DefaultLanguageHighlighterColors.KEYWORD);
 
     public static final TextAttributesKey GT = createTextAttributesKey("GT", DefaultLanguageHighlighterColors.OPERATION_SIGN);
     public static final TextAttributesKey LT = createTextAttributesKey("LT", DefaultLanguageHighlighterColors.OPERATION_SIGN);
@@ -91,7 +86,6 @@ public class NixSyntaxHighlighter extends SyntaxHighlighterBase {
     public static final TextAttributesKey[] COLON_KEYS = new TextAttributesKey[]{COLON};
     public static final TextAttributesKey[] ID_KEYS = new TextAttributesKey[]{ID};
     public static final TextAttributesKey[] NUMBER_KEYS = new TextAttributesKey[]{INT};
-    public static final TextAttributesKey[] PREDEFINED_KEYS = new TextAttributesKey[]{BOOL};
     public static final TextAttributesKey[] OPERATOR_KEYS = new TextAttributesKey[]{
             GT,LT,LEQ,GEQ,NEQ,EQ,PLUS,MINUS,DIVIDE,TIMES,NOT,AND,OR,IMPL,UPDATE,IS,NAMED,CONCAT,DOT,DOLLAR_CURLY
     };
@@ -100,18 +94,16 @@ public class NixSyntaxHighlighter extends SyntaxHighlighterBase {
     public static final TextAttributesKey[] FUNCTION_KEYS = new TextAttributesKey[]{LAMBDA};
     public static final TextAttributesKey[] PARAMETER_KEYS = new TextAttributesKey[]{FORMAL};
 
-    @NotNull
     @Override
-    public Lexer getHighlightingLexer() {
-        return new NixLexerAdapter();
+    public @NotNull Lexer getHighlightingLexer() {
+        return new NixLexer();
     }
 
-    @NotNull
     @Override
-    public TextAttributesKey[] getTokenHighlights(IElementType tokenType) {
+    public TextAttributesKey @NotNull [] getTokenHighlights(IElementType tokenType) {
         {
             if (tokenType == NixTypes.PARAM) return PARAMETER_KEYS;
-            if (tokenType == NixTypes.LAMBDA) return FUNCTION_KEYS;
+            if (tokenType == NixTypes.EXPR_LAMBDA) return FUNCTION_KEYS;
             if (tokenType == NixTypes.COMMA) return COMMA_KEYS;
             if (tokenType == NixTypes.SEMI) return SEMI_KEYS;
             if (tokenType == NixTypes.COLON) {
@@ -126,9 +118,6 @@ public TextAttributesKey[] getTokenHighlights(IElementType tokenType) {
             if (tokenType == NixTypes.ID) {
                 return ID_KEYS;
             }
-            if (tokenType == NixTypes.BOOL) {
-                return PREDEFINED_KEYS;
-            }
             if (tokenType == NixTypes.INT) {
                 return NUMBER_KEYS;
             }
@@ -148,8 +137,8 @@ public TextAttributesKey[] getTokenHighlights(IElementType tokenType) {
                     tokenType == NixTypes.OR ||
                     tokenType == NixTypes.IMPL ||
                     tokenType == NixTypes.UPDATE ||
-                    tokenType == NixTypes.IS ||
-                    tokenType == NixTypes.NAMED ||
+                    tokenType == NixTypes.HAS ||
+                    tokenType == NixTypes.AT ||
                     tokenType == NixTypes.CONCAT ||
                     tokenType == NixTypes.DOT )
                 return OPERATOR_KEYS;
@@ -159,29 +148,24 @@ public TextAttributesKey[] getTokenHighlights(IElementType tokenType) {
                     tokenType == NixTypes.RPAREN ||
                     tokenType == NixTypes.LPAREN ||
                     tokenType == NixTypes.RCURLY ||
-                    tokenType == NixTypes.DOLLAR_CURLY ||
+                    tokenType == NixTypes.DOLLAR ||
                     tokenType == NixTypes.LCURLY) {
                 return PAREN_KEYS;
             }
             if (tokenType == NixTypes.STR || tokenType == NixTypes.IND_STR) {
                 return STRING_KEYS;
             }
             if (
-                    tokenType == NixTypes.ASSERT ||
-                    tokenType == NixTypes.IN ||
-                    tokenType == NixTypes.OR_KW ||
-                    tokenType == NixTypes.REC ||
                     tokenType == NixTypes.IF ||
-                    tokenType == NixTypes.ELSE ||
                     tokenType == NixTypes.THEN ||
-                    tokenType == NixTypes.LET ||
+                    tokenType == NixTypes.ELSE ||
+                    tokenType == NixTypes.ASSERT ||
                     tokenType == NixTypes.WITH ||
+                    tokenType == NixTypes.LET ||
                     tokenType == NixTypes.IN ||
+                    tokenType == NixTypes.REC ||
                     tokenType == NixTypes.INHERIT ||
-                    tokenType == NixTypes.IMPORT ||
-                    tokenType == NixTypes.IMPORTS ||
-                    tokenType == NixTypes.REQUIRE ||
-                    tokenType == NixTypes.REQUIRES) {
+                    tokenType == NixTypes.OR_KW) {
                 return KEYWORD_KEYS;
             }
             return EMPTY_ARRAY;

diff --git a/src/main/java/org/nixos/idea/psi/NixParserUtil.java b/src/main/java/org/nixos/idea/psi/NixParserUtil.java