NixOS · Jun 15, 2020 · Jun 15, 2020 · Jun 16, 2020 · Jun 18, 2020 · Jun 18, 2020
diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml
@@ -8,19 +8,32 @@ jobs:
     strategy:
       matrix:
         channel:
-          - nixos-unstable
-          - nixos-20.03
-          - nixos-19.09
+          - unstable
+          - 20.03
+          - 19.09
       fail-fast: true
     steps:
     - name: Checking out the repository
       uses: actions/checkout@v2
       with:
         fetch-depth: 0
+
     - name: Installing Nix
       uses: cachix/install-nix-action@v8
+
+    - name: Installing NixFlakes
+      run: |
+        nix-env -iA nixpkgs.nixFlakes
+        echo "::add-path::$HOME/.nix-profile/bin"
+        echo 'experimental-features = nix-command flakes' | sudo tee -a /etc/nix/nix.conf
+        nix --version
+        cat /etc/nix/nix.conf
+
+    - name: Building import_scripts
+      run: |
+        nix build ./#packages.x86_64-linux.import_scripts
+
     - name: Import ${{ matrix.channel }} channel
       run: |
-        cd scripts/
-        ./import-channel --es-url ${{ secrets.ELASTICSEARCH_URL }} --channel ${{ matrix.channel }} -vvv
+        ./result/bin/import-channel --es-url ${{ secrets.ELASTICSEARCH_URL }} --channel ${{ matrix.channel }} -vvv
       if: github.repository == 'NixOS/nixos-search'
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -12,13 +12,28 @@ jobs:
       uses: actions/checkout@v2
       with:
         fetch-depth: 0
+
     - name: Installing Nix
-      uses: cachix/install-nix-action@v8
+      uses: cachix/install-nix-action@v9
+
+    - name: Installing NixFlakes
+      run: |
+        nix-env -iA nixpkgs.nixFlakes
+        echo "::add-path::$HOME/.nix-profile/bin"
+        echo 'experimental-features = nix-command flakes' | sudo tee -a /etc/nix/nix.conf
+        nix --version
+        cat /etc/nix/nix.conf
+
+    - name: Building import_scripts
+      run: |
+        nix build ./#packages.x86_64-linux.import_scripts
+
     - name: Building search.nixos.org
       run: |
-        nix-build
+        nix build ./#packages.x86_64-linux.frontend
         mkdir ./dist
         cp -RL ./result/* ./dist/
+
     - name: Deploy to Netlify
       uses: nwtgck/actions-netlify@v1.0.5
       env:

diff --git a/.gitignore b/.gitignore
@@ -31,3 +31,4 @@ dist
 package-lock.json
 result
 scripts/eval-*
+eval-*
diff --git a/README.md b/README.md
@@ -18,6 +18,24 @@ For backend we are using Elasticsearch instance which is kindly sponsored by
 [Elm](https://elm-lang.org).
 
 
+## How search works?
+
+The use case we want to solve is that a visitor want to see if a package
+exists or to look up certain package's details.
+
+A user wants to converge to a single result if possible. The more characters
+are added to a search query the more narrow is search is and we should show
+less results.
+
+Very important is also ranking of search results. This will bring more relevant
+search results to the top, since a lot of times it is hard to produce search
+query that will output only one result item.
+
+A less important, but providing better user experience. are suggestions for
+writing better search query. Suggesting feature should guide user to write
+better queries which in turn will produce better results.
+
+
 ## Ideas we want to explore
 
 Apart from searching packages and options we would like to:

diff --git a/VERSION b/VERSION
@@ -0,0 +1 @@
+7
diff --git a/default.nix b/default.nix
@@ -1,15 +1,9 @@
-{ pkgs ? import <nixpkgs> {}
+{ pkgs ? import <nixpkgs> { }
+, version ? "0"
 }:
-
 let
   package = builtins.fromJSON (builtins.readFile ./package.json);
-  yarn2nix = import (pkgs.fetchFromGitHub {
-    owner = "moretea";
-    repo = "yarn2nix";
-    rev = "9e7279edde2a4e0f5ec04c53f5cd64440a27a1ae";
-    sha256 = "sha256-x77mYSvwA4bd/uCwL0rRw+8mwH+dR/UfWnn4YXmm4n8=";
-  }) { inherit pkgs; };
-  yarnPkg = yarn2nix.mkYarnPackage rec {
+  yarnPkg = pkgs.yarn2nix-moretea.mkYarnPackage rec {
     name = "${package.name}-yarn-${package.version}";
     src = null;
     dontUnpack = true;
@@ -28,24 +22,27 @@ let
     '';
     pkgConfig = {
       node-sass = {
-        buildInputs = [ pkgs.python pkgs.libsass pkgs.pkgconfig];
+        buildInputs = [ pkgs.python pkgs.libsass pkgs.pkgconfig ];
         postInstall = ''
           LIBSASS_EXT=auto yarn --offline run build
           rm build/config.gypi
         '';
       };
     };
     publishBinsFor =
-      [ "webpack"
+      [
+        "webpack"
         "webpack-dev-server"
       ];
   };
-in pkgs.stdenv.mkDerivation {
+in
+pkgs.stdenv.mkDerivation {
   name = "${package.name}-${package.version}";
   src = pkgs.lib.cleanSource ./.;
 
   buildInputs =
-    [ yarnPkg
+    [
+      yarnPkg
     ] ++
     (with pkgs; [
       nodejs
@@ -76,6 +73,7 @@ in pkgs.stdenv.mkDerivation {
   buildPhase = ''
     # Yarn writes cache directories etc to $HOME.
     export HOME=$PWD/yarn_home
+    sed -i -e "s|process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION|${version}|" src/index.js
     yarn prod
   '';
 
@@ -88,6 +86,7 @@ in pkgs.stdenv.mkDerivation {
     rm -rf node_modules
     ln -sf ${yarnPkg}/libexec/${package.name}/node_modules .
     export PATH=$PWD/node_modules/.bin:$PATH
+    export ELASTICSEARCH_MAPPING_SCHEMA_VERSION=${version}
     echo "============================"
     echo "= To develop run: yarn dev ="
     echo "============================"

diff --git a/elm.json b/elm.json
@@ -12,6 +12,7 @@
             "elm/html": "1.0.0",
             "elm/http": "2.0.0",
             "elm/json": "1.1.3",
+            "elm/regex": "1.0.0",
             "elm/url": "1.0.0",
             "hecrj/html-parser": "2.3.4",
             "krisajenkins/remotedata": "6.0.1",
@@ -21,7 +22,6 @@
             "elm/bytes": "1.0.8",
             "elm/file": "1.0.5",
             "elm/parser": "1.1.0",
-            "elm/regex": "1.0.0",
             "elm/time": "1.0.0",
             "elm/virtual-dom": "1.0.2",
             "rtfeldman/elm-hex": "1.0.0"

diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -0,0 +1,35 @@
+{
+  description = "Code behind search.nixos.org";
+
+  inputs = {
+    nixpkgs = { url = "nixpkgs/nixos-unstable"; };
+    poetry2nix = { url = "github:nix-community/poetry2nix"; };
+  };
+
+  outputs = { self, nixpkgs, poetry2nix }:
+    let
+      systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
+      forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
+      allPackages = system:
+        let
+          pkgs = import nixpkgs {
+            inherit system;
+            overlays = [
+              poetry2nix.overlay
+            ];
+          };
+          version = pkgs.lib.removeSuffix "\n" (builtins.readFile "${self}/VERSION");
+        in
+        {
+          import_scripts = import ./import-scripts {
+            inherit pkgs version;
+          };
+          frontend = import ./. {
+            inherit pkgs version;
+          };
+        };
+    in
+    {
+      packages = forAllSystems allPackages;
+    };
+}
diff --git a/import-scripts/default.nix b/import-scripts/default.nix
@@ -0,0 +1,23 @@
+{ pkgs ? import <nixpkgs> { }
+, version ? "0"
+}:
+let
+  inherit (pkgs.poetry2nix) mkPoetryApplication overrides;
+in
+mkPoetryApplication {
+  projectDir = ./.;
+  overrides = overrides.withDefaults (self: super: {
+    pypandoc = super.pypandoc.overridePythonAttrs (old: {
+      postPatch = ''
+        sed -i '/^__pandoc_path = None$/c__pandoc_path = "${pkgs.pandoc}/bin/pandoc"' pypandoc/__init__.py
+      '';
+    });
+  });
+  checkPhase = ''
+    black --diff --check ./import_scripts
+    flake8 --ignore W503,E501,E265,E203 ./import_scripts
+  '';
+  postInstall = ''
+    wrapProgram $out/bin/import-channel --set INDEX_SCHEMA_VERSION "${version}"
+  '';
+}
diff --git a/import-scripts/import_scripts/__init__.py b/import-scripts/import_scripts/__init__.py
diff --git a/scripts/import-channel → import-scripts/import_scripts/channel.py b/scripts/import-channel → import-scripts/import_scripts/channel.py
diff --git a/import-scripts/import_scripts/packages-config.nix b/import-scripts/import_scripts/packages-config.nix
@@ -0,0 +1,42 @@
+{
+  # Ensures no aliases are in the results.
+  allowAliases = false;
+
+  # Enable recursion into attribute sets that nix-env normally doesn't look into
+  # so that we can get a more complete picture of the available packages for the
+  # purposes of the index.
+  packageOverrides = super:
+  let
+    recurseIntoAttrs = sets:
+      super.lib.genAttrs
+        (builtins.filter (set: builtins.hasAttr set super) sets)
+        (set: super.recurseIntoAttrs (builtins.getAttr set super));
+  in recurseIntoAttrs [
+    "roundcubePlugins"
+    "emscriptenfastcompPackages"
+    "fdbPackages"
+    "nodePackages_latest"
+    "nodePackages"
+    "platformioPackages"
+    "haskellPackages"
+    "idrisPackages"
+    "sconsPackages"
+    "gns3Packages"
+    "quicklispPackagesClisp"
+    "quicklispPackagesSBCL"
+    "rPackages"
+    "apacheHttpdPackages_2_4"
+    "zabbix44"
+    "zabbix40"
+    "zabbix30"
+    "fusePackages"
+    "nvidiaPackages"
+    "sourceHanPackages"
+    "atomPackages"
+    "emacs25Packages"
+    "emacs26Packages"
+    "steamPackages"
+    "ut2004Packages"
+    "zeroadPackages"
+  ];
+}
diff --git a/import-scripts/poetry.lock b/import-scripts/poetry.lock
diff --git a/import-scripts/pyproject.toml b/import-scripts/pyproject.toml
@@ -0,0 +1,31 @@
+[tool.poetry]
+name = "import-scripts"
+version = "0.1.0"
+description = ""
+authors = ["Rok Garbas <rok@garbas.si>"]
+include = [
+  "import_scripts/packages-config.nix"
+]
+
+[tool.poetry.scripts]
+import-channel = 'import_scripts.channel:run'
+
+[tool.poetry.dependencies]
+python = "^3.8"
+requests = "^2.24.0"
+click = "^7.1.2"
+click-log = "^0.3.2"
+elasticsearch = "^7.8.0"
+boto3 = "^1.14.5"
+tqdm = "^4.46.1"
+pypandoc = "^1.5"
+
+[tool.poetry.dev-dependencies]
+ipdb = "^0.13.2"
+black = "^19.10b0"
+flake8 = "^3.8.3"
+mypy = "^0.780"
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"
diff --git a/scripts/packages-config.nix b/scripts/packages-config.nix
diff --git a/src/Page/Packages.elm b/src/Page/Packages.elm
diff --git a/src/Search.elm b/src/Search.elm
@@ -91,6 +91,7 @@ type alias ResultItem a =
     , id : String
     , score : Float
     , source : a
+    , matched_queries : Maybe (List String)
     }
 
 
@@ -622,8 +623,9 @@ decodeResultHitsTotal =
 
 decodeResultItem : Json.Decode.Decoder a -> Json.Decode.Decoder (ResultItem a)
 decodeResultItem decodeResultItemSource =
-    Json.Decode.map4 ResultItem
+    Json.Decode.map5 ResultItem
         (Json.Decode.field "_index" Json.Decode.string)
         (Json.Decode.field "_id" Json.Decode.string)
         (Json.Decode.field "_score" Json.Decode.float)
         (Json.Decode.field "_source" decodeResultItemSource)
+        (Json.Decode.maybe (Json.Decode.field "matched_queries" (Json.Decode.list Json.Decode.string)))
diff --git a/src/index.js b/src/index.js
@@ -6,7 +6,7 @@ const {Elm} = require('./Main');
 
 Elm.Main.init({
   flags: {
-    elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 5,
+    elasticsearchMappingSchemaVersion: process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION || 0,
     elasticsearchUrl: process.env.ELASTICSEARCH_URL || 'https://nixos-search-5886075189.us-east-1.bonsaisearch.net:443',
     elasticsearchUsername : process.env.ELASTICSEARCH_USERNAME || 'z3ZFJ6y2mR',
     elasticsearchPassword : process.env.ELASTICSEARCH_PASSWORD || 'ds8CEvALPf9pui7XG'
-Original file line number
+Diff line change
@@ @@ -31,3 +31,4 @@ dist @@
     package-lock.json
     result
     scripts/eval-*
+    eval-*