diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index f6093eb7e14a..848cb5317baf 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1375,6 +1375,7 @@ in qemu-vm-volatile-root = runTest ./qemu-vm-volatile-root.nix; qgis = handleTest ./qgis.nix { package = pkgs.qgis; }; qgis-ltr = handleTest ./qgis.nix { package = pkgs.qgis-ltr; }; + qlever = runTest ./qlever.nix; qownnotes = runTest ./qownnotes.nix; qtile = runTestOn [ "x86_64-linux" "aarch64-linux" ] ./qtile/default.nix; qtile-extras = runTestOn [ "x86_64-linux" "aarch64-linux" ] ./qtile-extras/default.nix; diff --git a/nixos/tests/qlever.nix b/nixos/tests/qlever.nix new file mode 100644 index 000000000000..93e390d755d2 --- /dev/null +++ b/nixos/tests/qlever.nix @@ -0,0 +1,52 @@ +{ + pkgs, + lib, + ... +}: + +let + olympics-qleverfile = "${pkgs.qlever-control.src}/src/qlever/Qleverfiles/Qleverfile.olympics"; + + olympics-dataset = pkgs.fetchurl { + url = "https://github.com/wallscope/olympics-rdf/raw/54483d539082641d48e1d49873662b3af628ca4d/data/olympics-nt-nodup.zip"; + hash = "sha256-dY28CQKaMDUUw/pw+p9yX0EtJOnbAAplodMFaedL1B8="; + }; +in + +{ + name = "Basic QLever test"; + meta.maintainers = lib.teams.ngi.members; + + nodes = { + machine = + { + pkgs, + ... + }: + { + environment.systemPackages = with pkgs; [ + qlever + qlever-control + unzip + ]; + }; + }; + + testScript = + # python + '' + machine.succeed("cp ${olympics-dataset} /tmp/olympics.zip") + machine.succeed("cp ${olympics-qleverfile} /tmp/Qleverfile") + machine.succeed("unzip -q olympics.zip") + + # use qlever binary instead of docker + machine.succeed("sed -i 's/SYSTEM = docker/SYSTEM = native/' /tmp/Qleverfile") + + machine.succeed("qlever index") + + machine.succeed("qlever start") + machine.wait_for_open_port(7019) # default server port + + machine.succeed("qlever query") + ''; +} diff --git a/pkgs/by-name/ql/qlever/package.nix b/pkgs/by-name/ql/qlever/package.nix new file mode 100644 index 000000000000..0eac6aa36ba3 --- /dev/null +++ b/pkgs/by-name/ql/qlever/package.nix @@ -0,0 +1,170 @@ +{ + lib, + stdenv, + fetchFromGitHub, + + pkg-config, + cmake, + + boost, + bzip2, + icu, + openssl, + zlib, + zstd, + jemalloc, + + nixosTests, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "qlever"; + version = "0.5.46"; + + src = fetchFromGitHub { + owner = "ad-freiburg"; + repo = "qlever"; + tag = "v${finalAttrs.version}"; + hash = "sha256-UUfSWy1mImmyL7Ha2xCbxm9srdY/rTJS4oUJbJiDtwQ="; + fetchSubmodules = true; + }; + + strictDeps = true; + + nativeBuildInputs = [ + cmake + pkg-config + ]; + + buildInputs = [ + boost + bzip2 + icu + jemalloc + openssl + zlib + zstd + ]; + + env.NIX_CFLAGS_COMPILE = toString [ + # fixes error: inlining failed in call to 'always_inline' ... : + # function body can be overwritten at link time + "-fno-semantic-interposition" + ]; + + cmakeFlags = [ + (lib.cmakeFeature "CMAKE_BUILD_TYPE" "Release") + (lib.cmakeFeature "LOGLEVEL" "INFO") + (lib.cmakeBool "USE_PARALLEL" true) + (lib.cmakeBool "_NO_TIMING_TESTS" true) + (lib.cmakeBool "JEMALLOC_MANUALLY_INSTALLED" true) + + # disable fetching external dependencies + (lib.cmakeBool "USE_CONAN" false) + (lib.cmakeBool "FETCHCONTENT_FULLY_DISCONNECTED" true) + ] + ++ (with finalAttrs.passthru.deps; [ + # map external dependencies to FetchContent names + (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_FSST" "${fsst}") + (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_RE2" "${re2}") + (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_GOOGLETEST" "${googletest}") + (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_NLOHMANN-JSON" "${nlohmann-json}") + (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_ANTLR" "${antlr}") + (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_RANGE-V3" "${range-v3}") + (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_SPATIALJOIN" "${spatialjoin}") + (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_CTRE" "${ctre}") + (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_ABSEIL" "${abseil}") + (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_S2" "${s2}") + ]); + + passthru = { + deps = { + fsst = fetchFromGitHub { + owner = "cwida"; + repo = "fsst"; + rev = "b228af6356196095eaf9f8f5654b0635f969661e"; + hash = "sha256-XuE/nalt2HEYaII9NytUs0rCLGHOUFEclO+0h7pu4V0="; + }; + + re2 = fetchFromGitHub { + owner = "google"; + repo = "re2"; + rev = "bc0faab533e2b27b85b8ad312abf061e33ed6b5d"; + hash = "sha256-cKXe8r5MUag/z+seem4Zg/gmqIQjaCY7DBxiKlrnXPs="; + }; + + googletest = fetchFromGitHub { + owner = "google"; + repo = "googletest"; + rev = "7917641ff965959afae189afb5f052524395525c"; + hash = "sha256-Pfkx/hgtqryPz3wI0jpZwlRRco0s2FLcvUX1EgTGFIw="; + }; + + nlohmann-json = fetchFromGitHub { + owner = "nlohmann"; + repo = "json"; + tag = "v3.12.0"; + hash = "sha256-cECvDOLxgX7Q9R3IE86Hj9JJUxraDQvhoyPDF03B2CY="; + }; + + antlr = fetchFromGitHub { + owner = "antlr"; + repo = "antlr4"; + rev = "cc82115a4e7f53d71d9d905caa2c2dfa4da58899"; + hash = "sha256-DxxRL+FQFA+x0RudIXtLhewseU50aScHKSCDX7DE9bY="; + }; + + range-v3 = fetchFromGitHub { + owner = "joka921"; + repo = "range-v3"; + rev = "42340ef354f7b4e4660268b788e37008d9cc85aa"; + hash = "sha256-/17XLLLuEkcqeklVtqlgtu19tNTT3bLRHrU1aOPLhTw="; + }; + + spatialjoin = fetchFromGitHub { + owner = "ad-freiburg"; + repo = "spatialjoin"; + rev = "c358e479ebb5f40df99522e69a0b52d73416020b"; + hash = "sha256-/BQzyCx1KxnOeLLZkvqno2KN/VHAEu228zrsJaqYu/c="; + fetchSubmodules = true; + }; + + ctre = fetchFromGitHub { + owner = "hanickadot"; + repo = "compile-time-regular-expressions"; + rev = "e34c26ba149b9fd9c34aa0f678e39739641a0d1e"; + hash = "sha256-/44oZi6j8+a1D6ZGZpoy82GHjPtqzOvuS7d3SPbH7fs="; + }; + + abseil = fetchFromGitHub { + owner = "abseil"; + repo = "abseil-cpp"; + rev = "93ac3a4f9ee7792af399cebd873ee99ce15aed08"; + hash = "sha256-a18+Yj9fvDigza4b2g38L96hge5feMwU6fgPmL/KVQU="; + }; + + s2 = fetchFromGitHub { + owner = "google"; + repo = "s2geometry"; + rev = "5b5eccd54a08ae03b4467e79ffbb076d0b5f221e"; + hash = "sha256-VjgGcGgQlKmjUq+JU0JpyhOZ9pqwPcBUFEPGV9XoHc0="; + }; + }; + + tests = nixosTests.qlever; + + updateScript = ./update.sh; + }; + + __structuredAttrs = true; + + meta = { + description = "Graph database implementing the RDF and SPARQL standards"; + homepage = "https://github.com/ad-freiburg/qlever"; + mainProgram = "qlever"; + platforms = lib.platforms.all; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ eljamm ]; + teams = with lib.teams; [ ngi ]; + }; +}) diff --git a/pkgs/by-name/ql/qlever/update.sh b/pkgs/by-name/ql/qlever/update.sh new file mode 100755 index 000000000000..90e86cfee76c --- /dev/null +++ b/pkgs/by-name/ql/qlever/update.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env nix-shell +#!nix-shell -I nixpkgs=./. -i bash -p bash nix curl jq gawk gnused nixfmt + +set -euo pipefail + +SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")" +FILE_PATH="$SCRIPT_DIR/package.nix" + +LATEST_TAG=$(curl -s https://api.github.com/repos/ad-freiburg/qlever/releases/latest | jq -r '.tag_name') + +VERSION_OLD="${UPDATE_NIX_OLD_VERSION:-$(grep -oP '(?<=version = ")[^"]+' "$FILE_PATH")}" +VERSION_NEW="${LATEST_TAG#v}" + +if [[ "$VERSION_NEW" == "$VERSION_OLD" ]]; then + echo "QLever already up to date: $VERSION_NEW" + exit 0 +fi + +CMAKE_URL="https://raw.githubusercontent.com/ad-freiburg/qlever/$LATEST_TAG/CMakeLists.txt" +CMAKE_FILE=$(curl -s "$CMAKE_URL") + +prefetch_github_hash() { + local owner=$1 + local repo=$2 + local rev=$3 + local fetch_submodules=${4:-false} + + local expr="with import {}; fetchFromGitHub { \ + owner = \"$owner\"; \ + repo = \"$repo\"; \ + rev = \"$rev\"; \ + fetchSubmodules = $fetch_submodules; \ + hash = \"\"; \ + }" + + output=$(nix-build --no-out-link -E "$expr" 2>&1) + + echo "$output" | awk '/got:/ {print $NF}' +} + +# Extract the pinned revision/tag for a FetchContent_Declare block. +extract_git_tag() { + local dep_name=$1 + local result + + result=$(echo "$CMAKE_FILE" | awk -v dep="$dep_name" ' + /FetchContent_Declare\(/ { expect_name=1; next } + expect_name { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", $0) + in_block = ($0 == dep) + expect_name = 0 + next + } + in_block && /GIT_TAG/ { + for (i=1; i<=NF; i++) + if ($i == "GIT_TAG") { print $(i+1); exit } + } + in_block && /[[:space:]]URL[[:space:]]/ && !/URL_HASH/ { + for (i=1; i<=NF; i++) + if ($i == "URL") { print "URL:" $(i+1); exit } + } + in_block && /^[[:space:]]*\)/ { in_block=0 } + ') + + if [[ "$result" == URL:* ]]; then + # e.g. .../download/v3.12.0/json.tar.xz -> v3.12.0 + echo "${result#URL:}" | grep -Po '(?<=/download/)[^/]+' + else + echo "$result" + fi +} + +# Get the current revision/tag for a fetchFromGitHub block. +get_current_git_tag() { + local attr_name=$1 + local type=$2 + + awk "/$attr_name = fetchFromGitHub/,/};/" "$FILE_PATH" | + sed -n "s/.*$type = \"\([^\"]*\)\".*/\1/p" +} + +FAILED=() + +update_dep() { + local cmake_dep=$1 + local owner=$2 + local repo=$3 + local type=${4:-rev} + local submodules=${5:-false} + + echo "- $cmake_dep:" + + rev=$(extract_git_tag "$cmake_dep") + if [ -z "$rev" ]; then + echo " Error: could not find GIT_TAG for '$cmake_dep' in CMakeLists.txt" + FAILED+=("$cmake_dep") + return 0 + fi + + current_rev=$(get_current_git_tag "$cmake_dep" "$type") + if [ "$current_rev" = "$rev" ]; then + echo " Skipping: $type already up to date ($rev)" + return 0 + fi + + echo " $type: $current_rev -> $rev. Prefetching hash..." + + hash=$(prefetch_github_hash "$owner" "$repo" "$rev" "$submodules") + if [ -z "$hash" ]; then + echo " Error: Failed to prefetch hash for $owner/$repo@$rev (Is the repository/rev valid?)" + FAILED+=("$cmake_dep") + return 0 + fi + + sed -i "/$cmake_dep = fetchFromGitHub {/,/};/ s|$type = \".*\";|$type = \"$rev\";|" "$FILE_PATH" + sed -i "/$cmake_dep = fetchFromGitHub {/,/};/ s|hash = \".*\";|hash = \"$hash\";|" "$FILE_PATH" + echo " Done." +} + +echo "Updating QLever to $VERSION_NEW" + +sed -i "s|version = \"$VERSION_OLD\"|version = \"$VERSION_NEW\"|" "$FILE_PATH" + +HASH=$(prefetch_github_hash "ad-freiburg" "qlever" "$LATEST_TAG" true) +sed -i "/src = fetchFromGitHub {/,/};/ s|hash = \".*\";|hash = \"$HASH\";|" "$FILE_PATH" + +echo "" +echo "Updating dependencies:" + +# NOTE: cmake_dep must match the identifier on the line after FetchContent_Declare( + +# Read configuration line-by-line. Skip empty lines and comments. +while read -r cmake_dep owner repo type submodules; do + [[ -z "$cmake_dep" || "$cmake_dep" == \#* ]] && continue + + # use "-" as a placeholder for visual alignment + [[ "$type" == "-" ]] && type="" + [[ "$submodules" == "-" ]] && submodules="" + + update_dep "$cmake_dep" "$owner" "$repo" "${type:-rev}" "${submodules:-false}" +done <<'EOF' +# CMAKE_DEP OWNER REPO TYPE SUBMODULES +fsst cwida fsst - - +re2 google re2 - - +googletest google googletest - - +nlohmann-json nlohmann json tag - +antlr antlr antlr4 - - +range-v3 joka921 range-v3 - - +spatialjoin ad-freiburg spatialjoin - true +ctre hanickadot compile-time-regular-expressions - - +abseil abseil abseil-cpp - - +s2 google s2geometry - - +EOF + +echo "" +echo "Formatting $FILE_PATH" + +nixfmt "$FILE_PATH" + +if [ ${#FAILED[@]} -gt 0 ]; then + echo "" + echo "Warning: the following deps could not be updated:" + printf ' - %s\n' "${FAILED[@]}" + exit 1 +fi + +echo "" +echo "All dependencies updated for QLever $LATEST_TAG"