Changes

9 changed files (+78/-45)

• README.md
  • @@ -4,6 +4,8 @@ Semantic file search using FastEmbed and sqlite-vec
    ## Installation + TODO: THESE INSTRUCTIONS ARE OUTDATED. We probably want separate installation for users and installation for devs sections + Run `pip install -e .` or alternatively `uv sync`. For Intel GPUs, install OpenCL drivers (for example `intel-compute-runtime`) and run `uv sync --extra intel_gpu`. ## Usage
  • @@ -18,11 +20,11 @@ Check out [TIDY](https://github.com/slavabarkov/tidy) (for Android) and [rclip](https://github.com/yurijmikhalevich/rclip) for similar projects, although this one is probably fastest!
    ## TODO - - Support OSes other than Linux? It should be fairly portable now other than a few things like `os.environ["XDG_RUNTIME_DIR"]`. + - Come up with a less generic name + - Test portability - GUI - Investigate race condition bugs and stress test this more - Detect if `DIRS_TO_INDEX` contains nested dirs - Nvidia support (should be easy using `fastembed-gpu`) - - Put result symlinks in `$XDG_RUNTIME_DIR` to avoid cluttering `/tmp`? - Unload the image embedding model to save RAM unless doing a file to file search - - Write entry points in `pyproject.toml` for the client and server and write a systemd `.service` and `.socket` for running the server with socket activation + - Write a systemd `.service` and `.socket` for running the server with socket activation
• client.py (deleted)
  • @@ -1,26 +0,0 @@
    - import os - import sys - import tempfile - import xmlrpc.client - - from unixsocket import UnixStreamTransport - - sockpath = os.path.join(os.environ["XDG_RUNTIME_DIR"], "search.sock") - proxy = xmlrpc.client.ServerProxy( - "http://localhost", transport=UnixStreamTransport(sockpath) - ) - res = proxy.search(sys.argv[1], sys.argv[2]) - - if len(sys.argv) == 3: - print("\n".join(res)) - else: - temp_dir = tempfile.TemporaryDirectory(delete=False) - for i, path in enumerate(res): - os.symlink(path, os.path.join(temp_dir.name, str(i))) - print(temp_dir.name) - # Very slow - # elif sys.argv[3] == "sixel": - # for imgpath in res: - # print(imgpath) - # c = converter.SixelConverter(imgpath, h=50, w=50) - # c.write(sys.stdout)
• model.py > search/model.py
• pyproject.toml
  • @@ -7,6 +7,7 @@ requires-python = ">=3.12"
    dependencies = [ "fastembed>=0.5.0", "pillow-avif-plugin>=1.4.6", + "platformdirs>=4.3.6", "sqlite-vec>=0.1.6", "watchdog>=6.0.0", ]
  • @@ -15,3 +16,10 @@ [project.optional-dependencies]
    intel-gpu = [ "onnxruntime-openvino>=1.20.0", ] + + [project.scripts] + search-server = "search:server" + search-client = "search:client" + + [tool.uv] + package = true
• search/__init__.py (new)
• search/client.py (new)
  • @@ -0,0 +1,31 @@
    + import os + import sys + import uuid + import xmlrpc.client + + from platformdirs import PlatformDirs + + from .unixsocket import UnixStreamTransport + + dirs = PlatformDirs("search", "unnamed") + os.makedirs(dirs.user_runtime_dir, exist_ok=True) + sockpath = os.path.join(dirs.user_runtime_dir, "sock") + proxy = xmlrpc.client.ServerProxy( + "http://localhost", transport=UnixStreamTransport(sockpath) + ) + res = proxy.search(sys.argv[1], sys.argv[2]) + + if len(sys.argv) == 3: + print("\n".join(res)) + else: + res_dir_name = os.path.join(dirs.user_runtime_dir, str(uuid.uuid4())) + os.mkdir(res_dir_name) + for i, path in enumerate(res): + os.symlink(path, os.path.join(res_dir_name, str(i))) + print(res_dir_name) + # Very slow + # elif sys.argv[3] == "sixel": + # for imgpath in res: + # print(imgpath) + # c = converter.SixelConverter(imgpath, h=50, w=50) + # c.write(sys.stdout)
• server.py > search/server.py
  • @@ -8,16 +8,21 @@
    import numpy as np import PIL import sqlite_vec + from platformdirs import PlatformDirs from watchdog.events import FileOpenedEvent, FileSystemEventHandler from watchdog.observers import Observer - import model - from unixsocket import UnixStreamXMLRPCServer + from .model import embed_text, embed_image + from .unixsocket import UnixStreamXMLRPCServer DIM = 768 + dirs = PlatformDirs("search", "unnamed") print("Connecting to DB") - con = sqlite3.connect("search.db", check_same_thread=False) + os.makedirs(dirs.user_data_dir, exist_ok=True) + con = sqlite3.connect( + os.path.join(dirs.user_data_dir, "index.db"), check_same_thread=False + ) con.execute("PRAGMA journal_mode=WAL") con.enable_load_extension(True) sqlite_vec.load(con)
  • @@ -90,7 +95,7 @@ # Only support image embeddings for now
    return try: - emb = model.embed_image(path) + emb = embed_image(path) except PIL.UnidentifiedImageError: print("Couldn't index", path) return
  • @@ -137,9 +142,9 @@
    def search(text, limit): print("Search", text, limit) if os.path.exists(text): - emb = model.embed_image(text) + emb = embed_image(text) else: - emb = model.embed_text(text) + emb = embed_text(text) res = cur.execute( "SELECT path FROM idx WHERE emb MATCH ? AND k = ? ORDER BY distance", (emb, limit),
  • @@ -169,7 +174,9 @@ con.commit()
    print("Starting RPC server") - sockpath = os.path.join(os.environ["XDG_RUNTIME_DIR"], "search.sock") + + os.makedirs(dirs.user_runtime_dir, exist_ok=True) + sockpath = os.path.join(dirs.user_runtime_dir, "sock") pathlib.Path(sockpath).unlink(missing_ok=True) server = UnixStreamXMLRPCServer(sockpath) server.register_function(search)
• unixsocket.py > search/unixsocket.py
• uv.lock
  • @@ -376,17 +376,26 @@ { url = "https://files.pythonhosted.org/packages/30/1e/315c22079058ffc2b8c6028cb18e33f7dc420ec56c806e8428419347f427/pillow_avif_plugin-1.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:10e9b2ef297a9825b461715359ae233d6518d9863c877a8652c14d6acae6e9f0", size = 10729094 },
    ] [[package]] + name = "platformdirs" + version = "4.3.6" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/13/fc/128cc9cb8f03208bdbf93d3aa862e16d376844a14f9a0ce5cf4507372de4/platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907", size = 21302 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb", size = 18439 }, + ] + + [[package]] name = "protobuf" - version = "5.29.2" + version = "5.29.3" source = { registry = "https://pypi.org/simple" } - sdist = { url = "https://files.pythonhosted.org/packages/a5/73/4e6295c1420a9d20c9c351db3a36109b4c9aa601916cb7c6871e3196a1ca/protobuf-5.29.2.tar.gz", hash = "sha256:b2cc8e8bb7c9326996f0e160137b0861f1a82162502658df2951209d0cb0309e", size = 424901 } + sdist = { url = "https://files.pythonhosted.org/packages/f7/d1/e0a911544ca9993e0f17ce6d3cc0932752356c1b0a834397f28e63479344/protobuf-5.29.3.tar.gz", hash = "sha256:5da0f41edaf117bde316404bad1a486cb4ededf8e4a54891296f648e8e076620", size = 424945 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/42/6db5387124708d619ffb990a846fb123bee546f52868039f8fa964c5bc54/protobuf-5.29.2-cp310-abi3-win32.whl", hash = "sha256:c12ba8249f5624300cf51c3d0bfe5be71a60c63e4dcf51ffe9a68771d958c851", size = 422697 }, - { url = "https://files.pythonhosted.org/packages/6c/38/2fcc968b377b531882d6ab2ac99b10ca6d00108394f6ff57c2395fb7baff/protobuf-5.29.2-cp310-abi3-win_amd64.whl", hash = "sha256:842de6d9241134a973aab719ab42b008a18a90f9f07f06ba480df268f86432f9", size = 434495 }, - { url = "https://files.pythonhosted.org/packages/cb/26/41debe0f6615fcb7e97672057524687ed86fcd85e3da3f031c30af8f0c51/protobuf-5.29.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a0c53d78383c851bfa97eb42e3703aefdc96d2036a41482ffd55dc5f529466eb", size = 417812 }, - { url = "https://files.pythonhosted.org/packages/e4/20/38fc33b60dcfb380507b99494aebe8c34b68b8ac7d32808c4cebda3f6f6b/protobuf-5.29.2-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:494229ecd8c9009dd71eda5fd57528395d1eacdf307dbece6c12ad0dd09e912e", size = 319562 }, - { url = "https://files.pythonhosted.org/packages/90/4d/c3d61e698e0e41d926dbff6aa4e57428ab1a6fc3b5e1deaa6c9ec0fd45cf/protobuf-5.29.2-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:b6b0d416bbbb9d4fbf9d0561dbfc4e324fd522f61f7af0fe0f282ab67b22477e", size = 319662 }, - { url = "https://files.pythonhosted.org/packages/f3/fd/c7924b4c2a1c61b8f4b64edd7a31ffacf63432135a2606f03a2f0d75a750/protobuf-5.29.2-py3-none-any.whl", hash = "sha256:fde4554c0e578a5a0bcc9a276339594848d1e89f9ea47b4427c80e5d72f90181", size = 172539 }, + { url = "https://files.pythonhosted.org/packages/dc/7a/1e38f3cafa022f477ca0f57a1f49962f21ad25850c3ca0acd3b9d0091518/protobuf-5.29.3-cp310-abi3-win32.whl", hash = "sha256:3ea51771449e1035f26069c4c7fd51fba990d07bc55ba80701c78f886bf9c888", size = 422708 }, + { url = "https://files.pythonhosted.org/packages/61/fa/aae8e10512b83de633f2646506a6d835b151edf4b30d18d73afd01447253/protobuf-5.29.3-cp310-abi3-win_amd64.whl", hash = "sha256:a4fa6f80816a9a0678429e84973f2f98cbc218cca434abe8db2ad0bffc98503a", size = 434508 }, + { url = "https://files.pythonhosted.org/packages/dd/04/3eaedc2ba17a088961d0e3bd396eac764450f431621b58a04ce898acd126/protobuf-5.29.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a8434404bbf139aa9e1300dbf989667a83d42ddda9153d8ab76e0d5dcaca484e", size = 417825 }, + { url = "https://files.pythonhosted.org/packages/4f/06/7c467744d23c3979ce250397e26d8ad8eeb2bea7b18ca12ad58313c1b8d5/protobuf-5.29.3-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:daaf63f70f25e8689c072cfad4334ca0ac1d1e05a92fc15c54eb9cf23c3efd84", size = 319573 }, + { url = "https://files.pythonhosted.org/packages/a8/45/2ebbde52ad2be18d3675b6bee50e68cd73c9e0654de77d595540b5129df8/protobuf-5.29.3-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:c027e08a08be10b67c06bf2370b99c811c466398c357e615ca88c91c07f0910f", size = 319672 }, + { url = "https://files.pythonhosted.org/packages/fd/b2/ab07b09e0f6d143dfb839693aa05765257bceaa13d03bf1a696b78323e7a/protobuf-5.29.3-py3-none-any.whl", hash = "sha256:0a18ed4a24198528f2333802eb075e59dea9d679ab7a6c5efb017a59004d849f", size = 172550 }, ] [[package]]
  • @@ -460,10 +469,11 @@
    [[package]] name = "search" version = "0.1.0" - source = { virtual = "." } + source = { editable = "." } dependencies = [ { name = "fastembed" }, { name = "pillow-avif-plugin" }, + { name = "platformdirs" }, { name = "sqlite-vec" }, { name = "watchdog" }, ]
  • @@ -478,6 +488,7 @@ requires-dist = [
    { name = "fastembed", specifier = ">=0.5.0" }, { name = "onnxruntime-openvino", marker = "extra == 'intel-gpu'", specifier = ">=1.20.0" }, { name = "pillow-avif-plugin", specifier = ">=1.4.6" }, + { name = "platformdirs", specifier = ">=4.3.6" }, { name = "sqlite-vec", specifier = ">=0.1.6" }, { name = "watchdog", specifier = ">=6.0.0" }, ]