Release v0.2.0: Add HuggingFace model resolution and encoding caching

This commit introduces the ability to load Kimi-compatible encodings directly
from HuggingFace repositories. It includes a new resolution mechanism that
downloads and caches required files locally.

Key changes:

Introduced TiktokenEx.HuggingFace for resolving files from HuggingFace. It
includes local file system caching under the user cache directory, atomic
writes to prevent corruption during concurrent access, and injectable
fetchers for network-independent testing.

Added TiktokenEx.Cache, an optional ETS-based cache for built encodings.
This allows users to reuse built encoding structures across their
application by keying them against repository name and revision.

Expanded TiktokenEx.Kimi with the from_hf_repo/2 function. This simplifies
the workflow for using Kimi-style models by automatically fetching the
tiktoken.model and tokenizer_config.json files from the remote repository.

Updated dependencies and project configuration. The package now includes
Inets, SSL, and Public Key as extra applications to support HTTPS
downloads. Credo has been added for static analysis, and the version
is bumped to 0.2.0.

Comprehensive tests were added for the cache logic, file resolution
sanitization, and the integration between Kimi and HuggingFace.

Author: nshkrdotcom

CHANGELOG.md

@@ -0,0 +1,9 @@
+# Changelog
+
+## 0.2.0 - 2025-12-27
+- Added HuggingFace file resolution with local caching and injectable fetchers.
+- Added `Kimi.from_hf_repo/2` with optional ETS encoding caching.
+- Added `TiktokenEx.Cache` helper for opt-in encoding reuse.
+
+## 0.1.0 - 2025-12-12
+- Initial release.

README.md

@@ -28,7 +28,7 @@ Add `tiktoken_ex` to your dependencies:
 ```elixir
 def deps do
   [
-    {:tiktoken_ex, "~> 0.1.0"}
+    {:tiktoken_ex, "~> 0.2.0"}
   ]
 end
 ```
@@ -76,6 +76,26 @@ alias TiktokenEx.{Encoding, Kimi}
 {:ok, decoded} = Encoding.decode(enc, ids)
 ```
 
+### Load a Kimi K2 encoding from a HuggingFace repo (cached)
+
+`from_hf_repo/2` downloads and caches `tiktoken.model` and
+`tokenizer_config.json` under your user cache directory.
+
+```elixir
+alias TiktokenEx.{Encoding, Kimi}
+
+{:ok, enc} =
+  Kimi.from_hf_repo(
+    "moonshotai/Kimi-K2-Thinking",
+    revision: "main",
+    encoding_cache: true
+  )
+
+{:ok, ids} = Encoding.encode(enc, "Say hi")
+```
+
+To test without network, inject a `:fetch_fun` (see `TiktokenEx.HuggingFace`).
+
 ### Special tokens
 
 Special tokens are recognized by default. To treat them as plain text:

lib/tiktoken_ex/cache.ex

@@ -0,0 +1,64 @@
+defmodule TiktokenEx.Cache do
+  @moduledoc """
+  Optional ETS cache for encodings keyed by repo and revision.
+
+  This cache is opt-in and safe to ignore in most usage.
+  """
+
+  @table __MODULE__
+
+  @spec get_or_load(term(), (-> {:ok, term()} | {:error, term()})) ::
+          {:ok, term()} | {:error, term()}
+  def get_or_load(key, loader) when is_function(loader, 0) do
+    ensure_table()
+
+    case :ets.lookup(@table, key) do
+      [{^key, value}] ->
+        {:ok, value}
+
+      [] ->
+        case loader.() do
+          {:ok, value} ->
+            true = :ets.insert(@table, {key, value})
+            {:ok, value}
+
+          other ->
+            other
+        end
+    end
+  end
+
+  @spec clear() :: :ok
+  def clear do
+    case :ets.whereis(@table) do
+      :undefined ->
+        :ok
+
+      tid ->
+        :ets.delete_all_objects(tid)
+        :ok
+    end
+  end
+
+  defp ensure_table do
+    case :ets.whereis(@table) do
+      :undefined ->
+        try do
+          :ets.new(@table, [
+            :set,
+            :public,
+            :named_table,
+            read_concurrency: true,
+            write_concurrency: true
+          ])
+
+          :ok
+        rescue
+          ArgumentError -> :ok
+        end
+
+      _ ->
+        :ok
+    end
+  end
+end

lib/tiktoken_ex/encoding.ex

@@ -1,3 +1,5 @@
+# credo:disable-for-this-file Credo.Check.Refactor.Nesting
+# credo:disable-for-this-file Credo.Check.Refactor.CyclomaticComplexity
 defmodule TiktokenEx.Encoding do
   @moduledoc """
   A TikToken-style encoding: regex-based splitting + byte-pair encoding + specials.

lib/tiktoken_ex/hugging_face.ex

@@ -0,0 +1,157 @@
+defmodule TiktokenEx.HuggingFace do
+  @moduledoc """
+  Resolve HuggingFace files with a local cache and injectable fetchers.
+
+  The default fetcher uses `:httpc` and writes to the user cache directory.
+  """
+
+  @base_url "https://huggingface.co"
+
+  @spec resolve_file(String.t(), String.t(), String.t(), keyword()) ::
+          {:ok, String.t()} | {:error, term()}
+  def resolve_file(repo_id, revision, filename, opts \\ [])
+      when is_binary(repo_id) and is_binary(revision) and is_binary(filename) and is_list(opts) do
+    cache_root = Keyword.get(opts, :cache_dir, default_cache_dir())
+    repo_segment = sanitize_repo_id(repo_id)
+    path = Path.join([cache_root, "hf", repo_segment, revision, filename])
+
+    if File.exists?(path) do
+      {:ok, path}
+    else
+      with :ok <- File.mkdir_p(Path.dirname(path)),
+           {:ok, body} <- fetch_file(repo_id, revision, filename, opts),
+           :ok <- write_atomic(path, body) do
+        {:ok, path}
+      else
+        {:error, reason} -> {:error, reason}
+      end
+    end
+  end
+
+  defp fetch_file(repo_id, revision, filename, opts) do
+    case Keyword.get(opts, :fetch_fun) do
+      fun when is_function(fun, 4) ->
+        fun.(repo_id, revision, filename, opts)
+
+      nil ->
+        fetch_httpc(repo_id, revision, filename, opts)
+
+      other ->
+        {:error, {:invalid_fetch_fun, other}}
+    end
+  end
+
+  defp fetch_httpc(repo_id, revision, filename, opts) do
+    url = "#{@base_url}/#{repo_id}/resolve/#{revision}/#{filename}"
+    timeout_ms = Keyword.get(opts, :http_timeout_ms, 120_000)
+    headers = [{~c"user-agent", ~c"tiktoken_ex"}]
+
+    with :ok <- ensure_httpc_started() do
+      ssl_options =
+        [
+          verify: :verify_peer,
+          cacerts: public_key_cacerts(),
+          depth: 3
+        ]
+        |> maybe_add_hostname_check()
+
+      http_options = [
+        timeout: timeout_ms,
+        connect_timeout: timeout_ms,
+        autoredirect: true,
+        ssl: ssl_options
+      ]
+
+      options = [body_format: :binary, full_result: true]
+
+      case :httpc.request(:get, {String.to_charlist(url), headers}, http_options, options) do
+        {:ok, {{_, status, _}, _resp_headers, body}}
+        when is_integer(status) and status >= 200 and status < 300 ->
+          {:ok, body}
+
+        {:ok, {{_, 404, _}, _resp_headers, _body}} ->
+          {:error, {:not_found, repo_id, revision, filename}}
+
+        {:ok, {{_, status, _}, _resp_headers, body}} ->
+          {:error, {:http_status, status, body}}
+
+        {:error, reason} ->
+          {:error, {:http_error, reason}}
+      end
+    end
+  end
+
+  defp ensure_httpc_started do
+    with {:ok, _} <- Application.ensure_all_started(:inets),
+         {:ok, _} <- Application.ensure_all_started(:public_key),
+         {:ok, _} <- Application.ensure_all_started(:ssl) do
+      :ok
+    else
+      {:error, reason} -> {:error, {:httpc_start_failed, reason}}
+    end
+  end
+
+  defp default_cache_dir do
+    :filename.basedir(:user_cache, "tiktoken_ex")
+  end
+
+  defp sanitize_repo_id(repo_id) do
+    repo_id
+    |> String.replace("/", "__")
+    |> String.replace("..", "_")
+  end
+
+  defp write_atomic(path, body) when is_binary(path) and is_binary(body) do
+    dir = Path.dirname(path)
+    tmp_path = Path.join(dir, ".#{Path.basename(path)}.#{System.unique_integer([:positive])}.tmp")
+
+    with :ok <- File.write(tmp_path, body),
+         :ok <- finalize_atomic_write(tmp_path, path) do
+      :ok
+    else
+      {:error, reason} -> {:error, {:cache_write_failed, path, reason}}
+    end
+  end
+
+  defp finalize_atomic_write(tmp_path, path) do
+    case File.rename(tmp_path, path) do
+      :ok ->
+        :ok
+
+      {:error, reason} ->
+        _ = File.rm(tmp_path)
+
+        if File.exists?(path) do
+          :ok
+        else
+          {:error, reason}
+        end
+    end
+  end
+
+  defp public_key_cacerts do
+    if Code.ensure_loaded?(:public_key) and function_exported?(:public_key, :cacerts_get, 0) do
+      # credo:disable-for-next-line Credo.Check.Refactor.Apply
+      apply(:public_key, :cacerts_get, [])
+    else
+      []
+    end
+  end
+
+  defp maybe_add_hostname_check(ssl_options) do
+    case public_key_hostname_match_fun() do
+      nil -> ssl_options
+      match_fun -> Keyword.put(ssl_options, :customize_hostname_check, match_fun: match_fun)
+    end
+  end
+
+  defp public_key_hostname_match_fun do
+    if Code.ensure_loaded?(:public_key) and
+         function_exported?(:public_key, :pkix_verify_hostname_match_fun, 1) do
+      # credo:disable-for-next-line Credo.Check.Refactor.Apply
+      apply(:public_key, :pkix_verify_hostname_match_fun, [:https])
+    else
+      nil
+    end
+  end
+end

lib/tiktoken_ex/kimi.ex

@@ -10,7 +10,7 @@ defmodule TiktokenEx.Kimi do
   `TiktokenEx.Encoding` from HuggingFace-style tokenizer configs.
   """
 
-  alias TiktokenEx.Encoding
+  alias TiktokenEx.{Cache, Encoding, HuggingFace}
 
   @num_reserved_special_tokens 256
 
@@ -50,6 +50,49 @@ defmodule TiktokenEx.Kimi do
     |> translate_intersection_classes()
   end
 
+  @doc """
+  Load a Kimi-compatible encoding from a HuggingFace repo.
+
+  ## Options
+
+    * `:revision` - Git revision or tag (default: `"main"`).
+    * `:cache_dir` - Override the HuggingFace cache root.
+    * `:fetch_fun` - Custom fetcher `fun.(repo_id, revision, filename, opts)`.
+    * `:http_timeout_ms` - Timeout for the default HTTP fetcher.
+    * `:encoding_cache` - When true, reuse an ETS cache keyed by repo + revision.
+    * `:pat_str` - Override the pattern used for splitting.
+    * `:special_token_matching` - Pass-through to `TiktokenEx.Encoding.new/1`.
+  """
+  @spec from_hf_repo(String.t(), keyword()) :: {:ok, Encoding.t()} | {:error, term()}
+  def from_hf_repo(repo_id, opts \\ []) when is_binary(repo_id) and is_list(opts) do
+    revision = Keyword.get(opts, :revision, "main")
+    pat_str = Keyword.get(opts, :pat_str, pat_str())
+    special_token_matching = Keyword.get(opts, :special_token_matching, :parity)
+    use_encoding_cache = Keyword.get(opts, :encoding_cache, false)
+
+    hf_opts = Keyword.take(opts, [:cache_dir, :fetch_fun, :http_timeout_ms])
+
+    loader = fn ->
+      with {:ok, model_path} <-
+             HuggingFace.resolve_file(repo_id, revision, "tiktoken.model", hf_opts),
+           {:ok, config_path} <-
+             HuggingFace.resolve_file(repo_id, revision, "tokenizer_config.json", hf_opts) do
+        from_hf_files(
+          tiktoken_model_path: model_path,
+          tokenizer_config_path: config_path,
+          pat_str: pat_str,
+          special_token_matching: special_token_matching
+        )
+      end
+    end
+
+    if use_encoding_cache do
+      Cache.get_or_load({repo_id, revision, pat_str, special_token_matching}, loader)
+    else
+      loader.()
+    end
+  end
+
   @doc """
   Load a Kimi-compatible encoding from local HuggingFace files.
 
@@ -67,15 +110,13 @@ defmodule TiktokenEx.Kimi do
 
     with {:ok, mergeable_ranks} <- load_tiktoken_model(model_path),
          {:ok, config} <- load_json(config_path),
-         {:ok, special_tokens} <- build_special_tokens(config, map_size(mergeable_ranks)),
-         {:ok, encoding} <-
-           Encoding.new(
-             pat_str: pat_str,
-             mergeable_ranks: mergeable_ranks,
-             special_tokens: special_tokens,
-             special_token_matching: special_token_matching
-           ) do
-      {:ok, encoding}
+         {:ok, special_tokens} <- build_special_tokens(config, map_size(mergeable_ranks)) do
+      Encoding.new(
+        pat_str: pat_str,
+        mergeable_ranks: mergeable_ranks,
+        special_tokens: special_tokens,
+        special_token_matching: special_token_matching
+      )
     end
   end