diff --git a/README.md b/README.md index 7034a3d..4be0f58 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Also works as a module: `python -m rdm_card_reader`. | `--demo` | – | Simulated reads; needs no hardware and no `rdm6300` | | `--demo-count` | `3` | Number of simulated reads in `--demo` | | `--no-install-deps` | off | Never install missing libraries automatically | -| `--wheel-dir` | – | Install missing libraries from local wheels (offline) | +| `--wheel-dir` | bundled | Install from local wheels in this DIR instead of the bundled ones | | `--version` | – | Print the version | With `--serve`: `--host` (`127.0.0.1`), `--bridge-port` (`8765`), `--mode` @@ -115,32 +115,46 @@ is what you want when piping into another program: ### Automatic dependency install -On first start the reader checks that it can import its libraries and installs -any that are missing, so a freshly flashed device works without manual setup: +The wheels for `rdm6300` and `pyserial` **ship inside this package** +(`rdm_card_reader/wheels/`), so a device with no internet can still set itself +up. On first start the reader installs whatever is missing from those wheels: ``` Reader library not found - checking dependencies... Missing reader dependencies: rdm6300 -Installing from PyPI ... +Using the wheels bundled with this package: + /opt/rdm-card-reader/.venv/lib/python3.13/site-packages/rdm_card_reader/wheels +Installing from .../rdm_card_reader/wheels Dependencies ready: rdm6300 ``` +No network access is needed. They ship with the package rather than being +fetched at build time, because the build machine and the target device are not +necessarily the same. + +Both wheels are **pure Python** (`py3-none-any` / `py2.py3-none-any`), which is +what makes this possible: the same files install on armv7l, aarch64 and x86_64. +This differs from the workstation app's `Files/repository/` cache, which held +architecture-specific binaries (aarch64 manylinux wheels) and so could not be +committed — that cache only ever existed on the Pi devices themselves. + * Nothing is installed when the libraries are already present, so a normal start does no network work at all. -* A failed install does not crash: it reports `exit 2` and tells you the exact - command to run by hand. +* The bundled install runs with `--no-index`, so it works with no route to PyPI + at all — no proxy or DNS needed. +* Wheels are verified against a bundled `SHA256SUMS` before installation. **A + wheel that fails its checksum is refused, not installed** — then it falls back + to PyPI, and if that is unreachable it reports `exit 2` rather than proceeding + with something unverified. * Outside a virtualenv pip is given `--break-system-packages`, which Debian and Raspberry Pi OS require before they will touch the system Python. +* A failed install does not crash: it reports `exit 2` and tells you the exact + command to run by hand. -For a device with **no internet**, put the wheels next to the app and install -from there — no index is contacted: - -```bash -pip download rdm6300 pyserial -d wheels # on a machine with internet -rdm-card-reader --serve --wheel-dir ./wheels -``` - -To manage dependencies yourself, pass `--no-install-deps`. +To install from your own wheel directory instead (for example a newer version +than the bundled one), pass `--wheel-dir`. An explicit directory that is +incomplete or fails verification is reported and then PyPI is used, rather than +failing outright. To manage dependencies yourself, pass `--no-install-deps`. ### Exit codes diff --git a/deploy/rdm-card-reader.service b/deploy/rdm-card-reader.service index eda0ddc..c1c6e28 100644 --- a/deploy/rdm-card-reader.service +++ b/deploy/rdm-card-reader.service @@ -21,7 +21,8 @@ WorkingDirectory=/opt/rdm-card-reader # --port points at the udev alias, so the reader is the same device every boot. # Autodetect refuses to guess if more than one USB adapter is present, which is # the safe behaviour: a wrong guess means authenticating the wrong person. -# Add --wheel-dir /opt/rdm-card-reader/wheels to install dependencies offline. +# The wheels for rdm6300 and pyserial are bundled inside the package, so no +# --wheel-dir is needed here: the install works with no internet. ExecStart=/opt/rdm-card-reader/.venv/bin/rdm-card-reader \ --serve \ --port /dev/rdm-reader \ diff --git a/pyproject.toml b/pyproject.toml index f5de87b..88d15ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ rdm-card-reader = "rdm_card_reader.cli:main" where = ["src"] [tool.setuptools.package-data] -rdm_card_reader = ["py.typed"] +rdm_card_reader = ["py.typed", "wheels/*.whl", "wheels/SHA256SUMS"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/rdm_card_reader/bootstrap.py b/src/rdm_card_reader/bootstrap.py index 1dbc7a5..eb2a9fa 100644 --- a/src/rdm_card_reader/bootstrap.py +++ b/src/rdm_card_reader/bootstrap.py @@ -5,6 +5,12 @@ dying with "please install this yourself", the reader installs it on first run and then carries on - the same behaviour the workstation app had, but without shelling out to a bare ``pip`` with no error handling. +The wheels for both libraries are **shipped inside this package** (in +``rdm_card_reader/wheels``), so a device with no internet can still set itself +up. They are pure-Python (``py3-none-any`` / ``py2.py3-none-any``), so the same +files work on any architecture - unlike the workstation app's cache, which held +architecture-specific binaries and therefore could not be committed. + Two things this deliberately does *not* do: * It never installs anything when the library is already importable, so a @@ -15,6 +21,7 @@ Two things this deliberately does *not* do: from __future__ import annotations +import hashlib import importlib import os import subprocess @@ -34,6 +41,89 @@ MANUAL_HINT = ( ) +def bundled_wheel_dir() -> Optional[str]: + """Directory of the wheels shipped inside this package. + + Returns ``None`` when it is absent, which is the case for a source checkout + that has not vendored them. + """ + folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), "wheels") + if not os.path.isdir(folder): + return None + if not any(name.endswith(".whl") for name in os.listdir(folder)): + return None + return folder + + +def local_wheel_missing(requirements: Sequence[str], wheel_dir: str) -> List[str]: + """Requirements that have no matching wheel in *wheel_dir*. + + ``--no-index`` makes pip fail outright if a wheel is absent, so check first + and fall back to PyPI rather than dying on a half-provisioned directory. + """ + try: + present = os.listdir(wheel_dir) + except OSError: + return list(requirements) + + missing = [] + for requirement in requirements: + # "rdm6300>=0.1.1" -> "rdm6300" + name = requirement.split(">=")[0].split("==")[0].split("[")[0].strip() + # Both sides are dash/underscore-normalised, so the prefix separator + # must be "_" - comparing against "name-" never matches. + normalised = name.replace("-", "_").lower() + if not any( + entry.replace("-", "_").lower().startswith(normalised + "_") + for entry in present + ): + missing.append(requirement) + return missing + + +def verify_wheels(wheel_dir: str, sums_file: Optional[str] = None) -> List[str]: + """Check wheels against ``SHA256SUMS``. Returns a list of problems. + + A tampered wheel is an arbitrary-code-execution risk on every device that + installs it, so a mismatch is reported rather than installed. + """ + problems: List[str] = [] + sums_path = sums_file or os.path.join(wheel_dir, "SHA256SUMS") + if not os.path.exists(sums_path): + return problems # nothing to check against + + try: + with open(sums_path, "r") as handle: + lines = [line.strip() for line in handle if line.strip()] + except OSError as exc: + return ["cannot read {}: {}".format(sums_path, exc)] + + for line in lines: + parts = line.split(None, 1) + if len(parts) != 2: + continue + expected, filename = parts[0], parts[1].lstrip("*").strip() + path = os.path.join(wheel_dir, filename) + if not os.path.exists(path): + problems.append("missing {}".format(filename)) + continue + digest = hashlib.sha256() + try: + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(65536), b""): + digest.update(chunk) + except OSError as exc: + problems.append("cannot read {}: {}".format(filename, exc)) + continue + if digest.hexdigest() != expected: + problems.append( + "checksum mismatch for {} (expected {}, got {})".format( + filename, expected[:12], digest.hexdigest()[:12] + ) + ) + return problems + + def missing_packages( packages: Sequence[tuple] = REQUIRED_PACKAGES, ) -> List[tuple]: @@ -136,8 +226,40 @@ def ensure_dependencies( python=sys.executable, requirements=" ".join(requirements))) return False + # Prefer the wheels shipped with the package: a device with no internet + # must still be able to set itself up. + explicit = wheel_dir is not None + if not explicit: + wheel_dir = bundled_wheel_dir() + if wheel_dir: + say("Using the wheels bundled with this package:") + say(" {}".format(wheel_dir)) + if wheel_dir: - say("Installing from local wheels in {} ...".format(wheel_dir)) + problems = verify_wheels(wheel_dir) + for problem in problems: + say(" x {}".format(problem)) + if problems: + # A bad or incomplete bundle is not fatal: fall back to PyPI rather + # than installing something we could not verify. + if explicit: + say(" x the --wheel-dir you gave is unusable") + say("Bundle unusable - falling back to PyPI.") + wheel_dir = None + else: + uncovered = local_wheel_missing(requirements, wheel_dir) + if uncovered: + # Do not attempt a --no-index install that is guaranteed to + # fail; PyPI covers whatever the bundle does not contain. + say(" local wheels do not cover: {}".format( + ", ".join(uncovered))) + if explicit: + say(" x the --wheel-dir you gave is incomplete") + say("Falling back to PyPI for those.") + wheel_dir = None + + if wheel_dir: + say("Installing from {}".format(wheel_dir)) else: say("Installing from PyPI ...") diff --git a/src/rdm_card_reader/wheels/SHA256SUMS b/src/rdm_card_reader/wheels/SHA256SUMS new file mode 100644 index 0000000..296ea5b --- /dev/null +++ b/src/rdm_card_reader/wheels/SHA256SUMS @@ -0,0 +1,2 @@ +c4451db6ba391ca6ca299fb3ec7bae67a5c55dde170964c7a14ceefec02f2cf0 pyserial-3.5-py2.py3-none-any.whl +afb067da8b8d23a477138ca67d4ff1d198f12d157b64668c9dd3cbd7df54e74e rdm6300-0.1.1-py3-none-any.whl diff --git a/src/rdm_card_reader/wheels/pyserial-3.5-py2.py3-none-any.whl b/src/rdm_card_reader/wheels/pyserial-3.5-py2.py3-none-any.whl new file mode 100644 index 0000000..7751288 Binary files /dev/null and b/src/rdm_card_reader/wheels/pyserial-3.5-py2.py3-none-any.whl differ diff --git a/src/rdm_card_reader/wheels/rdm6300-0.1.1-py3-none-any.whl b/src/rdm_card_reader/wheels/rdm6300-0.1.1-py3-none-any.whl new file mode 100644 index 0000000..4aebddf Binary files /dev/null and b/src/rdm_card_reader/wheels/rdm6300-0.1.1-py3-none-any.whl differ diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 9a5ece8..db6912f 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -7,6 +7,7 @@ clearly instead of crashing. import os import sys +import tempfile SRC = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "src" @@ -161,10 +162,11 @@ def test_no_install_flag_reports_manual_hint(monkeypatch): def test_wheel_dir_is_passed_through(monkeypatch): + """An explicit --wheel-dir that covers everything is used as given.""" seen = {} monkeypatch.setattr( bootstrap, "missing_packages", - lambda packages=None: [] if seen.get("done") else [("x", "x")], + lambda packages=None: [] if seen.get("done") else [("x", "x>=1")], ) def fake_install(requirements, wheel_dir=None, logger=None, timeout=300): @@ -173,5 +175,27 @@ def test_wheel_dir_is_passed_through(monkeypatch): return True monkeypatch.setattr(bootstrap, "install", fake_install) - ensure_dependencies(wheel_dir="/opt/wheels") - assert seen["wheel_dir"] == "/opt/wheels" + + with tempfile.TemporaryDirectory() as folder: + open(os.path.join(folder, "x-1.0-py3-none-any.whl"), "wb").close() + ensure_dependencies(wheel_dir=folder) + assert seen["wheel_dir"] == folder + + +def test_incomplete_explicit_wheel_dir_is_flagged(monkeypatch): + """An unusable --wheel-dir should be called out, not silently ignored.""" + seen = {} + monkeypatch.setattr( + bootstrap, "missing_packages", + lambda packages=None: [] if seen.get("done") else [("x", "x>=1")], + ) + + def fake_install(requirements, wheel_dir=None, logger=None, timeout=300): + seen["done"] = True + return True + + monkeypatch.setattr(bootstrap, "install", fake_install) + lines = [] + with tempfile.TemporaryDirectory() as folder: + ensure_dependencies(wheel_dir=folder, logger=lines.append) + assert any("you gave is incomplete" in line for line in lines) diff --git a/tests/test_wheels.py b/tests/test_wheels.py new file mode 100644 index 0000000..be3a439 --- /dev/null +++ b/tests/test_wheels.py @@ -0,0 +1,229 @@ +"""Tests for the vendored wheels and their verification. + +The bundled wheels let a device with no internet set itself up, so the things +that matter are: they are actually present and pure-Python (portable), the +checksums match, and a tampered wheel is refused rather than installed. +""" + +import os +import shutil +import sys + +SRC = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "src" +) +if SRC not in sys.path: + sys.path.insert(0, SRC) + +import pytest # noqa: E402 + +from rdm_card_reader import bootstrap # noqa: E402 +from rdm_card_reader.bootstrap import ( # noqa: E402 + bundled_wheel_dir, + ensure_dependencies, + local_wheel_missing, + verify_wheels, +) + + +# -- the wheels that ship with the package --------------------------------- + +def test_bundled_wheel_dir_exists(): + folder = bundled_wheel_dir() + assert folder is not None + assert os.path.isdir(folder) + + +def test_both_required_wheels_are_present(): + folder = bundled_wheel_dir() + names = os.listdir(folder) + assert any(n.startswith("rdm6300-") and n.endswith(".whl") for n in names) + assert any(n.startswith("pyserial-") and n.endswith(".whl") for n in names) + + +def test_bundled_wheels_are_pure_python(): + """A compiled wheel would only work on one architecture; these must not.""" + folder = bundled_wheel_dir() + wheels = [n for n in os.listdir(folder) if n.endswith(".whl")] + assert wheels + for name in wheels: + assert name.endswith("-none-any.whl"), ( + "{} is architecture-specific and cannot be shipped".format(name) + ) + + +def test_bundled_wheels_verify_against_their_checksums(): + folder = bundled_wheel_dir() + assert verify_wheels(folder) == [] + + +def test_checksums_file_is_shipped(): + folder = bundled_wheel_dir() + assert os.path.exists(os.path.join(folder, "SHA256SUMS")) + + +def test_total_vendored_size_is_reasonable(): + """Guard against someone dropping a huge binary into the repo.""" + folder = bundled_wheel_dir() + total = sum( + os.path.getsize(os.path.join(folder, name)) + for name in os.listdir(folder) + ) + assert total < 2 * 1024 * 1024 + + +# -- verification ----------------------------------------------------------- + +def test_tampered_wheel_is_detected(tmp_path): + folder = tmp_path / "wheels" + folder.mkdir() + wheel = folder / "rdm6300-0.1.1-py3-none-any.whl" + wheel.write_bytes(b"not the real wheel") + (folder / "SHA256SUMS").write_text( + "afb067da8b8d23a477138ca67d4ff1d198f12d157b64668c9dd3cbd7df54e74e" + " rdm6300-0.1.1-py3-none-any.whl\n" + ) + problems = verify_wheels(str(folder)) + assert problems + assert any("checksum mismatch" in p for p in problems) + + +def test_missing_wheel_is_reported(tmp_path): + folder = tmp_path / "wheels" + folder.mkdir() + (folder / "SHA256SUMS").write_text( + "abc123 rdm6300-0.1.1-py3-none-any.whl\n" + ) + problems = verify_wheels(str(folder)) + assert any("missing" in p for p in problems) + + +def test_no_sums_file_means_nothing_to_check(tmp_path): + folder = tmp_path / "wheels" + folder.mkdir() + (folder / "x-1.0-py3-none-any.whl").write_bytes(b"x") + assert verify_wheels(str(folder)) == [] + + +def test_verification_passes_for_untouched_copy(tmp_path): + source = bundled_wheel_dir() + dest = tmp_path / "wheels" + shutil.copytree(source, dest) + assert verify_wheels(str(dest)) == [] + + +# -- coverage check --------------------------------------------------------- + +def test_local_wheel_missing_detects_absent_requirement(tmp_path): + folder = tmp_path / "wheels" + folder.mkdir() + assert local_wheel_missing(["rdm6300>=0.1.1"], str(folder)) == ["rdm6300>=0.1.1"] + + +def test_local_wheel_missing_accepts_present_requirement(): + folder = bundled_wheel_dir() + assert local_wheel_missing(["rdm6300>=0.1.1", "pyserial>=3.5"], folder) == [] + + +def test_local_wheel_missing_normalises_underscores(tmp_path): + folder = tmp_path / "wheels" + folder.mkdir() + (folder / "charset_normalizer-3.4.1-py3-none-any.whl").write_bytes(b"x") + assert local_wheel_missing(["charset-normalizer>=3"], str(folder)) == [] + + +# -- integration into ensure_dependencies ----------------------------------- + +def test_bundled_wheels_used_automatically(monkeypatch): + """No --wheel-dir given, so the bundled ones must be picked up.""" + seen = {} + monkeypatch.setattr( + bootstrap, "missing_packages", + lambda packages=None: [] if seen.get("done") else [("rdm6300", "rdm6300>=0.1.1")], + ) + + def fake_install(requirements, wheel_dir=None, logger=None, timeout=300): + seen["wheel_dir"] = wheel_dir + seen["done"] = True + return True + + monkeypatch.setattr(bootstrap, "install", fake_install) + lines = [] + assert ensure_dependencies(logger=lines.append) is True + assert seen["wheel_dir"] == bundled_wheel_dir() + assert any("bundled" in line for line in lines) + + +def test_tampered_bundle_falls_back_to_pypi(monkeypatch, tmp_path): + """A bad checksum must not block installation - but must be reported.""" + folder = tmp_path / "wheels" + folder.mkdir() + (folder / "rdm6300-0.1.1-py3-none-any.whl").write_bytes(b"tampered") + (folder / "SHA256SUMS").write_text("deadbeef rdm6300-0.1.1-py3-none-any.whl\n") + + seen = {} + monkeypatch.setattr(bootstrap, "bundled_wheel_dir", lambda: str(folder)) + monkeypatch.setattr( + bootstrap, "missing_packages", + lambda packages=None: [] if seen.get("done") else [("rdm6300", "rdm6300>=0.1.1")], + ) + + def fake_install(requirements, wheel_dir=None, logger=None, timeout=300): + seen["wheel_dir"] = wheel_dir + seen["done"] = True + return True + + monkeypatch.setattr(bootstrap, "install", fake_install) + lines = [] + assert ensure_dependencies(logger=lines.append) is True + assert seen["wheel_dir"] is None # fell back to PyPI + assert any("checksum mismatch" in line for line in lines) + + +def test_explicit_wheel_dir_overrides_bundled(monkeypatch, tmp_path): + folder = tmp_path / "wheels" + folder.mkdir() + (folder / "rdm6300-0.1.1-py3-none-any.whl").write_bytes(b"x") + (folder / "pyserial-3.5-py2.py3-none-any.whl").write_bytes(b"x") + + seen = {} + monkeypatch.setattr( + bootstrap, "missing_packages", + lambda packages=None: [] if seen.get("done") else [("rdm6300", "rdm6300>=0.1.1")], + ) + + def fake_install(requirements, wheel_dir=None, logger=None, timeout=300): + seen["wheel_dir"] = wheel_dir + seen["done"] = True + return True + + monkeypatch.setattr(bootstrap, "install", fake_install) + ensure_dependencies(wheel_dir=str(folder)) + assert seen["wheel_dir"] == str(folder) + + +def test_bundle_not_covering_a_requirement_falls_back(monkeypatch, tmp_path): + """A bundle missing one package must not cause a guaranteed pip failure.""" + folder = tmp_path / "wheels" + folder.mkdir() + (folder / "rdm6300-0.1.1-py3-none-any.whl").write_bytes(b"x") + + seen = {} + monkeypatch.setattr(bootstrap, "bundled_wheel_dir", lambda: str(folder)) + monkeypatch.setattr( + bootstrap, "missing_packages", + lambda packages=None: [] if seen.get("done") + else [("rdm6300", "rdm6300>=0.1.1"), ("serial", "pyserial>=3.5")], + ) + + def fake_install(requirements, wheel_dir=None, logger=None, timeout=300): + seen["wheel_dir"] = wheel_dir + seen["wheel_dir_at_call"] = wheel_dir + seen["done"] = True + return True + + monkeypatch.setattr(bootstrap, "install", fake_install) + lines = [] + ensure_dependencies(logger=lines.append) + assert seen["wheel_dir"] is None + assert any("do not cover" in line for line in lines)