From d4b3d7a7217336fc4b85a81ffefadc422d74c7a5 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 18:03:23 +0200 Subject: [PATCH 01/20] webdav: serve archive contents read-only via HTTP, #9942 New "borg webdav" command: it starts a read-only HTTP server on localhost, so archive contents can be browsed and files downloaded with a web browser. This first step implements plain HTTP only (GET/HEAD, HTML directory listings, file downloads). The WebDAV methods (OPTIONS/PROPFIND), which enable native mounting by Windows Explorer, macOS Finder and the Linux desktop file managers, are planned to be added on top of this later. Implementation notes: - only uses the python standard library, no new dependencies. - the directory tree of an archive is built in memory when it is first accessed, so a big archive only costs time/memory if it is browsed. - all repository access is serialized with a lock, because borgstore connections are not thread-safe. - a LockRefresher keeps the repository lock of an idle server alive, so it is not killed as stale, see #9872. - missing chunks abort the download connection, so damaged files are never served as silently corrupted content. - symlinks and special files are listed, but not downloadable. Note: downloads via HTTP do not preserve POSIX metadata (owner, group, mode, timestamps, xattrs, ACLs) - use borg extract or borg export-tar for full-fidelity restores. Co-Authored-By: Claude Fable 5 --- docs/man/borg-webdav.1 | 123 ++++++ docs/usage.rst | 1 + docs/usage/webdav.rst | 13 + docs/usage/webdav.rst.inc | 100 +++++ src/borg/archiver/__init__.py | 3 + src/borg/archiver/webdav_cmd.py | 85 ++++ .../testsuite/archiver/webdav_cmd_test.py | 141 ++++++ src/borg/webdav.py | 400 ++++++++++++++++++ 8 files changed, 866 insertions(+) create mode 100644 docs/man/borg-webdav.1 create mode 100644 docs/usage/webdav.rst create mode 100644 docs/usage/webdav.rst.inc create mode 100644 src/borg/archiver/webdav_cmd.py create mode 100644 src/borg/testsuite/archiver/webdav_cmd_test.py create mode 100644 src/borg/webdav.py diff --git a/docs/man/borg-webdav.1 b/docs/man/borg-webdav.1 new file mode 100644 index 0000000000..30a720f8cc --- /dev/null +++ b/docs/man/borg-webdav.1 @@ -0,0 +1,123 @@ +.\" Man page generated from reStructuredText +.\" by the Docutils 0.22.4 manpage writer. +. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.TH "borg-webdav" "1" "2026-07-23" "" "borg backup tool" +.SH Name +borg-webdav \- Serve archive contents via a read-only HTTP server on localhost. +.SH SYNOPSIS +.sp +borg [common options] webdav [options] +.SH DESCRIPTION +.sp +This command serves the contents of the selected archives via a read\-only +HTTP server, so files can be browsed and downloaded with a web browser. +.sp +The server listens on localhost (127.0.0.1) only and offers no +authentication and no encryption \- anything that can connect to +localhost TCP ports on the machine can read the served archive contents. +.sp +The top level lists the selected archives (use the archive filter options +to select fewer archives); below that, the archive contents can be +browsed like a directory tree. The directory tree of an archive is built +in memory when it is first entered, so expect some delay for big archives. +.sp +Notes: +.INDENT 0.0 +.IP \(bu 2 +Downloads via HTTP do not preserve any POSIX metadata (owner, group, +mode, timestamps, xattrs, ACLs). Use \fBborg extract\fP or +\fBborg export\-tar\fP for full\-fidelity restores. +.IP \(bu 2 +Symbolic links are shown in the directory listings, but are neither +followed nor downloadable. Special files (devices, fifos, sockets) +are shown, but not downloadable. +.IP \(bu 2 +Damaged files (with chunks missing in the repository) cause the +download connection to be aborted \- the server never silently serves +corrupted file content. +.UNINDENT +.sp +The command runs in the foreground until it is interrupted with Ctrl\-C. +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SS options +.INDENT 0.0 +.TP +.BI \-\-port \ PORT +TCP port to listen on (on localhost); default: 8000 +.UNINDENT +.SS Archive filters +.INDENT 0.0 +.TP +.BI \-a \ PATTERN\fR,\fB \ \-\-match\-archives \ PATTERN +only consider archives matching all patterns. See \(dqborg help match\-archives\(dq. +.TP +.BI \-\-sort\-by \ KEYS +Comma\-separated list of sorting keys; valid keys are: timestamp, archive, name, id, tags, host, user; default is: timestamp +.TP +.BI \-\-first \ N +consider the first N archives after other filters are applied +.TP +.BI \-\-last \ N +consider the last N archives after other filters are applied +.TP +.BI \-\-oldest \ TIMESPAN +consider archives between the oldest archive\(aqs timestamp and (oldest + TIMESPAN), e.g., 7d or 12m. +.TP +.BI \-\-newest \ TIMESPAN +consider archives between the newest archive\(aqs timestamp and (newest \- TIMESPAN), e.g., 7d or 12m. +.TP +.BI \-\-older \ TIMESPAN +consider archives older than (now \- TIMESPAN), e.g., 7d or 12m. +.TP +.BI \-\-newer \ TIMESPAN +consider archives newer than (now \- TIMESPAN), e.g., 7d or 12m. +.UNINDENT +.SH EXAMPLES +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +# Serve all archives of the repository on http://127.0.0.1:8000/, +# then browse and download files with a web browser. +$ borg webdav + +# Serve only one archive on a different port. +$ borg webdav \-\-port 8123 \-\-match\-archives my\-archive +.EE +.UNINDENT +.UNINDENT +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP +.SH Author +The Borg Collective +.\" End of generated man page. diff --git a/docs/usage.rst b/docs/usage.rst index 6921d0bc7c..c62faad033 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -60,6 +60,7 @@ Usage usage/info usage/analyze usage/mount + usage/webdav usage/recreate usage/tar diff --git a/docs/usage/webdav.rst b/docs/usage/webdav.rst new file mode 100644 index 0000000000..07a8d845d8 --- /dev/null +++ b/docs/usage/webdav.rst @@ -0,0 +1,13 @@ +.. include:: webdav.rst.inc + +Examples +~~~~~~~~ + +:: + + # Serve all archives of the repository on http://127.0.0.1:8000/, + # then browse and download files with a web browser. + $ borg webdav + + # Serve only one archive on a different port. + $ borg webdav --port 8123 --match-archives my-archive diff --git a/docs/usage/webdav.rst.inc b/docs/usage/webdav.rst.inc new file mode 100644 index 0000000000..8527d26798 --- /dev/null +++ b/docs/usage/webdav.rst.inc @@ -0,0 +1,100 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_webdav: + +borg webdav +----------- +.. code-block:: none + + borg [common options] webdav [options] + +.. only:: html + + .. class:: borg-options-table + + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | **options** | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | | ``--port PORT`` | TCP port to listen on (on localhost); default: 8000 | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | .. class:: borg-common-opt-ref | + | | + | :ref:`common_options` | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | **Archive filters** — Archive filters can be applied to repository targets. | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | | ``-a PATTERN``, ``--match-archives PATTERN`` | only consider archives matching all patterns. See "borg help match-archives". | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | | ``--sort-by KEYS`` | Comma-separated list of sorting keys; valid keys are: timestamp, archive, name, id, tags, host, user; default is: timestamp | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | | ``--first N`` | consider the first N archives after other filters are applied | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | | ``--last N`` | consider the last N archives after other filters are applied | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | | ``--oldest TIMESPAN`` | consider archives between the oldest archive's timestamp and (oldest + TIMESPAN), e.g., 7d or 12m. | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | | ``--newest TIMESPAN`` | consider archives between the newest archive's timestamp and (newest - TIMESPAN), e.g., 7d or 12m. | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | | ``--older TIMESPAN`` | consider archives older than (now - TIMESPAN), e.g., 7d or 12m. | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | | ``--newer TIMESPAN`` | consider archives newer than (now - TIMESPAN), e.g., 7d or 12m. | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + + .. raw:: html + + + +.. only:: latex + + + + options + --port PORT TCP port to listen on (on localhost); default: 8000 + + + :ref:`common_options` + | + + Archive filters + -a PATTERN, --match-archives PATTERN only consider archives matching all patterns. See "borg help match-archives". + --sort-by KEYS Comma-separated list of sorting keys; valid keys are: timestamp, archive, name, id, tags, host, user; default is: timestamp + --first N consider the first N archives after other filters are applied + --last N consider the last N archives after other filters are applied + --oldest TIMESPAN consider archives between the oldest archive's timestamp and (oldest + TIMESPAN), e.g., 7d or 12m. + --newest TIMESPAN consider archives between the newest archive's timestamp and (newest - TIMESPAN), e.g., 7d or 12m. + --older TIMESPAN consider archives older than (now - TIMESPAN), e.g., 7d or 12m. + --newer TIMESPAN consider archives newer than (now - TIMESPAN), e.g., 7d or 12m. + + +Description +~~~~~~~~~~~ + +This command serves the contents of the selected archives via a read-only +HTTP server, so files can be browsed and downloaded with a web browser. + +The server listens on localhost (127.0.0.1) only and offers no +authentication and no encryption - anything that can connect to +localhost TCP ports on the machine can read the served archive contents. + +The top level lists the selected archives (use the archive filter options +to select fewer archives); below that, the archive contents can be +browsed like a directory tree. The directory tree of an archive is built +in memory when it is first entered, so expect some delay for big archives. + +Notes: + +- Downloads via HTTP do not preserve any POSIX metadata (owner, group, + mode, timestamps, xattrs, ACLs). Use ``borg extract`` or + ``borg export-tar`` for full-fidelity restores. +- Symbolic links are shown in the directory listings, but are neither + followed nor downloadable. Special files (devices, fifos, sockets) + are shown, but not downloadable. +- Damaged files (with chunks missing in the repository) cause the + download connection to be aborted - the server never silently serves + corrupted file content. + +The command runs in the foreground until it is interrupted with Ctrl-C. \ No newline at end of file diff --git a/src/borg/archiver/__init__.py b/src/borg/archiver/__init__.py index 9c62259a71..751ce0bdeb 100644 --- a/src/borg/archiver/__init__.py +++ b/src/borg/archiver/__init__.py @@ -94,6 +94,7 @@ from .transfer_cmd import TransferMixIn from .undelete_cmd import UnDeleteMixIn from .version_cmd import VersionMixIn +from .webdav_cmd import WebDAVMixIn class Archiver( @@ -127,6 +128,7 @@ class Archiver( TransferMixIn, UnDeleteMixIn, VersionMixIn, + WebDAVMixIn, ): def __init__(self, lock_wait=None, prog=None): self.lock_wait = lock_wait @@ -316,6 +318,7 @@ def build_parser(self): self.build_parser_transfer(subparsers, common_parser, mid_common_parser) self.build_parser_undelete(subparsers, common_parser, mid_common_parser) self.build_parser_version(subparsers, common_parser, mid_common_parser) + self.build_parser_webdav(subparsers, common_parser, mid_common_parser) return parser def get_args(self, argv, cmd): diff --git a/src/borg/archiver/webdav_cmd.py b/src/borg/archiver/webdav_cmd.py new file mode 100644 index 0000000000..8d337112fc --- /dev/null +++ b/src/borg/archiver/webdav_cmd.py @@ -0,0 +1,85 @@ +import threading +import time + +from ._common import with_repository, Highlander +from ..constants import * # NOQA +from ..helpers import sig_int +from ..helpers.argparsing import ArgumentParser +from ..manifest import Manifest + +from ..logger import create_logger + +logger = create_logger() + + +class WebDAVMixIn: + @with_repository(compatibility=(Manifest.Operation.READ,)) + def do_webdav(self, args, repository, manifest): + """Serve archive contents via a read-only HTTP server on localhost.""" + from ..webdav import make_server + from ..storelocking import LockRefresher + + server = make_server(manifest, args, port=args.port) + host, port = server.server_address[:2] + print(f"Serving selected archives read-only on http://{host}:{port}/ - press Ctrl-C to stop.") + # keep the repository lock of an idle server alive, so it is not killed as stale (see #9872). + lock_refreshing_thread = LockRefresher(manifest.repository.info, sleep_interval=60, lock=server.repo_lock) + lock_refreshing_thread.start() + # the first SIGINT only sets the sig_int flag (see SigIntManager), so serve in a + # thread and poll the flag here, to shut down cleanly on the first Ctrl-C already. + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + try: + while not sig_int: + time.sleep(0.1) + print("Shutting down...") + finally: + lock_refreshing_thread.terminate() + server.shutdown() + server.server_close() + server_thread.join(timeout=10) + + def build_parser_webdav(self, subparsers, common_parser, mid_common_parser): + from ._common import process_epilog, define_archive_filters_group + + webdav_epilog = process_epilog( + """ + This command serves the contents of the selected archives via a read-only + HTTP server, so files can be browsed and downloaded with a web browser. + + The server listens on localhost (127.0.0.1) only and offers no + authentication and no encryption - anything that can connect to + localhost TCP ports on the machine can read the served archive contents. + + The top level lists the selected archives (use the archive filter options + to select fewer archives); below that, the archive contents can be + browsed like a directory tree. The directory tree of an archive is built + in memory when it is first entered, so expect some delay for big archives. + + Notes: + + - Downloads via HTTP do not preserve any POSIX metadata (owner, group, + mode, timestamps, xattrs, ACLs). Use ``borg extract`` or + ``borg export-tar`` for full-fidelity restores. + - Symbolic links are shown in the directory listings, but are neither + followed nor downloadable. Special files (devices, fifos, sockets) + are shown, but not downloadable. + - Damaged files (with chunks missing in the repository) cause the + download connection to be aborted - the server never silently serves + corrupted file content. + + The command runs in the foreground until it is interrupted with Ctrl-C. + """ + ) + subparser = ArgumentParser(parents=[common_parser], description=self.do_webdav.__doc__, epilog=webdav_epilog) + subparsers.add_subcommand("webdav", subparser, help="serve archive contents via HTTP") + subparser.add_argument( + "--port", + metavar="PORT", + dest="port", + type=int, + default=8000, + action=Highlander, + help="TCP port to listen on (on localhost); default: %(default)s", + ) + define_archive_filters_group(subparser) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py new file mode 100644 index 0000000000..5dd1442aab --- /dev/null +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -0,0 +1,141 @@ +# This file tests the webdav command (read-only HTTP serving of archive contents). +# The server is started in-process (no borg CLI invocation for the serving part), +# so these tests run without any optional dependency. + +import os +import threading +import urllib.error +import urllib.request +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest + +from ...constants import * # NOQA +from ...manifest import Manifest +from ...repository import Repository +from ...webdav import make_server +from .. import are_symlinks_supported, are_hardlinks_supported +from . import RK_ENCRYPTION, cmd, create_regular_file, generate_archiver_tests + +pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local") # NOQA + +FUNNY_NAME = "a &c ä.txt" + + +def _create_archive(archiver): + create_regular_file(archiver.input_path, "file1", contents=b"data1") + create_regular_file(archiver.input_path, "subdir/file2", contents=b"subdir data") + create_regular_file(archiver.input_path, FUNNY_NAME, contents=b"funny data") + big = os.urandom(5 * 1024 * 1024) # big enough for multiple chunks with default chunker params + create_regular_file(archiver.input_path, "big", contents=big) + if are_symlinks_supported(): + os.symlink("somewhere/else", os.path.join(archiver.input_path, "link1")) + if are_hardlinks_supported(): + os.link(os.path.join(archiver.input_path, "file1"), os.path.join(archiver.input_path, "hardlink")) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test", "input") + return big + + +@contextmanager +def webdav_server(archiver): + """Start the webdav server in-process on an ephemeral localhost port.""" + args = SimpleNamespace( + sort_by="ts", match_archives=None, first=None, last=None, older=None, newer=None, oldest=None, newest=None + ) + repository = Repository(archiver.repository_path, exclusive=True) + with repository: + manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + server = make_server(manifest, args, port=0) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=10) + + +def get(url): + with urllib.request.urlopen(url) as response: + return response.status, dict(response.headers), response.read() + + +def test_webdav_browse(archivers, request): + archiver = request.getfixturevalue(archivers) + _create_archive(archiver) + with webdav_server(archiver) as base_url: + # root: archive list + status, headers, body = get(base_url + "/") + assert status == 200 + assert headers["Content-Type"] == "text/html; charset=utf-8" + page = body.decode("utf-8") + assert 'test/' in page + # missing trailing slash on a directory redirects (followed by urllib) + status, _, body = get(base_url + "/test/input") + assert status == 200 + page = body.decode("utf-8") + assert 'file1' in page + assert 'subdir/' in page + # precise sizes in bytes, with dots as thousands separators + assert ">5.242.880<" in page + # funny file name is html-escaped in text and percent-encoded in the link + assert "a <b>&c ä.txt" in page + assert 'href="a%20%3Cb%3E%26c%20%C3%A4.txt"' in page + if are_symlinks_supported(): + assert "link1 -> somewhere/else" in page + assert 'href="link1"' not in page # symlinks are not downloadable + + +def test_webdav_download(archivers, request): + archiver = request.getfixturevalue(archivers) + big = _create_archive(archiver) + with webdav_server(archiver) as base_url: + status, headers, body = get(base_url + "/test/input/file1") + assert status == 200 + assert body == b"data1" + assert headers["Content-Length"] == "5" + assert "GMT" in headers["Last-Modified"] + assert headers["Content-Type"] == "application/octet-stream" # no file extension + # multi-chunk file round-trips intact + status, headers, body = get(base_url + "/test/input/big") + assert status == 200 + assert body == big + assert headers["Content-Length"] == str(len(big)) + # a file in a subdirectory, with a percent-encoded url + status, headers, body = get(base_url + "/test/input/a%20%3Cb%3E%26c%20%C3%A4.txt") + assert status == 200 + assert body == b"funny data" + assert headers["Content-Type"].startswith("text/plain") + + +@pytest.mark.skipif(not are_hardlinks_supported(), reason="hardlinks not supported") +def test_webdav_hardlinks(archivers, request): + archiver = request.getfixturevalue(archivers) + _create_archive(archiver) + with webdav_server(archiver) as base_url: + # each hardlinked regular file item has its own chunks list, so both must serve content + for name in "file1", "hardlink": + status, _, body = get(f"{base_url}/test/input/{name}") + assert status == 200 + assert body == b"data1" + + +def test_webdav_errors(archivers, request): + archiver = request.getfixturevalue(archivers) + _create_archive(archiver) + with webdav_server(archiver) as base_url: + with pytest.raises(urllib.error.HTTPError) as exc_info: + get(base_url + "/no-such-archive/") + assert exc_info.value.code == 404 + with pytest.raises(urllib.error.HTTPError) as exc_info: + get(base_url + "/test/input/no-such-file") + assert exc_info.value.code == 404 + with pytest.raises(urllib.error.HTTPError) as exc_info: + get(base_url + "/test/input/../input/file1") + assert exc_info.value.code == 404 + with pytest.raises(urllib.error.HTTPError) as exc_info: + urllib.request.urlopen(urllib.request.Request(base_url + "/", method="POST")) + assert exc_info.value.code == 405 diff --git a/src/borg/webdav.py b/src/borg/webdav.py new file mode 100644 index 0000000000..c86f06e2d8 --- /dev/null +++ b/src/borg/webdav.py @@ -0,0 +1,400 @@ +"""Read-only HTTP server providing browser access to archive contents (``borg webdav``). + +Phase 1 implements plain HTTP only (GET/HEAD, HTML directory listings, file +downloads), bound to localhost. WebDAV methods (OPTIONS/PROPFIND) for native +mounting by Explorer/Finder/GVFS are planned to be added on top of this later. + +This module only uses the Python standard library, so it works without any +optional dependencies. +""" + +import html +import mimetypes +import stat +import threading +from datetime import datetime, timezone +from email.utils import formatdate +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import quote, unquote_to_bytes + +from . import __version__ +from .archive import Archive +from .constants import * # NOQA +from .helpers import bin_to_hex, remove_surrogates +from .logger import create_logger + +logger = create_logger(__name__) + +DEFAULT_DIR_MODE = 0o40755 + + +class Node: + """A node in an archive's directory tree: a directory (children is a dict) or a leaf.""" + + __slots__ = ("mode", "mtime", "size", "chunks", "target", "children") + + def __init__(self, mode, mtime=0, size=0, chunks=None, target=None, children=None): + self.mode = mode + self.mtime = mtime # ns + self.size = size + self.chunks = chunks + self.target = target # symlink target + self.children = children # {name(str): Node} for directories, None otherwise + + @property + def is_dir(self): + return self.children is not None + + +class ArchiveVFS: + """Read-only view of the archives selected by the archive filter args. + + The top level maps (deduplicated) archive names to lazily built directory trees. + All repository access happens under repo_lock, because borgstore connections are + not thread-safe. + """ + + def __init__(self, manifest, args, repo_lock): + self.manifest = manifest + self.repo_lock = repo_lock + archives = manifest.archives.list_considering(args) + # deduplicate archive names (archives of a series all share the same name) + name_counter = {} + for archive in archives: + name_counter[archive.name] = name_counter.get(archive.name, 0) + 1 + self.archives = {} # display name -> ArchiveInfo + for archive in archives: + name = archive.name + if name_counter[name] > 1: + name += f"-{bin_to_hex(archive.id):.8}" + self.archives[name] = archive + self._trees = {} # display name -> (root Node, DownloadPipeline) + + def get_root(self, name): + """Return (root Node, pipeline) for archive *name*, building the tree on first access.""" + try: + return self._trees[name] + except KeyError: + pass + archive_info = self.archives[name] # may raise KeyError -> 404 + with self.repo_lock: + if name not in self._trees: # re-check under lock + self._trees[name] = self._build_tree(archive_info) + return self._trees[name] + + def _build_tree(self, archive_info): + logger.debug("webdav: building tree for archive %s ...", remove_surrogates(archive_info.name)) + archive = Archive(self.manifest, archive_info.id) + archive_mtime = int(archive_info.ts.timestamp() * 1e9) + root = Node(DEFAULT_DIR_MODE, mtime=archive_mtime, children={}) + for item in archive.iter_items(): + segments = [s for s in item.path.split("/") if s] + if not segments: + continue + node = root + for segment in segments[:-1]: + child = node.children.get(segment) + if child is None or not child.is_dir: + # intermediate directory not (yet) seen as an item: synthesize it + child = Node(DEFAULT_DIR_MODE, mtime=archive_mtime, children={}) + node.children[segment] = child + node = child + name = segments[-1] + if stat.S_ISDIR(item.mode): + existing = node.children.get(name) + if existing is not None and existing.is_dir: + # was synthesized before the dir item was seen: update metadata in place + existing.mode = item.mode + existing.mtime = item.mtime + else: + node.children[name] = Node(item.mode, mtime=item.mtime, children={}) + else: + node.children[name] = Node( + item.mode, + mtime=item.mtime, + size=item.get_size(), + chunks=item.get("chunks"), + target=item.get("target"), + ) + return root, archive.pipeline + + def resolve(self, segments): + """Resolve path segments (first one is the archive name) to (Node, pipeline). + + Raises KeyError if not found. + """ + root, pipeline = self.get_root(segments[0]) + node = root + for segment in segments[1:]: + if not node.is_dir: + raise KeyError(segment) + node = node.children[segment] + return node, pipeline + + +def encode_path(path): + """Percent-encode a borg item path (str with surrogateescape) for use in a URL.""" + return quote(path.encode("utf-8", "surrogateescape")) + + +def decode_path(path): + """Decode a percent-encoded URL path to a borg item path (str with surrogateescape).""" + return unquote_to_bytes(path).decode("utf-8", "surrogateescape") + + +def http_date(mtime_ns): + return formatdate(mtime_ns / 1e9, usegmt=True) + + +def display_time(mtime_ns): + dt = datetime.fromtimestamp(mtime_ns / 1e9, tz=timezone.utc).astimezone() + return dt.isoformat(sep=" ", timespec="seconds") + + +def display_size(size): + """Format a precise byte count with dots as thousands separators, e.g. '123.456.789'.""" + return f"{size:,}".replace(",", ".") + + +# the official borg logo (docs/_static/logo.svg, "borg" in vectorized Black Ops One), +# without the background rect and with the fill color controlled via CSS currentColor. +LOGO_SVG = ( + '' + '' + '' + '' + '' + "" +) + +PAGE_TEMPLATE = """\ + + + + +{title} + + + +
+

{title}

+ +
+ + +{rows} +
NameSizeModified
+ + +""" + + +def render_page(title, rows): + page = PAGE_TEMPLATE.format(title=html.escape(title), rows="\n".join(rows), logo=LOGO_SVG) + return page.encode("utf-8") + + +def make_row(href, text, size="", mtime_ns=None): + modified = display_time(mtime_ns) if mtime_ns is not None else "" + if href is not None: + name_cell = f'{html.escape(text)}' + else: + name_cell = html.escape(text) + return f'{name_cell}{size}{modified}' + + +class WebDAVHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server_version = f"borg-webdav/{__version__}" + sys_version = "" # do not tell clients about the python version we use + + # set on the handler class by make_server(): + vfs = None + repo_lock = None + + def version_string(self): + # the base class would append sys_version, giving a trailing space if it is empty. + return self.server_version + + def log_message(self, format, *args): + logger.debug("webdav: %s - %s", self.address_string(), format % args) + + def do_GET(self): + self._handle(head=False) + + def do_HEAD(self): + self._handle(head=True) + + def _method_not_allowed(self): + self.send_response(405) + self.send_header("Allow", "GET, HEAD") + self.send_header("Content-Length", "0") + self.end_headers() + + do_POST = do_PUT = do_DELETE = do_OPTIONS = do_PATCH = _method_not_allowed + + def _handle(self, head): + try: + path, _, _query = self.path.partition("?") + path = decode_path(path) + segments = [s for s in path.split("/") if s] + if any(s in (".", "..") for s in segments): + self.send_error(404) + return + if not segments: + self._send_archive_list(head) + return + try: + node, pipeline = self.vfs.resolve(segments) + except KeyError: + self.send_error(404) + return + if node.is_dir: + if not path.endswith("/"): + self._redirect_to_dir() + return + self._send_dir_listing(segments, node, head) + elif stat.S_ISREG(node.mode): + self._send_file(segments[-1], node, pipeline, head) + elif stat.S_ISLNK(node.mode): + self.send_error( + 403, explain=f"symbolic link (target: {remove_surrogates(node.target or '?')}), not downloadable" + ) + else: + self.send_error(403, explain="special file, not downloadable") + except BrokenPipeError: + self.close_connection = True + except Exception: + logger.exception("webdav: unhandled error while serving %s", self.requestline) + try: + self.send_error(500) + except OSError: + self.close_connection = True + + def _redirect_to_dir(self): + path, _, _query = self.path.partition("?") + self.send_response(301) + self.send_header("Location", path + "/") + self.send_header("Content-Length", "0") + self.end_headers() + + def _send_page(self, page, head): + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(page))) + self.send_header("X-Content-Type-Options", "nosniff") + self.end_headers() + if not head: + self.wfile.write(page) + + def _send_archive_list(self, head): + rows = [] + for name in sorted(self.vfs.archives): + archive_info = self.vfs.archives[name] + mtime_ns = int(archive_info.ts.timestamp() * 1e9) + rows.append(make_row(encode_path(name) + "/", name + "/", mtime_ns=mtime_ns)) + self._send_page(render_page("Archives", rows), head) + + def _send_dir_listing(self, segments, node, head): + title = "/".join(remove_surrogates(s) for s in segments) + "/" + rows = [make_row("../", "..")] + children = sorted(node.children.items(), key=lambda kv: (not kv[1].is_dir, kv[0])) + for name, child in children: + display_name = remove_surrogates(name) + if child.is_dir: + rows.append(make_row(encode_path(name) + "/", display_name + "/", mtime_ns=child.mtime)) + elif stat.S_ISREG(child.mode): + rows.append( + make_row(encode_path(name), display_name, size=display_size(child.size), mtime_ns=child.mtime) + ) + elif stat.S_ISLNK(child.mode): + text = f"{display_name} -> {remove_surrogates(child.target or '?')}" + rows.append(make_row(None, text, mtime_ns=child.mtime)) + else: + rows.append(make_row(None, display_name, mtime_ns=child.mtime)) + self._send_page(render_page(title, rows), head) + + def _send_file(self, name, node, pipeline, head): + self.send_response(200) + ctype = mimetypes.guess_type(remove_surrogates(name), strict=False)[0] or "application/octet-stream" + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(node.size)) + self.send_header("Last-Modified", http_date(node.mtime)) + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("Content-Disposition", self._content_disposition(name)) + self.end_headers() + if head or not node.chunks: + return + chunk_iter = pipeline.fetch_many(node.chunks, ro_type=ROBJ_FILE_STREAM, replacement_chunk=False) + while True: + # serialize repository access, but write to the client outside the lock, + # so one slow client cannot block other requests for the whole download. + with self.repo_lock: + try: + data = next(chunk_iter) + except StopIteration: + break + if data is None: + # chunk missing in repository - never serve silently corrupted data: + # abort the connection, the client sees a short read (Content-Length mismatch). + logger.error( + "webdav: chunk missing while serving %s, aborting the connection.", remove_surrogates(name) + ) + self.close_connection = True + return + self.wfile.write(data) + + @staticmethod + def _content_disposition(name): + fallback = remove_surrogates(name).encode("ascii", "replace").decode("ascii").replace('"', "'") + encoded = quote(name.encode("utf-8", "surrogateescape")) + return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{encoded}" + + +def make_server(manifest, args, bind="127.0.0.1", port=8000): + """Create a ThreadingHTTPServer serving the archives selected by *args*. + + The server object gets a repo_lock attribute; all repository access of the + server threads is serialized with it (use it for e.g. LockRefresher, too). + """ + repo_lock = threading.RLock() + vfs = ArchiveVFS(manifest, args, repo_lock) + + handler_class = type("WebDAVHandler", (WebDAVHandler,), dict(vfs=vfs, repo_lock=repo_lock)) + server = ThreadingHTTPServer((bind, port), handler_class) + server.daemon_threads = True + server.repo_lock = repo_lock + return server From e7047c42be397a7f7060c1e50bb2bdd2ab4939da Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 18:24:44 +0200 Subject: [PATCH 02/20] webdav: implement WebDAV (class 1), so archives can be mounted, #9942 The served archives can now be mounted as a read-only network file system using the WebDAV client built into most operating systems and file managers: - Windows Explorer: "Map network drive" -> http://localhost:8000/ - macOS Finder: "Go > Connect to Server" -> http://localhost:8000 - GNOME Files / KDE Dolphin: dav://localhost:8000/ - Linux kernel mount: mount -t davfs (davfs2 package) Implemented (still python stdlib only): - OPTIONS advertising class 1 DAV support. - PROPFIND with Depth 0/1 (infinity is refused, as RFC 4918 permits), supporting allprop / propname / named-prop request bodies, answering with 207 multistatus XML. Request body size is limited. - live properties: resourcetype, displayname, getcontentlength, getcontenttype, getlastmodified, creationdate, getetag, and empty supportedlock/lockdiscovery (no locking - read-only server). - Range requests (single range): only the chunks overlapping the requested range are fetched and decrypted (chunk sizes are known in advance), so the many small ranged reads DAV file systems do for big files stay efficient. 206/416 handling, If-Range. - conditional requests: ETag (archive contents are immutable, so mtime+size is a solid validator), If-None-Match -> 304. - all writing/locking methods (PUT, DELETE, PROPPATCH, MKCOL, COPY, MOVE, LOCK, UNLOCK, ...) are rejected with 405 + Allow, and clients see a read-only file system. - symbolic links and special files are not exposed via WebDAV (the protocol has no concept of them); they remain visible in the web browser listings. Verified against real clients: rclone (recursive PROPFIND listing and reads) and the macOS WebDAV file system (mount_webdav): mount, ls, byte- identical reads of multi-chunk files, write attempts correctly rejected with EROFS. Co-Authored-By: Claude Fable 5 --- docs/man/borg-webdav.1 | 42 +- docs/usage/webdav.rst.inc | 31 +- src/borg/archiver/webdav_cmd.py | 35 +- .../testsuite/archiver/webdav_cmd_test.py | 157 ++++++- src/borg/webdav.py | 386 ++++++++++++++++-- 5 files changed, 582 insertions(+), 69 deletions(-) diff --git a/docs/man/borg-webdav.1 b/docs/man/borg-webdav.1 index 30a720f8cc..7bf77d2848 100644 --- a/docs/man/borg-webdav.1 +++ b/docs/man/borg-webdav.1 @@ -30,14 +30,35 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .TH "borg-webdav" "1" "2026-07-23" "" "borg backup tool" .SH Name -borg-webdav \- Serve archive contents via a read-only HTTP server on localhost. +borg-webdav \- Serve archive contents via a read-only WebDAV / HTTP server on localhost. .SH SYNOPSIS .sp borg [common options] webdav [options] .SH DESCRIPTION .sp This command serves the contents of the selected archives via a read\-only -HTTP server, so files can be browsed and downloaded with a web browser. +WebDAV / HTTP server, so the archive contents can be: +.INDENT 0.0 +.IP \(bu 2 +browsed and downloaded with a web browser, +.IP \(bu 2 +mounted as a read\-only network file system, using the WebDAV client +built into most operating systems and file managers. +.UNINDENT +.sp +Mounting examples: +.INDENT 0.0 +.IP \(bu 2 +Windows Explorer: \(dqMap network drive\(dq \-> \fBhttp://localhost:8000/\fP +(or on the command line: \fBnet use Z: http://localhost:8000/\fP). +.IP \(bu 2 +macOS Finder: \(dqGo > Connect to Server\(dq (Cmd\-K) \-> \fBhttp://localhost:8000\fP\&. +.IP \(bu 2 +GNOME Files / KDE Dolphin: open \fBdav://localhost:8000/\fP\&. +.IP \(bu 2 +Linux kernel mount: \fBmount \-t davfs http://localhost:8000/ /mnt/point\fP +(needs the davfs2 package). +.UNINDENT .sp The server listens on localhost (127.0.0.1) only and offers no authentication and no encryption \- anything that can connect to @@ -51,17 +72,22 @@ in memory when it is first entered, so expect some delay for big archives. Notes: .INDENT 0.0 .IP \(bu 2 -Downloads via HTTP do not preserve any POSIX metadata (owner, group, -mode, timestamps, xattrs, ACLs). Use \fBborg extract\fP or -\fBborg export\-tar\fP for full\-fidelity restores. +Downloads do not preserve any POSIX metadata (owner, group, mode, +timestamps, xattrs, ACLs). Use \fBborg extract\fP or \fBborg export\-tar\fP +for full\-fidelity restores. .IP \(bu 2 -Symbolic links are shown in the directory listings, but are neither -followed nor downloadable. Special files (devices, fifos, sockets) -are shown, but not downloadable. +Symbolic links and special files (devices, fifos, sockets) are shown +in the web browser listings, but are neither followed nor downloadable, +and they are not visible in WebDAV\-mounted directories (WebDAV has no +concept of them). .IP \(bu 2 Damaged files (with chunks missing in the repository) cause the download connection to be aborted \- the server never silently serves corrupted file content. +.IP \(bu 2 +The Windows WebDAV client limits file downloads to about 47 MiB by +default (\fBFileSizeLimitInBytes\fP registry value) \- use a web browser +or another WebDAV client to download bigger files. .UNINDENT .sp The command runs in the foreground until it is interrupted with Ctrl\-C. diff --git a/docs/usage/webdav.rst.inc b/docs/usage/webdav.rst.inc index 8527d26798..9cca04cc09 100644 --- a/docs/usage/webdav.rst.inc +++ b/docs/usage/webdav.rst.inc @@ -74,7 +74,20 @@ Description ~~~~~~~~~~~ This command serves the contents of the selected archives via a read-only -HTTP server, so files can be browsed and downloaded with a web browser. +WebDAV / HTTP server, so the archive contents can be: + +- browsed and downloaded with a web browser, +- mounted as a read-only network file system, using the WebDAV client + built into most operating systems and file managers. + +Mounting examples: + +- Windows Explorer: "Map network drive" -> ``http://localhost:8000/`` + (or on the command line: ``net use Z: http://localhost:8000/``). +- macOS Finder: "Go > Connect to Server" (Cmd-K) -> ``http://localhost:8000``. +- GNOME Files / KDE Dolphin: open ``dav://localhost:8000/``. +- Linux kernel mount: ``mount -t davfs http://localhost:8000/ /mnt/point`` + (needs the davfs2 package). The server listens on localhost (127.0.0.1) only and offers no authentication and no encryption - anything that can connect to @@ -87,14 +100,18 @@ in memory when it is first entered, so expect some delay for big archives. Notes: -- Downloads via HTTP do not preserve any POSIX metadata (owner, group, - mode, timestamps, xattrs, ACLs). Use ``borg extract`` or - ``borg export-tar`` for full-fidelity restores. -- Symbolic links are shown in the directory listings, but are neither - followed nor downloadable. Special files (devices, fifos, sockets) - are shown, but not downloadable. +- Downloads do not preserve any POSIX metadata (owner, group, mode, + timestamps, xattrs, ACLs). Use ``borg extract`` or ``borg export-tar`` + for full-fidelity restores. +- Symbolic links and special files (devices, fifos, sockets) are shown + in the web browser listings, but are neither followed nor downloadable, + and they are not visible in WebDAV-mounted directories (WebDAV has no + concept of them). - Damaged files (with chunks missing in the repository) cause the download connection to be aborted - the server never silently serves corrupted file content. +- The Windows WebDAV client limits file downloads to about 47 MiB by + default (``FileSizeLimitInBytes`` registry value) - use a web browser + or another WebDAV client to download bigger files. The command runs in the foreground until it is interrupted with Ctrl-C. \ No newline at end of file diff --git a/src/borg/archiver/webdav_cmd.py b/src/borg/archiver/webdav_cmd.py index 8d337112fc..65c4c7745e 100644 --- a/src/borg/archiver/webdav_cmd.py +++ b/src/borg/archiver/webdav_cmd.py @@ -15,7 +15,7 @@ class WebDAVMixIn: @with_repository(compatibility=(Manifest.Operation.READ,)) def do_webdav(self, args, repository, manifest): - """Serve archive contents via a read-only HTTP server on localhost.""" + """Serve archive contents via a read-only WebDAV / HTTP server on localhost.""" from ..webdav import make_server from ..storelocking import LockRefresher @@ -45,7 +45,20 @@ def build_parser_webdav(self, subparsers, common_parser, mid_common_parser): webdav_epilog = process_epilog( """ This command serves the contents of the selected archives via a read-only - HTTP server, so files can be browsed and downloaded with a web browser. + WebDAV / HTTP server, so the archive contents can be: + + - browsed and downloaded with a web browser, + - mounted as a read-only network file system, using the WebDAV client + built into most operating systems and file managers. + + Mounting examples: + + - Windows Explorer: "Map network drive" -> ``http://localhost:8000/`` + (or on the command line: ``net use Z: http://localhost:8000/``). + - macOS Finder: "Go > Connect to Server" (Cmd-K) -> ``http://localhost:8000``. + - GNOME Files / KDE Dolphin: open ``dav://localhost:8000/``. + - Linux kernel mount: ``mount -t davfs http://localhost:8000/ /mnt/point`` + (needs the davfs2 package). The server listens on localhost (127.0.0.1) only and offers no authentication and no encryption - anything that can connect to @@ -58,21 +71,25 @@ def build_parser_webdav(self, subparsers, common_parser, mid_common_parser): Notes: - - Downloads via HTTP do not preserve any POSIX metadata (owner, group, - mode, timestamps, xattrs, ACLs). Use ``borg extract`` or - ``borg export-tar`` for full-fidelity restores. - - Symbolic links are shown in the directory listings, but are neither - followed nor downloadable. Special files (devices, fifos, sockets) - are shown, but not downloadable. + - Downloads do not preserve any POSIX metadata (owner, group, mode, + timestamps, xattrs, ACLs). Use ``borg extract`` or ``borg export-tar`` + for full-fidelity restores. + - Symbolic links and special files (devices, fifos, sockets) are shown + in the web browser listings, but are neither followed nor downloadable, + and they are not visible in WebDAV-mounted directories (WebDAV has no + concept of them). - Damaged files (with chunks missing in the repository) cause the download connection to be aborted - the server never silently serves corrupted file content. + - The Windows WebDAV client limits file downloads to about 47 MiB by + default (``FileSizeLimitInBytes`` registry value) - use a web browser + or another WebDAV client to download bigger files. The command runs in the foreground until it is interrupted with Ctrl-C. """ ) subparser = ArgumentParser(parents=[common_parser], description=self.do_webdav.__doc__, epilog=webdav_epilog) - subparsers.add_subcommand("webdav", subparser, help="serve archive contents via HTTP") + subparsers.add_subcommand("webdav", subparser, help="serve archive contents via WebDAV / HTTP") subparser.add_argument( "--port", metavar="PORT", diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 5dd1442aab..59a94f169d 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -6,6 +6,7 @@ import threading import urllib.error import urllib.request +import xml.etree.ElementTree as ET from contextlib import contextmanager from types import SimpleNamespace @@ -58,11 +59,39 @@ def webdav_server(archiver): thread.join(timeout=10) -def get(url): - with urllib.request.urlopen(url) as response: +def get(url, headers=None): + request = urllib.request.Request(url, headers=headers or {}) + with urllib.request.urlopen(request) as response: return response.status, dict(response.headers), response.read() +def http_request(url, method, headers=None, body=None): + req = urllib.request.Request(url, data=body, method=method, headers=headers or {}) + with urllib.request.urlopen(req) as response: + return response.status, dict(response.headers), response.read() + + +def propfind(url, depth="1", body=None): + return http_request(url, "PROPFIND", headers={"Depth": depth}, body=body) + + +def propfind_hrefs(xml): + """Parse a multistatus document, return {href: response element}.""" + tree = ET.fromstring(xml) + assert tree.tag == "{DAV:}multistatus" + return {resp.find("{DAV:}href").text: resp for resp in tree.findall("{DAV:}response")} + + +def prop_of(response, tag, status="200 OK"): + """Return the property element *tag* from the propstat with *status*, or None.""" + for propstat in response.findall("{DAV:}propstat"): + if propstat.find("{DAV:}status").text.endswith(status): + elem = propstat.find(f"{{DAV:}}prop/{tag}") + if elem is not None: + return elem + return None + + def test_webdav_browse(archivers, request): archiver = request.getfixturevalue(archivers) _create_archive(archiver) @@ -139,3 +168,127 @@ def test_webdav_errors(archivers, request): with pytest.raises(urllib.error.HTTPError) as exc_info: urllib.request.urlopen(urllib.request.Request(base_url + "/", method="POST")) assert exc_info.value.code == 405 + # writing/locking WebDAV methods are rejected, too + for method in "PUT", "MKCOL", "LOCK", "PROPPATCH": + with pytest.raises(urllib.error.HTTPError) as exc_info: + http_request(base_url + "/test/input/file1", method) + assert exc_info.value.code == 405 + assert "PROPFIND" in exc_info.value.headers["Allow"] + + +def test_webdav_options(archivers, request): + archiver = request.getfixturevalue(archivers) + _create_archive(archiver) + with webdav_server(archiver) as base_url: + status, headers, _ = http_request(base_url + "/", "OPTIONS") + assert status == 200 + assert headers["DAV"] == "1" + assert "PROPFIND" in headers["Allow"] + + +def test_webdav_propfind(archivers, request): + archiver = request.getfixturevalue(archivers) + _create_archive(archiver) + with webdav_server(archiver) as base_url: + # depth 0 on a file: exactly one response with the file's properties + status, headers, body = propfind(base_url + "/test/input/file1", depth="0") + assert status == 207 + assert headers["Content-Type"].startswith("application/xml") + hrefs = propfind_hrefs(body) + assert list(hrefs) == ["/test/input/file1"] + response = hrefs["/test/input/file1"] + assert prop_of(response, "{DAV:}getcontentlength").text == "5" + assert prop_of(response, "{DAV:}resourcetype").find("{DAV:}collection") is None + assert prop_of(response, "{DAV:}getetag").text.startswith('"') + assert "GMT" in prop_of(response, "{DAV:}getlastmodified").text + # depth 1 on a directory: the directory itself and its children + status, _, body = propfind(base_url + "/test/input/", depth="1") + hrefs = propfind_hrefs(body) + assert "/test/input/" in hrefs + assert "/test/input/subdir/" in hrefs + assert "/test/input/file1" in hrefs + assert "/test/input/a%20%3Cb%3E%26c%20%C3%A4.txt" in hrefs + assert prop_of(hrefs["/test/input/subdir/"], "{DAV:}resourcetype").find("{DAV:}collection") is not None + # symlinks are not exposed via WebDAV + assert not any("link1" in href for href in hrefs) + # depth 1 on the server root: archives are listed as collections + status, _, body = propfind(base_url + "/", depth="1") + hrefs = propfind_hrefs(body) + assert "/" in hrefs and "/test/" in hrefs + assert prop_of(hrefs["/test/"], "{DAV:}resourcetype").find("{DAV:}collection") is not None + + +def test_webdav_propfind_body(archivers, request): + archiver = request.getfixturevalue(archivers) + _create_archive(archiver) + with webdav_server(archiver) as base_url: + # a named-prop request returns the known prop with 200 and the unknown one with 404 + body = ( + b'' + b"" + b"" + ) + status, _, result = propfind(base_url + "/test/input/file1", depth="0", body=body) + assert status == 207 + response = propfind_hrefs(result)["/test/input/file1"] + assert prop_of(response, "{DAV:}getcontentlength").text == "5" + assert prop_of(response, "{DAV:}nosuchprop", status="404 Not Found") is not None + # garbage body -> 400 + with pytest.raises(urllib.error.HTTPError) as exc_info: + propfind(base_url + "/test/input/file1", depth="0", body=b"") + assert exc_info.value.code == 400 + # depth infinity is refused + with pytest.raises(urllib.error.HTTPError) as exc_info: + propfind(base_url + "/test/input/", depth="infinity") + assert exc_info.value.code == 403 + # PROPFIND on a symlink or a missing path -> 404 + for path in "/test/input/link1", "/test/input/no-such-file": + with pytest.raises(urllib.error.HTTPError) as exc_info: + propfind(base_url + path, depth="0") + assert exc_info.value.code == 404 + + +def test_webdav_ranges(archivers, request): + archiver = request.getfixturevalue(archivers) + big = _create_archive(archiver) + size = len(big) + with webdav_server(archiver) as base_url: + url = base_url + "/test/input/big" + # a range within the first chunk + status, headers, body = get(url, headers={"Range": "bytes=10-99"}) + assert status == 206 + assert body == big[10:100] + assert headers["Content-Range"] == f"bytes 10-99/{size}" + assert headers["Content-Length"] == "90" + # a range spanning chunk boundaries, somewhere in the middle + start, end = size // 2 - 1000, size // 2 + 1000 + status, headers, body = get(url, headers={"Range": f"bytes={start}-{end}"}) + assert status == 206 + assert body == big[start : end + 1] + # an open-ended range and a suffix range + status, _, body = get(url, headers={"Range": f"bytes={size - 10}-"}) + assert status == 206 and body == big[-10:] + status, _, body = get(url, headers={"Range": "bytes=-10"}) + assert status == 206 and body == big[-10:] + # an unsatisfiable range + with pytest.raises(urllib.error.HTTPError) as exc_info: + get(url, headers={"Range": f"bytes={size}-"}) + assert exc_info.value.code == 416 + assert exc_info.value.headers["Content-Range"] == f"bytes */{size}" + + +def test_webdav_conditional_get(archivers, request): + archiver = request.getfixturevalue(archivers) + _create_archive(archiver) + with webdav_server(archiver) as base_url: + url = base_url + "/test/input/file1" + status, headers, _ = get(url) + assert status == 200 + etag = headers["ETag"] + assert headers["Accept-Ranges"] == "bytes" + with pytest.raises(urllib.error.HTTPError) as exc_info: + get(url, headers={"If-None-Match": etag}) + assert exc_info.value.code == 304 + # a non-matching etag serves the content normally + status, _, body = get(url, headers={"If-None-Match": '"different"'}) + assert status == 200 and body == b"data1" diff --git a/src/borg/webdav.py b/src/borg/webdav.py index c86f06e2d8..153a12b9e2 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -1,8 +1,11 @@ -"""Read-only HTTP server providing browser access to archive contents (``borg webdav``). +"""Read-only WebDAV / HTTP server providing access to archive contents (``borg webdav``). -Phase 1 implements plain HTTP only (GET/HEAD, HTML directory listings, file -downloads), bound to localhost. WebDAV methods (OPTIONS/PROPFIND) for native -mounting by Explorer/Finder/GVFS are planned to be added on top of this later. +Web browsers get HTML directory listings and file downloads (GET/HEAD, incl. +Range requests and conditional requests). WebDAV clients (class 1: OPTIONS, +PROPFIND) can mount the served archives as a read-only network file system - +such clients are built into Windows Explorer, macOS Finder, the common Linux +file managers (gvfs/KIO) and davfs2. All methods that would modify something +(PUT, DELETE, PROPPATCH, MKCOL, COPY, MOVE, LOCK, ...) are rejected. This module only uses the Python standard library, so it works without any optional dependencies. @@ -10,12 +13,14 @@ import html import mimetypes +import re import stat import threading from datetime import datetime, timezone from email.utils import formatdate from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import quote, unquote_to_bytes +from xml.etree import ElementTree as ET from . import __version__ from .archive import Archive @@ -69,6 +74,8 @@ def __init__(self, manifest, args, repo_lock): name += f"-{bin_to_hex(archive.id):.8}" self.archives[name] = archive self._trees = {} # display name -> (root Node, DownloadPipeline) + timestamps = [archive.ts for archive in archives] + self.root_mtime = int(max(timestamps).timestamp() * 1e9) if timestamps else 0 def get_root(self, name): """Return (root Node, pipeline) for archive *name*, building the tree on first access.""" @@ -156,6 +163,151 @@ def display_size(size): return f"{size:,}".replace(",", ".") +def guess_content_type(name): + return mimetypes.guess_type(remove_surrogates(name), strict=False)[0] or "application/octet-stream" + + +def make_etag(node): + # archive contents are immutable, so mtime+size identify the content well enough. + return f'"{node.mtime:x}-{node.size:x}"' + + +def parse_byte_range(header, size): + """Parse a Range header value against a resource of *size* bytes. + + Returns (start, end) (both inclusive), None if the header shall be ignored + (serve the full body then), or "unsatisfiable" (respond with 416 then). + """ + m = re.fullmatch(r"bytes=(\d*)-(\d*)", header.strip()) + if not m: # multiple ranges / other units: ignoring the header is allowed + return None + start_s, end_s = m.groups() + if not start_s and not end_s: + return None + if not start_s: # suffix form: the last N bytes + n = int(end_s) + if n == 0 or size == 0: + return "unsatisfiable" + return max(size - n, 0), size - 1 + start = int(start_s) + if start >= size: + return "unsatisfiable" + end = min(int(end_s), size - 1) if end_s else size - 1 + if end < start: + return None + return start, end + + +# WebDAV support (class 1, read-only), see RFC 4918. + +ET.register_namespace("D", "DAV:") + +ALLOWED_METHODS = "OPTIONS, GET, HEAD, PROPFIND" + +# all live properties this server can provide +DAV_PROPS = ( + "{DAV:}resourcetype", + "{DAV:}displayname", + "{DAV:}getcontentlength", + "{DAV:}getcontenttype", + "{DAV:}getlastmodified", + "{DAV:}creationdate", + "{DAV:}getetag", + "{DAV:}supportedlock", + "{DAV:}lockdiscovery", +) + + +def iso8601(mtime_ns): + return datetime.fromtimestamp(mtime_ns / 1e9, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def parse_propfind(body): + """Parse a PROPFIND request body. + + Returns a (mode, props) tuple: ("allprop", None), ("propname", None) or + ("prop", [tag, ...]). Raises ValueError or ET.ParseError for bodies we do + not understand. Note: the maximum body size is limited by the caller and + expat has built-in protection against entity expansion attacks. + """ + if not body.strip(): + return "allprop", None # RFC 4918: an empty body means allprop + root = ET.fromstring(body) + if root.tag != "{DAV:}propfind": + raise ValueError("root element is not DAV: propfind") + if root.find("{DAV:}propname") is not None: + return "propname", None + prop = root.find("{DAV:}prop") + if prop is not None: + return "prop", [child.tag for child in prop] + return "allprop", None # allprop, maybe with an include element + + +def make_prop_element(tag, name, node): + """Build the XML element for live property *tag* of resource *node*. + + Returns None if the property is not defined for this resource. + """ + elem = ET.Element(tag) + if tag == "{DAV:}resourcetype": + if node.is_dir: + ET.SubElement(elem, "{DAV:}collection") + elif tag == "{DAV:}displayname": + elem.text = remove_surrogates(name) + elif tag == "{DAV:}getlastmodified": + elem.text = http_date(node.mtime) + elif tag == "{DAV:}creationdate": + elem.text = iso8601(node.mtime) + elif tag == "{DAV:}getcontentlength": + if node.is_dir: + return None + elem.text = str(node.size) + elif tag == "{DAV:}getcontenttype": + if node.is_dir: + return None + elem.text = guess_content_type(name) + elif tag == "{DAV:}getetag": + if node.is_dir: + return None + elem.text = make_etag(node) + elif tag in ("{DAV:}supportedlock", "{DAV:}lockdiscovery"): + pass # empty elements: locking is not supported + else: + return None + return elem + + +def render_multistatus(resources, mode, requested): + """Render a PROPFIND result as a multistatus XML document (bytes). + + *resources* is a list of (href, displayname, node) tuples, *mode* / *requested* + are the parse_propfind() results. + """ + multistatus = ET.Element("{DAV:}multistatus") + for href, name, node in resources: + response = ET.SubElement(multistatus, "{DAV:}response") + ET.SubElement(response, "{DAV:}href").text = href + found = ET.Element("{DAV:}prop") + missing = ET.Element("{DAV:}prop") + if mode == "propname": + for tag in DAV_PROPS: + if make_prop_element(tag, name, node) is not None: + ET.SubElement(found, tag) + else: + for tag in DAV_PROPS if mode == "allprop" else requested: + elem = make_prop_element(tag, name, node) + if elem is not None: + found.append(elem) + else: + missing.append(ET.Element(tag)) + for prop, status in ((found, "200 OK"), (missing, "404 Not Found")): + if len(prop) or status == "200 OK": # always emit the 200 propstat, even if empty + propstat = ET.SubElement(response, "{DAV:}propstat") + propstat.append(prop) + ET.SubElement(propstat, "{DAV:}status").text = f"HTTP/1.1 {status}" + return b'\n' + ET.tostring(multistatus) + + # the official borg logo (docs/_static/logo.svg, "borg" in vectorized Black Ops One), # without the background rect and with the fill color controlled via CSS currentColor. LOGO_SVG = ( @@ -253,48 +405,36 @@ def log_message(self, format, *args): logger.debug("webdav: %s - %s", self.address_string(), format % args) def do_GET(self): - self._handle(head=False) + self._guarded(self._handle_get_head, False) def do_HEAD(self): - self._handle(head=True) + self._guarded(self._handle_get_head, True) + + def do_OPTIONS(self): + self.send_response(200) + self.send_header("Allow", ALLOWED_METHODS) + self.send_header("DAV", "1") + self.send_header("MS-Author-Via", "DAV") # helps (older) Windows WebDAV clients + self.send_header("Content-Length", "0") + self.end_headers() + + def do_PROPFIND(self): + self._guarded(self._handle_propfind) def _method_not_allowed(self): + # this is a read-only server, reject everything that would modify or lock something. self.send_response(405) - self.send_header("Allow", "GET, HEAD") + self.send_header("Allow", ALLOWED_METHODS) self.send_header("Content-Length", "0") self.end_headers() + self.close_connection = True # we did not read a request body the client may have sent - do_POST = do_PUT = do_DELETE = do_OPTIONS = do_PATCH = _method_not_allowed + do_POST = do_PUT = do_DELETE = do_PATCH = _method_not_allowed + do_PROPPATCH = do_MKCOL = do_COPY = do_MOVE = do_LOCK = do_UNLOCK = _method_not_allowed - def _handle(self, head): + def _guarded(self, method, *args): try: - path, _, _query = self.path.partition("?") - path = decode_path(path) - segments = [s for s in path.split("/") if s] - if any(s in (".", "..") for s in segments): - self.send_error(404) - return - if not segments: - self._send_archive_list(head) - return - try: - node, pipeline = self.vfs.resolve(segments) - except KeyError: - self.send_error(404) - return - if node.is_dir: - if not path.endswith("/"): - self._redirect_to_dir() - return - self._send_dir_listing(segments, node, head) - elif stat.S_ISREG(node.mode): - self._send_file(segments[-1], node, pipeline, head) - elif stat.S_ISLNK(node.mode): - self.send_error( - 403, explain=f"symbolic link (target: {remove_surrogates(node.target or '?')}), not downloadable" - ) - else: - self.send_error(403, explain="special file, not downloadable") + method(*args) except BrokenPipeError: self.close_connection = True except Exception: @@ -304,6 +444,118 @@ def _handle(self, head): except OSError: self.close_connection = True + def _parse_segments(self): + """Parse self.path into decoded path segments; returns None for invalid paths.""" + path, _, _query = self.path.partition("?") + path = decode_path(path) + segments = [s for s in path.split("/") if s] + if any(s in (".", "..") for s in segments): + return None + return segments, path.endswith("/") + + def _read_body(self): + """Read a request body; returns None (after sending an error) if that is not possible.""" + if self.headers.get("Transfer-Encoding"): + self.send_error(501, explain="request bodies with Transfer-Encoding are not supported") + self.close_connection = True + return None + try: + length = int(self.headers.get("Content-Length") or 0) + except ValueError: + self.send_error(400) + self.close_connection = True + return None + if length < 0 or length > 1024 * 1024: + self.send_error(413) + self.close_connection = True + return None + return self.rfile.read(length) + + def _handle_get_head(self, head): + parsed = self._parse_segments() + if parsed is None: + self.send_error(404) + return + segments, dir_syntax = parsed + if not segments: + self._send_archive_list(head) + return + try: + node, pipeline = self.vfs.resolve(segments) + except KeyError: + self.send_error(404) + return + if node.is_dir: + if not dir_syntax: + self._redirect_to_dir() + return + self._send_dir_listing(segments, node, head) + elif stat.S_ISREG(node.mode): + self._send_file(segments[-1], node, pipeline, head) + elif stat.S_ISLNK(node.mode): + self.send_error( + 403, explain=f"symbolic link (target: {remove_surrogates(node.target or '?')}), not downloadable" + ) + else: + self.send_error(403, explain="special file, not downloadable") + + def _handle_propfind(self): + body = self._read_body() + if body is None: + return + parsed = self._parse_segments() + if parsed is None: + self.send_error(404) + return + depth = self.headers.get("Depth", "infinity").strip().lower() + if depth not in ("0", "1"): + # RFC 4918 allows servers to reject PROPFIND requests with unlimited depth. + self.send_error(403, explain="PROPFIND with Depth: infinity is not supported") + return + try: + mode, requested = parse_propfind(body) + except (ET.ParseError, ValueError): + self.send_error(400, explain="invalid PROPFIND request body") + return + try: + resources = self._propfind_resources(parsed[0], depth) + except KeyError: + self.send_error(404) + return + result = render_multistatus(resources, mode, requested) + self.send_response(207, "Multi-Status") + self.send_header("Content-Type", 'application/xml; charset="utf-8"') + self.send_header("Content-Length", str(len(result))) + self.end_headers() + self.wfile.write(result) + + def _propfind_resources(self, segments, depth): + """Return the [(href, displayname, node), ...] a PROPFIND on *segments* refers to.""" + resources = [] + if not segments: # server root: the list of archives + resources.append(("/", "/", Node(DEFAULT_DIR_MODE, mtime=self.vfs.root_mtime, children={}))) + if depth == "1": + for name in sorted(self.vfs.archives): + archive_info = self.vfs.archives[name] + node = Node(DEFAULT_DIR_MODE, mtime=int(archive_info.ts.timestamp() * 1e9), children={}) + resources.append(("/" + encode_path(name) + "/", name, node)) + return resources + node, _ = self.vfs.resolve(segments) # may raise KeyError + if not (node.is_dir or stat.S_ISREG(node.mode)): + raise KeyError(segments[-1]) # symlinks and special files are not exposed via WebDAV + base = "/" + "/".join(encode_path(s) for s in segments) + if not node.is_dir: + return [(base, segments[-1], node)] + resources.append((base + "/", segments[-1], node)) + if depth == "1": + for name, child in sorted(node.children.items()): + if child.is_dir: + resources.append((f"{base}/{encode_path(name)}/", name, child)) + elif stat.S_ISREG(child.mode): + resources.append((f"{base}/{encode_path(name)}", name, child)) + # symlinks and special files are not exposed via WebDAV + return resources + def _redirect_to_dir(self): path, _, _query = self.path.partition("?") self.send_response(301) @@ -348,18 +600,60 @@ def _send_dir_listing(self, segments, node, head): self._send_page(render_page(title, rows), head) def _send_file(self, name, node, pipeline, head): - self.send_response(200) - ctype = mimetypes.guess_type(remove_surrogates(name), strict=False)[0] or "application/octet-stream" - self.send_header("Content-Type", ctype) - self.send_header("Content-Length", str(node.size)) + etag = make_etag(node) + if_none_match = self.headers.get("If-None-Match") + if if_none_match: + client_tags = [t.strip() for t in if_none_match.split(",")] + if "*" in client_tags or etag in client_tags: + self.send_response(304) + self.send_header("ETag", etag) + self.end_headers() + return + byte_range = None + range_header = self.headers.get("Range") + if range_header: + if_range = self.headers.get("If-Range") + if if_range is None or if_range.strip() == etag: + byte_range = parse_byte_range(range_header, node.size) + if byte_range == "unsatisfiable": + self.send_response(416) + self.send_header("Content-Range", f"bytes */{node.size}") + self.send_header("Content-Length", "0") + self.end_headers() + return + if byte_range: + start, end = byte_range + self.send_response(206) + self.send_header("Content-Range", f"bytes {start}-{end}/{node.size}") + else: + start, end = 0, node.size - 1 + self.send_response(200) + self.send_header("Content-Type", guess_content_type(name)) + self.send_header("Content-Length", str(end - start + 1 if node.size else 0)) self.send_header("Last-Modified", http_date(node.mtime)) + self.send_header("ETag", etag) + self.send_header("Accept-Ranges", "bytes") self.send_header("X-Content-Type-Options", "nosniff") self.send_header("Content-Disposition", self._content_disposition(name)) self.end_headers() - if head or not node.chunks: + if head or not node.chunks or node.size == 0: return - chunk_iter = pipeline.fetch_many(node.chunks, ro_type=ROBJ_FILE_STREAM, replacement_chunk=False) - while True: + # select only the chunks overlapping the requested range, so nothing else + # gets fetched and decrypted (the chunk sizes are known in advance). + selected, first_offset, pos = [], 0, 0 + for entry in node.chunks: + if pos + entry.size <= start: + pos += entry.size + continue + if pos > end: + break + if not selected: + first_offset = start - pos + selected.append(entry) + pos += entry.size + remaining = end - start + 1 + chunk_iter = pipeline.fetch_many(selected, ro_type=ROBJ_FILE_STREAM, replacement_chunk=False) + while remaining > 0: # serialize repository access, but write to the client outside the lock, # so one slow client cannot block other requests for the whole download. with self.repo_lock: @@ -375,7 +669,13 @@ def _send_file(self, name, node, pipeline, head): ) self.close_connection = True return + if first_offset: + data = data[first_offset:] + first_offset = 0 + if len(data) > remaining: + data = data[:remaining] self.wfile.write(data) + remaining -= len(data) @staticmethod def _content_disposition(name): From 4ea62418814fb98a510135a2752fa324780dc3f5 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 18:32:30 +0200 Subject: [PATCH 03/20] webdav: fix HTTP response splitting via file names, harden Location Evaluation of the CodeQL py/http-response-splitting alerts: - Content-Disposition (real issue, fixed): the fallback file name in the header was built from the archived file name, which can contain any byte except NUL and "/" - including CR/LF. A file named e.g. "x\r\nEvil-Header: ..." planted in the backed up data would have injected headers into the download response. Now every non-printable character is replaced before building the fallback name (the RFC 8187 encoded name was already safe, quote() percent-encodes CR/LF). - Location (not exploitable, hardened anyway): the redirect echoed the raw request line path, which can never contain raw CR/LF (they would terminate the request line). The Location value is now rebuilt from the parsed path segments via percent-encoding, so it provably only contains URL-safe ASCII and no client input is echoed at all. Both send_header lines carry codeql[] suppression comments with the justification nearby, as CodeQL's taint tracking does not recognize these sanitizers. Add a test that plants file and directory names containing CR/LF in an archive and asserts that no header can be injected via download and redirect responses. Co-Authored-By: Claude Fable 5 --- .../testsuite/archiver/webdav_cmd_test.py | 36 +++++++++++++++++++ src/borg/webdav.py | 24 +++++++++---- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 59a94f169d..d692f5cc0f 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -2,6 +2,7 @@ # The server is started in-process (no borg CLI invocation for the serving part), # so these tests run without any optional dependency. +import http.client import os import threading import urllib.error @@ -9,6 +10,7 @@ import xml.etree.ElementTree as ET from contextlib import contextmanager from types import SimpleNamespace +from urllib.parse import urlsplit import pytest @@ -277,6 +279,40 @@ def test_webdav_ranges(archivers, request): assert exc_info.value.headers["Content-Range"] == f"bytes */{size}" +def test_webdav_header_injection(archivers, request): + # File/directory names from an archive may contain CR/LF - names like + # "x\r\nEvil-Header: 1" must not be able to inject headers into responses + # that echo the name (Content-Disposition, Location), see CodeQL + # py/http-response-splitting and the OWASP article on response splitting. + archiver = request.getfixturevalue(archivers) + evil_file = "inj\r\nEvil-Header: 1" + evil_dir = "dir\r\nEvil-Header: 2" + create_regular_file(archiver.input_path, evil_file, contents=b"gotcha") + create_regular_file(archiver.input_path, evil_dir + "/inner", contents=b"inner") + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test", "input") + with webdav_server(archiver) as base_url: + host_port = urlsplit(base_url).netloc + # downloading the evil file: Content-Disposition must be sanitized + status, headers, body = get(f"{base_url}/test/input/inj%0D%0AEvil-Header%3A%201") + assert status == 200 + assert body == b"gotcha" + assert "Evil-Header" not in headers + assert 'filename="inj__Evil-Header: 1"' in headers["Content-Disposition"] + assert "%0D%0A" in headers["Content-Disposition"] # RFC 8187 encoded form + # the directory redirect: Location must be percent-encoded, no header injected + # (plain http.client here, so the 301 is not followed) + conn = http.client.HTTPConnection(host_port) + try: + conn.request("GET", "/test/input/dir%0D%0AEvil-Header%3A%202") + response = conn.getresponse() + assert response.status == 301 + assert response.getheader("Evil-Header") is None + assert response.getheader("Location") == "/test/input/dir%0D%0AEvil-Header%3A%202/" + finally: + conn.close() + + def test_webdav_conditional_get(archivers, request): archiver = request.getfixturevalue(archivers) _create_archive(archiver) diff --git a/src/borg/webdav.py b/src/borg/webdav.py index 153a12b9e2..3331f91a11 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -487,7 +487,7 @@ def _handle_get_head(self, head): return if node.is_dir: if not dir_syntax: - self._redirect_to_dir() + self._redirect_to_dir(segments) return self._send_dir_listing(segments, node, head) elif stat.S_ISREG(node.mode): @@ -556,10 +556,13 @@ def _propfind_resources(self, segments, depth): # symlinks and special files are not exposed via WebDAV return resources - def _redirect_to_dir(self): - path, _, _query = self.path.partition("?") + def _redirect_to_dir(self, segments): + # Build the redirect target by percent-encoding the parsed path segments: + # quote() only outputs URL-safe ASCII, so the Location value cannot contain + # CR/LF or other header-splitting characters, no matter what the client sent. + location = "/" + "/".join(encode_path(s) for s in segments) + "/" self.send_response(301) - self.send_header("Location", path + "/") + self.send_header("Location", location) # codeql[py/http-response-splitting] self.send_header("Content-Length", "0") self.end_headers() @@ -634,7 +637,8 @@ def _send_file(self, name, node, pipeline, head): self.send_header("ETag", etag) self.send_header("Accept-Ranges", "bytes") self.send_header("X-Content-Type-Options", "nosniff") - self.send_header("Content-Disposition", self._content_disposition(name)) + content_disposition = self._content_disposition(name) # sanitized, see there + self.send_header("Content-Disposition", content_disposition) # codeql[py/http-response-splitting] self.end_headers() if head or not node.chunks or node.size == 0: return @@ -679,7 +683,15 @@ def _send_file(self, name, node, pipeline, head): @staticmethod def _content_disposition(name): - fallback = remove_surrogates(name).encode("ascii", "replace").decode("ascii").replace('"', "'") + # File names from an archive can contain any byte except NUL and "/", including + # CR/LF - a file name like 'x\r\nEvil-Header: ...' must not enable the client + # to be attacked via HTTP header injection ("response splitting"): + # - the fallback name replaces everything non-printable (this kills CR/LF) and + # non-ascii, and the quotes that could end the quoted-string. + # - the RFC 8187 encoded name percent-encodes everything critical (quote() + # only outputs URL-safe ASCII). + fallback = "".join(c if c.isprintable() else "_" for c in remove_surrogates(name)) + fallback = fallback.encode("ascii", "replace").decode("ascii").replace('"', "'") encoded = quote(name.encode("utf-8", "surrogateescape")) return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{encoded}" From 5c662e3cf42a6bc7df4dc4191362b02afb56554f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 18:41:05 +0200 Subject: [PATCH 04/20] webdav: make the heading a breadcrumb for quick navigation Each segment of the path shown in the heading (e.g. "archive/data/docs/") is now a link to the corresponding directory. The h1 links inherit the heading color, so the page looks unchanged. The browser tab title stays plain text. Co-Authored-By: Claude Fable 5 --- .../testsuite/archiver/webdav_cmd_test.py | 2 ++ src/borg/webdav.py | 22 +++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index d692f5cc0f..5724783787 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -112,6 +112,8 @@ def test_webdav_browse(archivers, request): assert 'subdir/' in page # precise sizes in bytes, with dots as thousands separators assert ">5.242.880<" in page + # the heading is a breadcrumb: every path segment links to its directory + assert '

test/input/

' in page # funny file name is html-escaped in text and percent-encoded in the link assert "a <b>&c ä.txt" in page assert 'href="a%20%3Cb%3E%26c%20%C3%A4.txt"' in page diff --git a/src/borg/webdav.py b/src/borg/webdav.py index 3331f91a11..1a23fe42e9 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -347,6 +347,7 @@ def render_multistatus(resources, mode, requested): margin: 2em; }} h1 {{ color: #22d045; font-size: 1.4em; }} +h1 a {{ color: inherit; }} a {{ color: #22d045; text-decoration: none; }} a:hover {{ text-decoration: underline; }} table {{ border-collapse: collapse; }} @@ -362,7 +363,7 @@ def render_multistatus(resources, mode, requested):
-

{title}

+

{heading}

@@ -374,11 +375,23 @@ def render_multistatus(resources, mode, requested): """ -def render_page(title, rows): - page = PAGE_TEMPLATE.format(title=html.escape(title), rows="\n".join(rows), logo=LOGO_SVG) +def render_page(title, rows, heading=None): + """Render a listing page; *title* is plain text, *heading* optional h1 HTML (default: the title).""" + heading = heading if heading is not None else html.escape(title) + page = PAGE_TEMPLATE.format(title=html.escape(title), heading=heading, rows="\n".join(rows), logo=LOGO_SVG) return page.encode("utf-8") +def make_breadcrumbs(segments): + """Build h1 HTML for a path: each segment linked to its directory, for quick navigation.""" + parts = [] + href = "/" + for segment in segments: + href += encode_path(segment) + "/" + parts.append(f'{html.escape(remove_surrogates(segment))}') + return "/".join(parts) + "/" + + def make_row(href, text, size="", mtime_ns=None): modified = display_time(mtime_ns) if mtime_ns is not None else "" if href is not None: @@ -585,6 +598,7 @@ def _send_archive_list(self, head): def _send_dir_listing(self, segments, node, head): title = "/".join(remove_surrogates(s) for s in segments) + "/" + heading = make_breadcrumbs(segments) rows = [make_row("../", "..")] children = sorted(node.children.items(), key=lambda kv: (not kv[1].is_dir, kv[0])) for name, child in children: @@ -600,7 +614,7 @@ def _send_dir_listing(self, segments, node, head): rows.append(make_row(None, text, mtime_ns=child.mtime)) else: rows.append(make_row(None, display_name, mtime_ns=child.mtime)) - self._send_page(render_page(title, rows), head) + self._send_page(render_page(title, rows, heading=heading), head) def _send_file(self, name, node, pipeline, head): etag = make_etag(node) From 4838896c6d06d0615a538c09c5a689e3269ff2d6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 18:44:58 +0200 Subject: [PATCH 05/20] webdav: reject DTDs in PROPFIND bodies (XML bomb defense) A malicious PROPFIND body could declare internal XML entities that expand exponentially ("billion laughs"). Reject any DTD before parsing, using expat's StartDoctypeDeclHandler: it fires at the start of the doctype declaration, before any entity is declared or expanded, so no custom entities can exist and no expansion is possible. Using expat's callback rather than a substring check on the raw body makes this independent of the document encoding (a DTD in e.g. UTF-16 does not contain the ascii bytes " --- .../testsuite/archiver/webdav_cmd_test.py | 18 +++++++++++ src/borg/webdav.py | 32 +++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 5724783787..0b91e5dbf6 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -241,6 +241,24 @@ def test_webdav_propfind_body(archivers, request): with pytest.raises(urllib.error.HTTPError) as exc_info: propfind(base_url + "/test/input/file1", depth="0", body=b"") assert exc_info.value.code == 400 + # bodies containing a DTD are rejected (entity expansion / "XML bomb" defense) + bomb = ( + b']>' + b'&b;' + ) + with pytest.raises(urllib.error.HTTPError) as exc_info: + propfind(base_url + "/test/input/file1", depth="0", body=bomb) + assert exc_info.value.code == 400 + # the DTD rejection is encoding-proof: a UTF-16 encoded DTD (whose bytes do + # not contain the ascii "' + ']>' + '&a;' + ).encode("utf-16") + with pytest.raises(urllib.error.HTTPError) as exc_info: + propfind(base_url + "/test/input/file1", depth="0", body=utf16_bomb) + assert exc_info.value.code == 400 # depth infinity is refused with pytest.raises(urllib.error.HTTPError) as exc_info: propfind(base_url + "/test/input/", depth="infinity") diff --git a/src/borg/webdav.py b/src/borg/webdav.py index 1a23fe42e9..d078e69fff 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -21,6 +21,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import quote, unquote_to_bytes from xml.etree import ElementTree as ET +from xml.parsers import expat from . import __version__ from .archive import Archive @@ -222,17 +223,42 @@ def iso8601(mtime_ns): return datetime.fromtimestamp(mtime_ns / 1e9, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +def reject_dtd(body): + """Raise ValueError if *body* contains an XML DTD (). + + A DTD is the only way to declare custom internal entities, so rejecting it + rules out entity expansion attacks ("billion laughs"). We use expat's own + doctype callback rather than a substring check on the raw bytes, so this is + independent of the document encoding (a DTD in e.g. UTF-16 does not contain + the ASCII bytes " Date: Thu, 23 Jul 2026 19:33:12 +0200 Subject: [PATCH 06/20] webdav: cache decrypted chunks (BORG_MOUNT_DATA_CACHE_ENTRIES) A mounted WebDAV file system reads a big file as many small, sequential range requests, which tend to hit the same content chunk over and over. Without caching, each request re-fetched and re-decrypted the chunk. Add a shared LRU cache of decrypted chunks (keyed by chunk id, so it is valid across archives and requests), the same approach borg mount uses. Its size is set by BORG_MOUNT_DATA_CACHE_ENTRIES (default: number of CPUs), just like for borg mount. The cache is guarded by repo_lock (the server is multi-threaded and LRUCache is not thread-safe), while writing to the client still happens outside the lock. Co-Authored-By: Claude Fable 5 --- docs/man/borg-webdav.1 | 7 +++ docs/usage/webdav.rst.inc | 7 +++ src/borg/archiver/webdav_cmd.py | 7 +++ .../testsuite/archiver/webdav_cmd_test.py | 30 ++++++++++ src/borg/webdav.py | 56 ++++++++++++++++--- 5 files changed, 98 insertions(+), 9 deletions(-) diff --git a/docs/man/borg-webdav.1 b/docs/man/borg-webdav.1 index 7bf77d2848..119dcc99b4 100644 --- a/docs/man/borg-webdav.1 +++ b/docs/man/borg-webdav.1 @@ -90,6 +90,13 @@ default (\fBFileSizeLimitInBytes\fP registry value) \- use a web browser or another WebDAV client to download bigger files. .UNINDENT .sp +Recently used file content chunks are kept decrypted in an in\-memory cache, +so that the many small, sequential range requests a mounted file system +does for a big file do not re\-fetch and re\-decrypt the same chunk over and +over. As for \fBborg mount\fP, the \fBBORG_MOUNT_DATA_CACHE_ENTRIES\fP environment +variable sets the number of cached chunks (default: number of CPUs); +additional memory usage can be up to the chunk size times this number. +.sp The command runs in the foreground until it is interrupted with Ctrl\-C. .SH OPTIONS .sp diff --git a/docs/usage/webdav.rst.inc b/docs/usage/webdav.rst.inc index 9cca04cc09..4c717473d7 100644 --- a/docs/usage/webdav.rst.inc +++ b/docs/usage/webdav.rst.inc @@ -114,4 +114,11 @@ Notes: default (``FileSizeLimitInBytes`` registry value) - use a web browser or another WebDAV client to download bigger files. +Recently used file content chunks are kept decrypted in an in-memory cache, +so that the many small, sequential range requests a mounted file system +does for a big file do not re-fetch and re-decrypt the same chunk over and +over. As for ``borg mount``, the ``BORG_MOUNT_DATA_CACHE_ENTRIES`` environment +variable sets the number of cached chunks (default: number of CPUs); +additional memory usage can be up to the chunk size times this number. + The command runs in the foreground until it is interrupted with Ctrl-C. \ No newline at end of file diff --git a/src/borg/archiver/webdav_cmd.py b/src/borg/archiver/webdav_cmd.py index 65c4c7745e..b27c7d127e 100644 --- a/src/borg/archiver/webdav_cmd.py +++ b/src/borg/archiver/webdav_cmd.py @@ -85,6 +85,13 @@ def build_parser_webdav(self, subparsers, common_parser, mid_common_parser): default (``FileSizeLimitInBytes`` registry value) - use a web browser or another WebDAV client to download bigger files. + Recently used file content chunks are kept decrypted in an in-memory cache, + so that the many small, sequential range requests a mounted file system + does for a big file do not re-fetch and re-decrypt the same chunk over and + over. As for ``borg mount``, the ``BORG_MOUNT_DATA_CACHE_ENTRIES`` environment + variable sets the number of cached chunks (default: number of CPUs); + additional memory usage can be up to the chunk size times this number. + The command runs in the foreground until it is interrupted with Ctrl-C. """ ) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 0b91e5dbf6..05cf06bc5a 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -333,6 +333,36 @@ def test_webdav_header_injection(archivers, request): conn.close() +def test_webdav_data_cache(archivers, request, monkeypatch): + # decrypted chunks are cached across requests, sized by BORG_MOUNT_DATA_CACHE_ENTRIES. + monkeypatch.setenv("BORG_MOUNT_DATA_CACHE_ENTRIES", "8") + archiver = request.getfixturevalue(archivers) + big = _create_archive(archiver) + args = SimpleNamespace( + sort_by="ts", match_archives=None, first=None, last=None, older=None, newer=None, oldest=None, newest=None + ) + repository = Repository(archiver.repository_path, exclusive=True) + with repository: + manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + server = make_server(manifest, args, port=0) + assert server.data_cache._capacity == 8 # the env var is honored + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.server_address[1]}/test/input/big" + assert len(server.data_cache) == 0 + status, _, body = get(url) # first read populates the cache + assert status == 200 and body == big + assert len(server.data_cache) > 0 # the multi-chunk file cached some chunks + # a second (ranged) read hits the cache and must return identical bytes + status, _, body = get(url, headers={"Range": "bytes=0-99"}) + assert status == 206 and body == big[:100] + finally: + server.shutdown() + server.server_close() + thread.join(timeout=10) + + def test_webdav_conditional_get(archivers, request): archiver = request.getfixturevalue(archivers) _create_archive(archiver) diff --git a/src/borg/webdav.py b/src/borg/webdav.py index d078e69fff..f1931562a5 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -13,6 +13,7 @@ import html import mimetypes +import os import re import stat import threading @@ -27,6 +28,7 @@ from .archive import Archive from .constants import * # NOQA from .helpers import bin_to_hex, remove_surrogates +from .helpers.lrucache import LRUCache from .logger import create_logger logger = create_logger(__name__) @@ -435,6 +437,7 @@ class WebDAVHandler(BaseHTTPRequestHandler): # set on the handler class by make_server(): vfs = None repo_lock = None + data_cache = None # LRUCache: chunk id -> decrypted chunk data, shared across requests def version_string(self): # the base class would append sys_version, giving a trailing space if it is empty. @@ -696,15 +699,13 @@ def _send_file(self, name, node, pipeline, head): selected.append(entry) pos += entry.size remaining = end - start + 1 - chunk_iter = pipeline.fetch_many(selected, ro_type=ROBJ_FILE_STREAM, replacement_chunk=False) - while remaining > 0: - # serialize repository access, but write to the client outside the lock, - # so one slow client cannot block other requests for the whole download. + for entry in selected: + if remaining <= 0: + break + # serialize repository and data cache access, but write to the client outside + # the lock, so one slow client cannot block other requests for the whole download. with self.repo_lock: - try: - data = next(chunk_iter) - except StopIteration: - break + data = self._get_chunk(pipeline, entry) if data is None: # chunk missing in repository - never serve silently corrupted data: # abort the connection, the client sees a short read (Content-Length mismatch). @@ -721,6 +722,32 @@ def _send_file(self, name, node, pipeline, head): self.wfile.write(data) remaining -= len(data) + def _get_chunk(self, pipeline, entry): + """Return the decrypted data of chunk *entry*, using the shared data cache. + + Returns None if the chunk is missing in the repository. Must be called while + holding repo_lock (it guards both the repository and the data cache, and the + data cache's LRUCache is not thread-safe). The cache is keyed by chunk id + (a content hash), so it is valid across archives and requests. Reads of a big + file over a mounted file system tend to come as many small sequential range + requests hitting the same chunk - the cache avoids re-fetching and re-decrypting + it every time. + + This complements, but does not duplicate, the repository's own pack cache: + fetch_many() -> Repository.get_many() caches whole packs (raw, still encrypted + and compressed on-disk bytes), whereas we cache the *decrypted+decompressed* + chunk data, i.e. the output of the per-chunk RepoObj.parse() that runs on every + fetch even on a pack cache hit. So the two layers save different work (repo I/O + vs. per-chunk decrypt+decompress). + """ + try: + return self.data_cache[entry.id] + except KeyError: + data = next(pipeline.fetch_many([entry], ro_type=ROBJ_FILE_STREAM, replacement_chunk=False)) + if data is not None: + self.data_cache[entry.id] = data + return data + @staticmethod def _content_disposition(name): # File names from an archive can contain any byte except NUL and "/", including @@ -736,6 +763,13 @@ def _content_disposition(name): return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{encoded}" +def data_cache_capacity(): + # number of decrypted chunks to keep cached; same knob as borg mount uses. + # default: number of CPUs (a small, non-zero value), see also borg mount --help. + capacity = int(os.environ.get("BORG_MOUNT_DATA_CACHE_ENTRIES", os.cpu_count() or 1)) + return max(1, capacity) + + def make_server(manifest, args, bind="127.0.0.1", port=8000): """Create a ThreadingHTTPServer serving the archives selected by *args*. @@ -744,9 +778,13 @@ def make_server(manifest, args, bind="127.0.0.1", port=8000): """ repo_lock = threading.RLock() vfs = ArchiveVFS(manifest, args, repo_lock) + capacity = data_cache_capacity() + logger.debug("webdav: data cache capacity: %d chunks", capacity) + data_cache = LRUCache(capacity=capacity) - handler_class = type("WebDAVHandler", (WebDAVHandler,), dict(vfs=vfs, repo_lock=repo_lock)) + handler_class = type("WebDAVHandler", (WebDAVHandler,), dict(vfs=vfs, repo_lock=repo_lock, data_cache=data_cache)) server = ThreadingHTTPServer((bind, port), handler_class) server.daemon_threads = True server.repo_lock = repo_lock + server.data_cache = data_cache return server From ccbd51e03b1b0c8e18f3d531ac4d134cf0d46540 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 19:53:49 +0200 Subject: [PATCH 07/20] webdav: fix tests on Windows (illegal characters in file names) The tests created files with names that are illegal on Windows file systems (< and > in the "funny name", CR/LF and : in the header injection test), so the file creation raised OSError in the test setup and all webdav tests failed on the windows CI. Guard the creation of and the assertions about such files with is_win32, and skip the header injection test on Windows (it fundamentally needs a CR/LF file name, which Windows cannot create). The escaping and header injection code is platform independent and stays fully covered on the Linux and macOS CI. The other webdav tests (download, ranges, PROPFIND, conditional GET, data cache, ...) now run on Windows, too. Co-Authored-By: Claude Fable 5 --- .../testsuite/archiver/webdav_cmd_test.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 05cf06bc5a..a7f32cec0c 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -16,6 +16,7 @@ from ...constants import * # NOQA from ...manifest import Manifest +from ...platform import is_win32 from ...repository import Repository from ...webdav import make_server from .. import are_symlinks_supported, are_hardlinks_supported @@ -23,13 +24,17 @@ pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local") # NOQA +# a file name that exercises HTML escaping and URL encoding. It contains characters +# (< > and, for the injection test below, CR/LF) that are illegal in Windows file +# names, so files with such names can only be created (and thus tested) on non-Windows. FUNNY_NAME = "a &c ä.txt" def _create_archive(archiver): create_regular_file(archiver.input_path, "file1", contents=b"data1") create_regular_file(archiver.input_path, "subdir/file2", contents=b"subdir data") - create_regular_file(archiver.input_path, FUNNY_NAME, contents=b"funny data") + if not is_win32: + create_regular_file(archiver.input_path, FUNNY_NAME, contents=b"funny data") big = os.urandom(5 * 1024 * 1024) # big enough for multiple chunks with default chunker params create_regular_file(archiver.input_path, "big", contents=big) if are_symlinks_supported(): @@ -115,8 +120,9 @@ def test_webdav_browse(archivers, request): # the heading is a breadcrumb: every path segment links to its directory assert '

test/input/

' in page # funny file name is html-escaped in text and percent-encoded in the link - assert "a <b>&c ä.txt" in page - assert 'href="a%20%3Cb%3E%26c%20%C3%A4.txt"' in page + if not is_win32: + assert "a <b>&c ä.txt" in page + assert 'href="a%20%3Cb%3E%26c%20%C3%A4.txt"' in page if are_symlinks_supported(): assert "link1 -> somewhere/else" in page assert 'href="link1"' not in page # symlinks are not downloadable @@ -137,11 +143,12 @@ def test_webdav_download(archivers, request): assert status == 200 assert body == big assert headers["Content-Length"] == str(len(big)) - # a file in a subdirectory, with a percent-encoded url - status, headers, body = get(base_url + "/test/input/a%20%3Cb%3E%26c%20%C3%A4.txt") - assert status == 200 - assert body == b"funny data" - assert headers["Content-Type"].startswith("text/plain") + # a file with a percent-encoded url (its name has html/url-special characters) + if not is_win32: + status, headers, body = get(base_url + "/test/input/a%20%3Cb%3E%26c%20%C3%A4.txt") + assert status == 200 + assert body == b"funny data" + assert headers["Content-Type"].startswith("text/plain") @pytest.mark.skipif(not are_hardlinks_supported(), reason="hardlinks not supported") @@ -211,7 +218,8 @@ def test_webdav_propfind(archivers, request): assert "/test/input/" in hrefs assert "/test/input/subdir/" in hrefs assert "/test/input/file1" in hrefs - assert "/test/input/a%20%3Cb%3E%26c%20%C3%A4.txt" in hrefs + if not is_win32: + assert "/test/input/a%20%3Cb%3E%26c%20%C3%A4.txt" in hrefs assert prop_of(hrefs["/test/input/subdir/"], "{DAV:}resourcetype").find("{DAV:}collection") is not None # symlinks are not exposed via WebDAV assert not any("link1" in href for href in hrefs) @@ -299,6 +307,7 @@ def test_webdav_ranges(archivers, request): assert exc_info.value.headers["Content-Range"] == f"bytes */{size}" +@pytest.mark.skipif(is_win32, reason="cannot create files with CR/LF in the name on Windows") def test_webdav_header_injection(archivers, request): # File/directory names from an archive may contain CR/LF - names like # "x\r\nEvil-Header: 1" must not be able to inject headers into responses From b12edb00ce63a3bd72a054868a079240ef5bc437 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 20:57:01 +0200 Subject: [PATCH 08/20] webdav: silence bandit B405/B314 for the (guarded) XML parse The CI "security" job runs bandit, which flags the xml.etree import (B405) and the ET.fromstring() call (B314) as parsing untrusted XML. The parse is guarded: reject_dtd() rejects any DTD before parsing, so no custom entities can be declared and no entity expansion is possible, and the request body is size-limited. Mark both lines with # nosec (with a justification), the same mechanism borg already uses elsewhere. Co-Authored-By: Claude Fable 5 --- src/borg/webdav.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/borg/webdav.py b/src/borg/webdav.py index f1931562a5..ac108e0cf5 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -21,7 +21,7 @@ from email.utils import formatdate from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import quote, unquote_to_bytes -from xml.etree import ElementTree as ET +from xml.etree import ElementTree as ET # nosec B405 # DTDs rejected before parsing, see reject_dtd() from xml.parsers import expat from . import __version__ @@ -260,7 +260,7 @@ def parse_propfind(body): # DTDs are rejected above, so no internal entities can be declared and no # entity expansion ("billion laughs" / "XML bomb") is possible - additionally, # the request body is limited to 1 MiB by _read_body(). - root = ET.fromstring(body) # codeql[py/xml-bomb]: DTDs (thus custom entities) rejected by reject_dtd() + root = ET.fromstring(body) # nosec B314 # codeql[py/xml-bomb]: DTDs rejected by reject_dtd() above if root.tag != "{DAV:}propfind": raise ValueError("root element is not DAV: propfind") if root.find("{DAV:}propname") is not None: From 4959c06f9b518b3f55953d78b6026250709d1e05 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 21:20:24 +0200 Subject: [PATCH 09/20] webdav: daemonize by default, like borg mount Add a -f/--foreground flag; without it, the command daemonizes and runs in the background, matching borg mount. Bind the socket in the foreground first (so bind errors like "port in use" are reported before forking, and the listening fd survives the fork), then migrate the repository lock to the forked process. The LockRefresher and server threads are started after the fork, since threads do not survive fork(). The daemon stops on SIGTERM (the usual way to stop a daemon) or SIGINT, shutting down cleanly and releasing the repository lock. Daemonizing needs os.fork(), which does not exist on Windows; there the command stays in the foreground (borg webdav is meant to be usable on Windows, unlike borg mount). Co-Authored-By: Claude Opus 4.8 --- docs/man/borg-webdav.1 | 9 ++++++- docs/usage/webdav.rst.inc | 11 +++++++-- src/borg/archiver/webdav_cmd.py | 44 ++++++++++++++++++++++++++++----- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/docs/man/borg-webdav.1 b/docs/man/borg-webdav.1 index 119dcc99b4..ce2bd8025e 100644 --- a/docs/man/borg-webdav.1 +++ b/docs/man/borg-webdav.1 @@ -97,13 +97,20 @@ over. As for \fBborg mount\fP, the \fBBORG_MOUNT_DATA_CACHE_ENTRIES\fP environme variable sets the number of cached chunks (default: number of CPUs); additional memory usage can be up to the chunk size times this number. .sp -The command runs in the foreground until it is interrupted with Ctrl\-C. +Unless the \fB\-\-foreground\fP option is given, the command daemonizes and runs +in the background until it is stopped by sending it a signal (e.g. \fBkill\fP +sends SIGTERM), which shuts the server down and releases the repository lock. +In the foreground, ^C / SIGINT stops it. Daemonizing is not available on +Windows, so the command always stays in the foreground there. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. .SS options .INDENT 0.0 .TP +.B \-f\fP,\fB \-\-foreground +stay in foreground, do not daemonize +.TP .BI \-\-port \ PORT TCP port to listen on (on localhost); default: 8000 .UNINDENT diff --git a/docs/usage/webdav.rst.inc b/docs/usage/webdav.rst.inc index 4c717473d7..c287befe4d 100644 --- a/docs/usage/webdav.rst.inc +++ b/docs/usage/webdav.rst.inc @@ -15,6 +15,8 @@ borg webdav +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ | **options** | +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ + | | ``-f``, ``--foreground`` | stay in foreground, do not daemonize | + +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ | | ``--port PORT`` | TCP port to listen on (on localhost); default: 8000 | +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ | .. class:: borg-common-opt-ref | @@ -53,7 +55,8 @@ borg webdav options - --port PORT TCP port to listen on (on localhost); default: 8000 + -f, --foreground stay in foreground, do not daemonize + --port PORT TCP port to listen on (on localhost); default: 8000 :ref:`common_options` @@ -121,4 +124,8 @@ over. As for ``borg mount``, the ``BORG_MOUNT_DATA_CACHE_ENTRIES`` environment variable sets the number of cached chunks (default: number of CPUs); additional memory usage can be up to the chunk size times this number. -The command runs in the foreground until it is interrupted with Ctrl-C. \ No newline at end of file +Unless the ``--foreground`` option is given, the command daemonizes and runs +in the background until it is stopped by sending it a signal (e.g. ``kill`` +sends SIGTERM), which shuts the server down and releases the repository lock. +In the foreground, ^C / SIGINT stops it. Daemonizing is not available on +Windows, so the command always stays in the foreground there. \ No newline at end of file diff --git a/src/borg/archiver/webdav_cmd.py b/src/borg/archiver/webdav_cmd.py index b27c7d127e..c9050c0799 100644 --- a/src/borg/archiver/webdav_cmd.py +++ b/src/borg/archiver/webdav_cmd.py @@ -1,9 +1,10 @@ +import os import threading import time from ._common import with_repository, Highlander from ..constants import * # NOQA -from ..helpers import sig_int +from ..helpers import sig_int, daemonizing, signal_handler from ..helpers.argparsing import ArgumentParser from ..manifest import Manifest @@ -19,20 +20,44 @@ def do_webdav(self, args, repository, manifest): from ..webdav import make_server from ..storelocking import LockRefresher + # bind the socket now (in the foreground), so a bind error (e.g. port in use) is + # reported here instead of in a detached background process. The listening socket + # survives the daemonizing fork below. server = make_server(manifest, args, port=args.port) host, port = server.server_address[:2] - print(f"Serving selected archives read-only on http://{host}:{port}/ - press Ctrl-C to stop.") + url = f"http://{host}:{port}/" + + # daemonizing needs os.fork(), which does not exist on Windows - stay in the + # foreground there (borg webdav is meant to be usable on Windows, unlike borg mount). + daemonize = not args.foreground and hasattr(os, "fork") + if not args.foreground and not daemonize: + logger.info("Daemonizing is not supported on this platform, staying in the foreground.") + + if daemonize: + print(f"Serving selected archives read-only on {url} in the background.") + with daemonizing(show_rc=getattr(args, "show_rc", False)) as (old_id, new_id): + # we forked, so the process holding the repository lock changed: migrate it. + manifest.repository.migrate_lock(old_id, new_id) + else: + print(f"Serving selected archives read-only on {url} - press Ctrl-C to stop.") + + # From here on we run in the background (grandchild) process if daemonized, or in the + # same process otherwise. Threads must be started here, after the possible fork above, + # because threads do not survive fork(). # keep the repository lock of an idle server alive, so it is not killed as stale (see #9872). lock_refreshing_thread = LockRefresher(manifest.repository.info, sleep_interval=60, lock=server.repo_lock) lock_refreshing_thread.start() # the first SIGINT only sets the sig_int flag (see SigIntManager), so serve in a # thread and poll the flag here, to shut down cleanly on the first Ctrl-C already. + # SIGTERM (the usual way to stop a daemon) also stops us, so the repo lock is released. server_thread = threading.Thread(target=server.serve_forever, daemon=True) server_thread.start() + terminated = threading.Event() try: - while not sig_int: - time.sleep(0.1) - print("Shutting down...") + with signal_handler("SIGTERM", lambda sig_no, stack: terminated.set()): + while not sig_int and not terminated.is_set(): + time.sleep(0.1) + logger.info("webdav: shutting down.") finally: lock_refreshing_thread.terminate() server.shutdown() @@ -92,11 +117,18 @@ def build_parser_webdav(self, subparsers, common_parser, mid_common_parser): variable sets the number of cached chunks (default: number of CPUs); additional memory usage can be up to the chunk size times this number. - The command runs in the foreground until it is interrupted with Ctrl-C. + Unless the ``--foreground`` option is given, the command daemonizes and runs + in the background until it is stopped by sending it a signal (e.g. ``kill`` + sends SIGTERM), which shuts the server down and releases the repository lock. + In the foreground, ^C / SIGINT stops it. Daemonizing is not available on + Windows, so the command always stays in the foreground there. """ ) subparser = ArgumentParser(parents=[common_parser], description=self.do_webdav.__doc__, epilog=webdav_epilog) subparsers.add_subcommand("webdav", subparser, help="serve archive contents via WebDAV / HTTP") + subparser.add_argument( + "-f", "--foreground", dest="foreground", action="store_true", help="stay in foreground, do not daemonize" + ) subparser.add_argument( "--port", metavar="PORT", From bc4b2aee938e7f83d2899a858c9653dd53193130 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 21:28:14 +0200 Subject: [PATCH 10/20] webdav: test non-ASCII file name percent-encoding on all platforms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FUNNY_NAME test file contains "<>" (and CR/LF for the injection test), which are illegal in Windows file names, so it is skipped on Windows - which left the Windows CI job exercising no non-ASCII file name at all. Add UNICODE_NAME ("grüße.txt"), which is legal on every platform, and assert its percent-encoding in the browser listing link, the download URL round-trip and the PROPFIND href, unconditionally. Co-Authored-By: Claude Opus 4.8 --- src/borg/testsuite/archiver/webdav_cmd_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index a7f32cec0c..094d951503 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -29,10 +29,17 @@ # names, so files with such names can only be created (and thus tested) on non-Windows. FUNNY_NAME = "a &c ä.txt" +# a non-ASCII file name that is legal on every platform (including Windows), so the +# percent-encoding of non-ASCII names is exercised everywhere - "grüße.txt" url-encodes +# to "gr%C3%BC%C3%9Fe.txt" (ü -> %C3%BC, ß -> %C3%9F). +UNICODE_NAME = "grüße.txt" +UNICODE_NAME_ENC = "gr%C3%BC%C3%9Fe.txt" + def _create_archive(archiver): create_regular_file(archiver.input_path, "file1", contents=b"data1") create_regular_file(archiver.input_path, "subdir/file2", contents=b"subdir data") + create_regular_file(archiver.input_path, UNICODE_NAME, contents=b"unicode data") if not is_win32: create_regular_file(archiver.input_path, FUNNY_NAME, contents=b"funny data") big = os.urandom(5 * 1024 * 1024) # big enough for multiple chunks with default chunker params @@ -119,6 +126,8 @@ def test_webdav_browse(archivers, request): assert ">5.242.880<" in page # the heading is a breadcrumb: every path segment links to its directory assert '

test/input/

' in page + # a non-ASCII (but everywhere-legal) name is percent-encoded in the link, shown as-is in text + assert f'{UNICODE_NAME}' in page # funny file name is html-escaped in text and percent-encoded in the link if not is_win32: assert "a <b>&c ä.txt" in page @@ -143,6 +152,10 @@ def test_webdav_download(archivers, request): assert status == 200 assert body == big assert headers["Content-Length"] == str(len(big)) + # a non-ASCII name (legal on all platforms) round-trips via its percent-encoded url + status, _, body = get(f"{base_url}/test/input/{UNICODE_NAME_ENC}") + assert status == 200 + assert body == b"unicode data" # a file with a percent-encoded url (its name has html/url-special characters) if not is_win32: status, headers, body = get(base_url + "/test/input/a%20%3Cb%3E%26c%20%C3%A4.txt") @@ -218,6 +231,8 @@ def test_webdav_propfind(archivers, request): assert "/test/input/" in hrefs assert "/test/input/subdir/" in hrefs assert "/test/input/file1" in hrefs + # a non-ASCII name is percent-encoded in the PROPFIND href on all platforms + assert f"/test/input/{UNICODE_NAME_ENC}" in hrefs if not is_win32: assert "/test/input/a%20%3Cb%3E%26c%20%C3%A4.txt" in hrefs assert prop_of(hrefs["/test/input/subdir/"], "{DAV:}resourcetype").find("{DAV:}collection") is not None From 7c80afccd69b671b1a58adfdb075b045c0dfcdf6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 21:32:51 +0200 Subject: [PATCH 11/20] webdav: parse PROPFIND with expat, drop xml.etree.fromstring Parse the PROPFIND request body with expat directly instead of xml.etree.ElementTree.fromstring(). This keeps the same, encoding-proof DTD rejection (expat's StartDoctypeDeclHandler fires before any entity can be declared, so entity expansion / "billion laughs" is impossible), but removes the xml.etree parsing sink entirely - the CodeQL py/xml-bomb query only recognises defusedxml as safe and cannot see the DTD rejection, so it kept flagging the fromstring() call even though the code was not vulnerable. xml.etree is still imported, but now only to *build* the response XML (which is not an XML-bomb sink), so its bandit B405 nosec stays and the B314 nosec on fromstring is gone. No new dependency. Behaviour is unchanged: valid allprop/propname/prop bodies parse identically and the existing UTF-8/UTF-16 bomb and garbage-body tests still get a 400. Co-Authored-By: Claude Opus 4.8 --- src/borg/webdav.py | 86 +++++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/src/borg/webdav.py b/src/borg/webdav.py index ac108e0cf5..a6c65e06fd 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -21,7 +21,7 @@ from email.utils import formatdate from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import quote, unquote_to_bytes -from xml.etree import ElementTree as ET # nosec B405 # DTDs rejected before parsing, see reject_dtd() +from xml.etree import ElementTree as ET # nosec B405 # only used to *build* response XML, never to parse input from xml.parsers import expat from . import __version__ @@ -225,49 +225,65 @@ def iso8601(mtime_ns): return datetime.fromtimestamp(mtime_ns / 1e9, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") -def reject_dtd(body): - """Raise ValueError if *body* contains an XML DTD (). +def parse_propfind(body): + """Parse a PROPFIND request body. - A DTD is the only way to declare custom internal entities, so rejecting it - rules out entity expansion attacks ("billion laughs"). We use expat's own - doctype callback rather than a substring check on the raw bytes, so this is - independent of the document encoding (a DTD in e.g. UTF-16 does not contain - the ASCII bytes "local"; using "}" as the + # separator and prefixing "{" yields the "{uri}local" notation used elsewhere. + parser = expat.ParserCreate(namespace_separator="}") + stack = [] # open elements, as "{uri}local" + saw_propname = [False] + saw_prop = [False] + prop_depth = [None] # nesting depth of the element, once seen + props = [] # direct children of (the requested property names) def forbid_dtd(name, sysid, pubid, has_internal_subset): raise ValueError("DTD in request body rejected") + def start_element(name, attrs): + tag = "{" + name if "}" in name else name + stack.append(tag) + if len(stack) == 1 and tag != "{DAV:}propfind": + raise ValueError("root element is not DAV: propfind") + if len(stack) == 2 and tag == "{DAV:}propname": + saw_propname[0] = True + elif len(stack) == 2 and tag == "{DAV:}prop": + saw_prop[0] = True + prop_depth[0] = len(stack) + elif prop_depth[0] is not None and len(stack) == prop_depth[0] + 1: + props.append(tag) # a requested property name + parser.StartDoctypeDeclHandler = forbid_dtd + parser.StartElementHandler = start_element + parser.EndElementHandler = lambda name: stack.pop() try: - parser.Parse(body if isinstance(body, bytes) else body.encode("utf-8"), True) - except expat.ExpatError: - pass # malformed XML: the real parse below raises the canonical ParseError - + parser.Parse(body, True) + except expat.ExpatError as e: + raise ValueError(f"malformed PROPFIND body: {e}") -def parse_propfind(body): - """Parse a PROPFIND request body. - - Returns a (mode, props) tuple: ("allprop", None), ("propname", None) or - ("prop", [tag, ...]). Raises ValueError or ET.ParseError for bodies we do - not understand. - """ - if not body.strip(): - return "allprop", None # RFC 4918: an empty body means allprop - reject_dtd(body) - # DTDs are rejected above, so no internal entities can be declared and no - # entity expansion ("billion laughs" / "XML bomb") is possible - additionally, - # the request body is limited to 1 MiB by _read_body(). - root = ET.fromstring(body) # nosec B314 # codeql[py/xml-bomb]: DTDs rejected by reject_dtd() above - if root.tag != "{DAV:}propfind": - raise ValueError("root element is not DAV: propfind") - if root.find("{DAV:}propname") is not None: + if saw_propname[0]: return "propname", None - prop = root.find("{DAV:}prop") - if prop is not None: - return "prop", [child.tag for child in prop] + if saw_prop[0]: + return "prop", props return "allprop", None # allprop, maybe with an include element @@ -556,7 +572,7 @@ def _handle_propfind(self): return try: mode, requested = parse_propfind(body) - except (ET.ParseError, ValueError): + except ValueError: self.send_error(400, explain="invalid PROPFIND request body") return try: From 7aa93814bc0c955b2fe24f6d7621496e475347da Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 22:10:15 +0200 Subject: [PATCH 12/20] webdav: download any directory as a (PAX) tar archive Add a per-directory tar download: append ?tar (or ?tar=1) to a directory URL, or click the download icon shown next to the heading in the browser listings. Unlike a plain file download, the tar preserves POSIX metadata (owner, group, mode, sub-second timestamps, symlinks, special files, xattrs, ACLs), so it is the metadata-lossless way to restore a whole directory tree over the server. The tar is rooted at the requested directory (paths above it are stripped); ?tar on the archive root exports the whole archive. The tar size is not known in advance (PAX header sizes vary), so it is streamed with chunked transfer encoding. Repository access (item iteration and chunk fetching) is serialized under repo_lock, but each chunk is written to the client outside the lock: a slow client cannot block other requests, and the LockRefresher can keep the repository lock alive during a long download (holding the lock for the whole stream would starve it and let the lock go stale). A missing chunk leaves the chunked stream unterminated and closes the connection, so the client detects the truncation instead of receiving a corrupt archive. To avoid duplicating the borg-item -> tar mapping, item_to_tarinfo() and item_to_paxheaders() are factored out of TarMixIn._export_tar to module level in tar_cmds.py (behaviour-preserving; item_to_tarinfo now returns needs_content and the caller builds the content stream). webdav writes the tar blocks itself (TarInfo.tobuf for the header, chunk data, 512-byte padding, end-of-archive marker) so it can control the per-chunk locking. Verified end-to-end with the system tar tool: modes, symlink targets and a multi-chunk file are preserved byte-identically. Co-Authored-By: Claude Opus 4.8 --- docs/man/borg-webdav.1 | 21 +- docs/usage/webdav.rst.inc | 21 +- src/borg/archiver/tar_cmds.py | 225 +++++++++--------- src/borg/archiver/webdav_cmd.py | 21 +- .../testsuite/archiver/webdav_cmd_test.py | 64 ++++- src/borg/webdav.py | 141 ++++++++++- 6 files changed, 360 insertions(+), 133 deletions(-) diff --git a/docs/man/borg-webdav.1 b/docs/man/borg-webdav.1 index ce2bd8025e..55845a85d0 100644 --- a/docs/man/borg-webdav.1 +++ b/docs/man/borg-webdav.1 @@ -69,17 +69,26 @@ to select fewer archives); below that, the archive contents can be browsed like a directory tree. The directory tree of an archive is built in memory when it is first entered, so expect some delay for big archives. .sp +Any directory can be downloaded as a tar archive by appending \fB?tar\fP to its +URL (the web browser listings show a download icon next to the heading for this). +Unlike a plain file +download, the tar preserves POSIX metadata (owner, group, mode, sub\-second +timestamps, symlinks, special files, xattrs, ACLs), so it is the metadata\- +lossless way to restore a whole directory tree over this server. It is a PAX +format tarball, streamed uncompressed. +.sp Notes: .INDENT 0.0 .IP \(bu 2 -Downloads do not preserve any POSIX metadata (owner, group, mode, -timestamps, xattrs, ACLs). Use \fBborg extract\fP or \fBborg export\-tar\fP -for full\-fidelity restores. +Plain (non\-tar) file downloads do not preserve any POSIX metadata (owner, +group, mode, timestamps, xattrs, ACLs). Use the \fB?tar\fP download above, or +\fBborg extract\fP / \fBborg export\-tar\fP, for full\-fidelity restores. .IP \(bu 2 Symbolic links and special files (devices, fifos, sockets) are shown -in the web browser listings, but are neither followed nor downloadable, -and they are not visible in WebDAV\-mounted directories (WebDAV has no -concept of them). +in the web browser listings, but are neither followed nor downloadable +individually, and they are not visible in WebDAV\-mounted directories +(WebDAV has no concept of them). They are, however, included in \fB?tar\fP +downloads. .IP \(bu 2 Damaged files (with chunks missing in the repository) cause the download connection to be aborted \- the server never silently serves diff --git a/docs/usage/webdav.rst.inc b/docs/usage/webdav.rst.inc index c287befe4d..d526ec4303 100644 --- a/docs/usage/webdav.rst.inc +++ b/docs/usage/webdav.rst.inc @@ -101,15 +101,24 @@ to select fewer archives); below that, the archive contents can be browsed like a directory tree. The directory tree of an archive is built in memory when it is first entered, so expect some delay for big archives. +Any directory can be downloaded as a tar archive by appending ``?tar`` to its +URL (the web browser listings show a download icon next to the heading for this). +Unlike a plain file +download, the tar preserves POSIX metadata (owner, group, mode, sub-second +timestamps, symlinks, special files, xattrs, ACLs), so it is the metadata- +lossless way to restore a whole directory tree over this server. It is a PAX +format tarball, streamed uncompressed. + Notes: -- Downloads do not preserve any POSIX metadata (owner, group, mode, - timestamps, xattrs, ACLs). Use ``borg extract`` or ``borg export-tar`` - for full-fidelity restores. +- Plain (non-tar) file downloads do not preserve any POSIX metadata (owner, + group, mode, timestamps, xattrs, ACLs). Use the ``?tar`` download above, or + ``borg extract`` / ``borg export-tar``, for full-fidelity restores. - Symbolic links and special files (devices, fifos, sockets) are shown - in the web browser listings, but are neither followed nor downloadable, - and they are not visible in WebDAV-mounted directories (WebDAV has no - concept of them). + in the web browser listings, but are neither followed nor downloadable + individually, and they are not visible in WebDAV-mounted directories + (WebDAV has no concept of them). They are, however, included in ``?tar`` + downloads. - Damaged files (with chunks missing in the repository) cause the download connection to be aborted - the server never silently serves corrupted file content. diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index 5c91ecb7c7..e3df5859e7 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -33,6 +33,119 @@ tarfile.TarFile.extraction_filter = staticmethod(tarfile.fully_trusted_filter) # type: ignore +def item_to_tarinfo(item, hlm, *, warning=None): + """Transform a Borg *item* into a tarfile.TarInfo object. + + Return a tuple (tarinfo, needs_content): *tarinfo* is None if the item cannot be + represented as a TarInfo (and should be skipped), else the TarInfo to write. + *needs_content* is True when the caller must append the item's file content (its + chunk data) after the header - this is the case for a regular file that is not a + tar hard link to an earlier one. + + *hlm* is a HardLinkManager(id_type=bytes, info_type=str) mapping an already-emitted + file's hlid to its (tar) path, so later hard links become tar LNKTYPE references. + *warning* is an optional callback(format, *args) for reporting unsupported types. + """ + tarinfo = tarfile.TarInfo() + tarinfo.name = item.path + tarinfo.mtime = item.mtime / 1e9 + tarinfo.mode = stat.S_IMODE(item.mode) + tarinfo.uid = item.get("uid", 0) + tarinfo.gid = item.get("gid", 0) + tarinfo.uname = item.get("user", "") + tarinfo.gname = item.get("group", "") + # The linkname in tar has 2 uses: + # for symlinks it means the destination, while for hard links it refers to the file. + # Since hard links in tar have a different type code (LNKTYPE) the format might + # support hardlinking arbitrary objects (including symlinks and directories), but + # whether implementations actually support that is a whole different question... + tarinfo.linkname = "" + + needs_content = False + modebits = stat.S_IFMT(item.mode) + if modebits == stat.S_IFREG: + tarinfo.type = tarfile.REGTYPE + if "hlid" in item: + linkname = hlm.retrieve(id=item.hlid) + if linkname is not None: + # the first hard link was already added to the archive, add a tar-hard-link reference to it. + tarinfo.type = tarfile.LNKTYPE + tarinfo.linkname = linkname + else: + tarinfo.size = item.get_size() + needs_content = True + hlm.remember(id=item.hlid, info=item.path) + else: + tarinfo.size = item.get_size() + needs_content = True + elif modebits == stat.S_IFDIR: + tarinfo.type = tarfile.DIRTYPE + elif modebits == stat.S_IFLNK: + tarinfo.type = tarfile.SYMTYPE + tarinfo.linkname = item.target + elif modebits == stat.S_IFBLK: + tarinfo.type = tarfile.BLKTYPE + tarinfo.devmajor = os.major(item.rdev) + tarinfo.devminor = os.minor(item.rdev) + elif modebits == stat.S_IFCHR: + tarinfo.type = tarfile.CHRTYPE + tarinfo.devmajor = os.major(item.rdev) + tarinfo.devminor = os.minor(item.rdev) + elif modebits == stat.S_IFIFO: + tarinfo.type = tarfile.FIFOTYPE + else: + if warning is not None: + warning("%s: unsupported file type %o for tar export", remove_surrogates(item.path), modebits) + return None, False + return tarinfo, needs_content + + +def item_to_paxheaders(format, item): + """Transform (parts of) a Borg *item* into a pax_headers dict.""" + # PAX format + # ---------- + # When using the PAX (POSIX) format, we can support some things that aren't possible + # with classic tar formats, including GNU tar, such as: + # - atime, ctime (DONE) + # - possibly Linux capabilities, security.* xattrs (TODO) + # - various additions supported by GNU tar in POSIX mode (TODO) + # + # BORG format + # ----------- + # This is based on PAX, but additionally adds BORG.* pax headers. + # Additionally to the standard tar / PAX metadata and data, it transfers + # ALL borg item metadata in a BORG specific way. + # + ph = {} + # note: for mtime this is a bit redundant as it is already done by tarfile module, + # but we just do it in our way to be consistent for sure. + for name in "atime", "ctime", "mtime": + if hasattr(item, name): + ns = getattr(item, name) + ph[name] = str(ns / 1e9) + if hasattr(item, "xattrs"): + for bkey, bvalue in item.xattrs.items(): + # we have bytes key and bytes value, but the tarfile code + # expects str key and str value. + key = SCHILY_XATTR + bkey.decode("utf-8", errors="surrogateescape") + value = bvalue.decode("utf-8", errors="surrogateescape") + ph[key] = value + # Add POSIX access and default ACL if present + acl_access = item.get("acl_access") + if acl_access is not None: + ph[SCHILY_ACL_ACCESS] = acl_access.decode("utf-8", errors="surrogateescape") + acl_default = item.get("acl_default") + if acl_default is not None: + ph[SCHILY_ACL_DEFAULT] = acl_default.decode("utf-8", errors="surrogateescape") + if format == "BORG": # BORG format additions + ph["BORG.item.version"] = "1" + # BORG.item.meta - just serialize all metadata we have: + meta_bin = msgpack.packb(item.as_dict()) + meta_text = base64.b64encode(meta_bin).decode() + ph["BORG.item.meta"] = meta_text + return ph + + def get_tar_filter(fname, decompress): # Note that filter is None if fname is '-'. if fname.endswith((".tar.gz", ".tgz")): @@ -121,125 +234,17 @@ def item_content_stream(item): else: return ChunkIteratorFileWrapper(chunk_iterator) - def item_to_tarinfo(item, original_path): - """ - Transform a Borg *item* into a tarfile.TarInfo object. - - Return a tuple (tarinfo, stream), where stream may be a file-like object that represents - the file contents, if any, and is None otherwise. When *tarinfo* is None, the *item* - cannot be represented as a TarInfo object and should be skipped. - """ - stream = None - tarinfo = tarfile.TarInfo() - tarinfo.name = item.path - tarinfo.mtime = item.mtime / 1e9 - tarinfo.mode = stat.S_IMODE(item.mode) - tarinfo.uid = item.get("uid", 0) - tarinfo.gid = item.get("gid", 0) - tarinfo.uname = item.get("user", "") - tarinfo.gname = item.get("group", "") - # The linkname in tar has 2 uses: - # for symlinks it means the destination, while for hard links it refers to the file. - # Since hard links in tar have a different type code (LNKTYPE) the format might - # support hardlinking arbitrary objects (including symlinks and directories), but - # whether implementations actually support that is a whole different question... - tarinfo.linkname = "" - - modebits = stat.S_IFMT(item.mode) - if modebits == stat.S_IFREG: - tarinfo.type = tarfile.REGTYPE - if "hlid" in item: - linkname = hlm.retrieve(id=item.hlid) - if linkname is not None: - # the first hard link was already added to the archive, add a tar-hard-link reference to it. - tarinfo.type = tarfile.LNKTYPE - tarinfo.linkname = linkname - else: - tarinfo.size = item.get_size() - stream = item_content_stream(item) - hlm.remember(id=item.hlid, info=item.path) - else: - tarinfo.size = item.get_size() - stream = item_content_stream(item) - elif modebits == stat.S_IFDIR: - tarinfo.type = tarfile.DIRTYPE - elif modebits == stat.S_IFLNK: - tarinfo.type = tarfile.SYMTYPE - tarinfo.linkname = item.target - elif modebits == stat.S_IFBLK: - tarinfo.type = tarfile.BLKTYPE - tarinfo.devmajor = os.major(item.rdev) - tarinfo.devminor = os.minor(item.rdev) - elif modebits == stat.S_IFCHR: - tarinfo.type = tarfile.CHRTYPE - tarinfo.devmajor = os.major(item.rdev) - tarinfo.devminor = os.minor(item.rdev) - elif modebits == stat.S_IFIFO: - tarinfo.type = tarfile.FIFOTYPE - else: - self.print_warning( - "%s: unsupported file type %o for tar export", remove_surrogates(item.path), modebits - ) - return None, stream - return tarinfo, stream - - def item_to_paxheaders(format, item): - """ - Transform (parts of) a Borg *item* into a pax_headers dict. - """ - # PAX format - # ---------- - # When using the PAX (POSIX) format, we can support some things that aren't possible - # with classic tar formats, including GNU tar, such as: - # - atime, ctime (DONE) - # - possibly Linux capabilities, security.* xattrs (TODO) - # - various additions supported by GNU tar in POSIX mode (TODO) - # - # BORG format - # ----------- - # This is based on PAX, but additionally adds BORG.* pax headers. - # Additionally to the standard tar / PAX metadata and data, it transfers - # ALL borg item metadata in a BORG specific way. - # - ph = {} - # note: for mtime this is a bit redundant as it is already done by tarfile module, - # but we just do it in our way to be consistent for sure. - for name in "atime", "ctime", "mtime": - if hasattr(item, name): - ns = getattr(item, name) - ph[name] = str(ns / 1e9) - if hasattr(item, "xattrs"): - for bkey, bvalue in item.xattrs.items(): - # we have bytes key and bytes value, but the tarfile code - # expects str key and str value. - key = SCHILY_XATTR + bkey.decode("utf-8", errors="surrogateescape") - value = bvalue.decode("utf-8", errors="surrogateescape") - ph[key] = value - # Add POSIX access and default ACL if present - acl_access = item.get("acl_access") - if acl_access is not None: - ph[SCHILY_ACL_ACCESS] = acl_access.decode("utf-8", errors="surrogateescape") - acl_default = item.get("acl_default") - if acl_default is not None: - ph[SCHILY_ACL_DEFAULT] = acl_default.decode("utf-8", errors="surrogateescape") - if format == "BORG": # BORG format additions - ph["BORG.item.version"] = "1" - # BORG.item.meta - just serialize all metadata we have: - meta_bin = msgpack.packb(item.as_dict()) - meta_text = base64.b64encode(meta_bin).decode() - ph["BORG.item.meta"] = meta_text - return ph - for item in archive.iter_items(filter): orig_path = item.path if strip_components: item.path = os.sep.join(orig_path.split(os.sep)[strip_components:]) - tarinfo, stream = item_to_tarinfo(item, orig_path) + tarinfo, needs_content = item_to_tarinfo(item, hlm, warning=self.print_warning) if tarinfo: if args.tar_format in ("BORG", "PAX"): tarinfo.pax_headers = item_to_paxheaders(args.tar_format, item) if output_list: logging.getLogger("borg.output.list").info(remove_surrogates(orig_path)) + stream = item_content_stream(item) if needs_content else None tar.addfile(tarinfo, stream) if pi: diff --git a/src/borg/archiver/webdav_cmd.py b/src/borg/archiver/webdav_cmd.py index c9050c0799..e77473b817 100644 --- a/src/borg/archiver/webdav_cmd.py +++ b/src/borg/archiver/webdav_cmd.py @@ -94,15 +94,24 @@ def build_parser_webdav(self, subparsers, common_parser, mid_common_parser): browsed like a directory tree. The directory tree of an archive is built in memory when it is first entered, so expect some delay for big archives. + Any directory can be downloaded as a tar archive by appending ``?tar`` to its + URL (the web browser listings show a download icon next to the heading for this). + Unlike a plain file + download, the tar preserves POSIX metadata (owner, group, mode, sub-second + timestamps, symlinks, special files, xattrs, ACLs), so it is the metadata- + lossless way to restore a whole directory tree over this server. It is a PAX + format tarball, streamed uncompressed. + Notes: - - Downloads do not preserve any POSIX metadata (owner, group, mode, - timestamps, xattrs, ACLs). Use ``borg extract`` or ``borg export-tar`` - for full-fidelity restores. + - Plain (non-tar) file downloads do not preserve any POSIX metadata (owner, + group, mode, timestamps, xattrs, ACLs). Use the ``?tar`` download above, or + ``borg extract`` / ``borg export-tar``, for full-fidelity restores. - Symbolic links and special files (devices, fifos, sockets) are shown - in the web browser listings, but are neither followed nor downloadable, - and they are not visible in WebDAV-mounted directories (WebDAV has no - concept of them). + in the web browser listings, but are neither followed nor downloadable + individually, and they are not visible in WebDAV-mounted directories + (WebDAV has no concept of them). They are, however, included in ``?tar`` + downloads. - Damaged files (with chunks missing in the repository) cause the download connection to be aborted - the server never silently serves corrupted file content. diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 094d951503..046bf8a7b8 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -3,7 +3,10 @@ # so these tests run without any optional dependency. import http.client +import io import os +import stat +import tarfile import threading import urllib.error import urllib.request @@ -124,8 +127,9 @@ def test_webdav_browse(archivers, request): assert 'subdir/' in page # precise sizes in bytes, with dots as thousands separators assert ">5.242.880<" in page - # the heading is a breadcrumb: every path segment links to its directory - assert '

test/input/

' in page + # the heading is a breadcrumb (every path segment links to its directory), followed + # by an icon link to download the directory as a tar archive. + assert '

test/input/' '{UNICODE_NAME}' in page # funny file name is html-escaped in text and percent-encoded in the link @@ -164,6 +168,62 @@ def test_webdav_download(archivers, request): assert headers["Content-Type"].startswith("text/plain") +def _get_tar(url): + """GET a ?tar download and return (headers, tarfile.TarFile opened on the body).""" + status, headers, body = get(url) + assert status == 200 + assert headers["Content-Type"] == "application/x-tar" + # the size is not known in advance, so the tar is streamed with chunked encoding + assert headers.get("Transfer-Encoding") == "chunked" + assert "Content-Length" not in headers + tar = tarfile.open(fileobj=io.BytesIO(body), mode="r") # raises if the tar is malformed/truncated + return headers, tar + + +def test_webdav_tar(archivers, request): + archiver = request.getfixturevalue(archivers) + big = _create_archive(archiver) + with webdav_server(archiver) as base_url: + headers, tar = _get_tar(base_url + "/test/input/?tar=1") + # the download is offered as a .tar attachment named after the directory + assert 'filename="input.tar"' in headers["Content-Disposition"] + members = {m.name: m for m in tar.getmembers()} + # the tree is rooted at the requested directory ("input"), with its full contents + assert tar.extractfile("input/file1").read() == b"data1" + assert tar.extractfile("input/big").read() == big # multi-chunk file, streamed + padded + assert tar.extractfile("input/subdir/file2").read() == b"subdir data" + assert tar.extractfile("input/" + UNICODE_NAME).read() == b"unicode data" + # unlike a plain file download, the tar preserves POSIX metadata: directories, + # sub-second mtime, owner/mode and - shown here - symlinks and hard links. + assert members["input/subdir"].isdir() + assert members["input/file1"].mode == stat.S_IMODE(os.stat(os.path.join(archiver.input_path, "file1")).st_mode) + if are_symlinks_supported(): + assert members["input/link1"].issym() + assert members["input/link1"].linkname == "somewhere/else" + if are_hardlinks_supported(): + # the hard link pair appears once as a regular file and once as a tar hard link, + # both resolving to the same content. + assert tar.extractfile("input/hardlink").read() == b"data1" + assert {members["input/file1"].type, members["input/hardlink"].type} == {tarfile.REGTYPE, tarfile.LNKTYPE} + + +def test_webdav_tar_subdir_and_head(archivers, request): + archiver = request.getfixturevalue(archivers) + _create_archive(archiver) + with webdav_server(archiver) as base_url: + # a tar of a nested directory is rooted at that directory (paths stripped above it) + headers, tar = _get_tar(base_url + "/test/input/subdir/?tar=1") + assert 'filename="subdir.tar"' in headers["Content-Disposition"] + names = tar.getnames() + assert "subdir" in names and "subdir/file2" in names + assert not any(n.startswith("input/") for n in names) # the "input/" prefix is stripped + # a HEAD request returns the tar headers but no body + status, headers, body = http_request(base_url + "/test/input/subdir/?tar=1", "HEAD") + assert status == 200 + assert headers["Content-Type"] == "application/x-tar" + assert body == b"" + + @pytest.mark.skipif(not are_hardlinks_supported(), reason="hardlinks not supported") def test_webdav_hardlinks(archivers, request): archiver = request.getfixturevalue(archivers) diff --git a/src/borg/webdav.py b/src/borg/webdav.py index a6c65e06fd..813c6bfd6a 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -16,18 +16,19 @@ import os import re import stat +import tarfile import threading from datetime import datetime, timezone from email.utils import formatdate from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from urllib.parse import quote, unquote_to_bytes +from urllib.parse import parse_qs, quote, unquote_to_bytes from xml.etree import ElementTree as ET # nosec B405 # only used to *build* response XML, never to parse input from xml.parsers import expat from . import __version__ from .archive import Archive from .constants import * # NOQA -from .helpers import bin_to_hex, remove_surrogates +from .helpers import bin_to_hex, remove_surrogates, HardLinkManager from .helpers.lrucache import LRUCache from .logger import create_logger @@ -374,6 +375,17 @@ def render_multistatus(resources, mode, requested): "" ) +# "download to tray" icon (inline SVG, color via currentColor), shown next to a +# directory heading to download that directory as a tar archive. +DOWNLOAD_ICON_SVG = ( + '' + '' + '' + '' + "" +) + PAGE_TEMPLATE = """\ @@ -392,6 +404,9 @@ def render_multistatus(resources, mode, requested): }} h1 {{ color: #22d045; font-size: 1.4em; }} h1 a {{ color: inherit; }} +h1 a.dl {{ margin-left: 0.5em; opacity: 0.75; }} +h1 a.dl:hover {{ opacity: 1; text-decoration: none; }} +h1 a.dl svg {{ width: 0.85em; height: 0.85em; vertical-align: -0.06em; }} a {{ color: #22d045; text-decoration: none; }} a:hover {{ text-decoration: underline; }} table {{ border-collapse: collapse; }} @@ -511,6 +526,11 @@ def _parse_segments(self): return None return segments, path.endswith("/") + def _wants_tar(self): + """True if the request asks for a tar download of a directory (``?tar`` or ``?tar=1``).""" + query = self.path.partition("?")[2] + return "tar" in parse_qs(query, keep_blank_values=True) + def _read_body(self): """Read a request body; returns None (after sending an error) if that is not possible.""" if self.headers.get("Transfer-Encoding"): @@ -547,6 +567,9 @@ def _handle_get_head(self, head): if not dir_syntax: self._redirect_to_dir(segments) return + if self._wants_tar(): + self._send_tar(segments, head) + return self._send_dir_listing(segments, node, head) elif stat.S_ISREG(node.mode): self._send_file(segments[-1], node, pipeline, head) @@ -643,7 +666,11 @@ def _send_archive_list(self, head): def _send_dir_listing(self, segments, node, head): title = "/".join(remove_surrogates(s) for s in segments) + "/" - heading = make_breadcrumbs(segments) + # heading = breadcrumb path + an icon to download this directory as a tar archive + heading = make_breadcrumbs(segments) + ( + '{DOWNLOAD_ICON_SVG}' + ) rows = [make_row("../", "..")] children = sorted(node.children.items(), key=lambda kv: (not kv[1].is_dir, kv[0])) for name, child in children: @@ -738,6 +765,114 @@ def _send_file(self, name, node, pipeline, head): self.wfile.write(data) remaining -= len(data) + def _send_chunked(self, data): + """Write one HTTP/1.1 chunked-transfer-encoding block; *data* must be non-empty.""" + self.wfile.write(b"%x\r\n" % len(data)) + self.wfile.write(data) + self.wfile.write(b"\r\n") + + def _tar_warning(self, fmt, *args): + logger.warning("webdav: " + fmt, *args) + + def _send_tar(self, segments, head): + """Stream the directory subtree at *segments* as a PAX tar archive. + + Unlike a plain file download, a tar preserves POSIX metadata (owner, group, + mode, sub-second timestamps, symlinks, special files, xattrs, ACLs), so it is + the metadata-lossless way to restore a whole directory over HTTP. The reused + item -> TarInfo conversion (borg export-tar) needs the full item metadata, so + we re-iterate the archive items here rather than using the lightweight VFS tree. + + The tar size is not known in advance (PAX header sizes vary), so we stream it + with chunked transfer encoding. Repository access (item iteration and chunk + fetching) is serialized under repo_lock, but each chunk is written to the client + outside the lock - so a slow client cannot block other requests, and the + LockRefresher can keep the repository lock alive during a long download. + """ + # lazy import: pulling in the archiver package at module import time would be + # heavy (all subcommands) and risks an import cycle; by the time we serve a + # request the package is fully imported anyway. + from .archiver.tar_cmds import item_to_tarinfo, item_to_paxheaders + + archive_name = segments[0] + prefix = "/".join(segments[1:]) # borg path of the requested dir ("" = whole archive) + # root the tar at the requested directory: strip everything above it from the paths. + parent = prefix.rsplit("/", 1)[0] if "/" in prefix else "" + strip = len(parent) + 1 if parent else 0 + + def want(item): + if not prefix: + return True + return item.path == prefix or item.path.startswith(prefix + "/") + + download_name = segments[-1] + ".tar" + self.send_response(200) + self.send_header("Content-Type", "application/x-tar") + # sanitized against header injection, see _content_disposition() + self.send_header( + "Content-Disposition", self._content_disposition(download_name) + ) # codeql[py/http-response-splitting] # noqa: E501 + self.send_header("Transfer-Encoding", "chunked") + self.send_header("X-Content-Type-Options", "nosniff") + self.end_headers() + if head: + return + + with self.repo_lock: + archive = Archive(self.vfs.manifest, self.vfs.archives[archive_name].id) + pipeline = archive.pipeline + item_iter = archive.iter_items(want) + + hlm = HardLinkManager(id_type=bytes, info_type=str) # hlid -> (stripped) path of the first link + complete = False + try: + while True: + with self.repo_lock: + try: + item = next(item_iter) + except StopIteration: + break + if strip: + item.path = item.path[strip:] + tarinfo, needs_content = item_to_tarinfo(item, hlm, warning=self._tar_warning) + if tarinfo is None: + continue # unsupported item type, skipped (with a warning) + tarinfo.pax_headers = item_to_paxheaders("PAX", item) + self._send_chunked(tarinfo.tobuf(tarfile.PAX_FORMAT, tarfile.ENCODING, "surrogateescape")) + if needs_content and not self._send_tar_content(item, tarinfo.size, pipeline): + return # a chunk was missing: leave the chunked stream unterminated (see finally) + # end-of-archive marker (two zero blocks), then terminate the chunked stream. + self._send_chunked(b"\0" * (tarfile.BLOCKSIZE * 2)) + self.wfile.write(b"0\r\n\r\n") + complete = True + finally: + if not complete: + # do not send the terminating 0-length chunk, so the client can tell the + # tar is truncated (never present a corrupt archive as if it were complete). + self.close_connection = True + + def _send_tar_content(self, item, size, pipeline): + """Stream *item*'s file content into the tar, padded to a 512-byte block boundary. + + Returns False (after logging) if a chunk is missing in the repository, so the + caller aborts the connection instead of emitting silently corrupted content. + """ + for entry in item.get("chunks") or []: + # fetch under the lock, write to the client outside it (see _send_tar). + with self.repo_lock: + data = next(pipeline.fetch_many([entry], ro_type=ROBJ_FILE_STREAM, replacement_chunk=False)) + if data is None: + logger.error( + "webdav: chunk missing while streaming tar member %s, aborting the connection.", + remove_surrogates(item.path), + ) + return False + self._send_chunked(data) + padding = (tarfile.BLOCKSIZE - size % tarfile.BLOCKSIZE) % tarfile.BLOCKSIZE + if padding: + self._send_chunked(b"\0" * padding) + return True + def _get_chunk(self, pipeline, entry): """Return the decrypted data of chunk *entry*, using the shared data cache. From 6bafdbe5d02f172e141251dc3a432da50694f6c0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 22:21:57 +0200 Subject: [PATCH 13/20] webdav: sanitize + suppress response-splitting on the tar Content-Disposition The tar download's file name is derived from the client-supplied URL path and flows into the Content-Disposition header, so CodeQL flags it as py/http-response-splitting - like the plain-download and redirect headers before it. The value is already sanitized: _content_disposition() strips non-printables (killing CR/LF) for the fallback name and percent-encodes the RFC 8187 name. Put the header on a single line so the # codeql[py/http-response-splitting] marker sits on the flagged line (it was on the wrapped closing-paren line before), matching the existing suppressions, and extend the header-injection test to a ?tar download of a directory whose name contains CR/LF. Co-Authored-By: Claude Opus 4.8 --- src/borg/testsuite/archiver/webdav_cmd_test.py | 6 ++++++ src/borg/webdav.py | 9 ++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 046bf8a7b8..076bf2e894 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -404,6 +404,12 @@ def test_webdav_header_injection(archivers, request): assert "Evil-Header" not in headers assert 'filename="inj__Evil-Header: 1"' in headers["Content-Disposition"] assert "%0D%0A" in headers["Content-Disposition"] # RFC 8187 encoded form + # a tar download of the evil directory: its name feeds Content-Disposition, which + # must be sanitized too (the directory name becomes the tar file name). + status, headers, _ = get(f"{base_url}/test/input/dir%0D%0AEvil-Header%3A%202/?tar=1") + assert status == 200 + assert "Evil-Header" not in headers + assert 'filename="dir__Evil-Header: 2.tar"' in headers["Content-Disposition"] # the directory redirect: Location must be percent-encoded, no header injected # (plain http.client here, so the 301 is not followed) conn = http.client.HTTPConnection(host_port) diff --git a/src/borg/webdav.py b/src/borg/webdav.py index 813c6bfd6a..fde7d801f9 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -805,13 +805,12 @@ def want(item): return True return item.path == prefix or item.path.startswith(prefix + "/") - download_name = segments[-1] + ".tar" + # download_name is derived from the client-supplied URL path, so sanitize it + # against header injection (CR/LF) just like a normal download, see there. + content_disposition = self._content_disposition(segments[-1] + ".tar") self.send_response(200) self.send_header("Content-Type", "application/x-tar") - # sanitized against header injection, see _content_disposition() - self.send_header( - "Content-Disposition", self._content_disposition(download_name) - ) # codeql[py/http-response-splitting] # noqa: E501 + self.send_header("Content-Disposition", content_disposition) # codeql[py/http-response-splitting] self.send_header("Transfer-Encoding", "chunked") self.send_header("X-Content-Type-Options", "nosniff") self.end_headers() From 56bf4f00fd1a346a9fb01d1113388ea39152a0ce Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 22:22:08 +0200 Subject: [PATCH 14/20] webdav: don't link the current directory in the breadcrumb heading In the breadcrumb path, only the parent segments need to be links (for navigating up); the last segment is the directory being viewed, so show it as plain text. Co-Authored-By: Claude Opus 4.8 --- src/borg/testsuite/archiver/webdav_cmd_test.py | 7 ++++--- src/borg/webdav.py | 9 ++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 076bf2e894..4aed2fa775 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -127,9 +127,10 @@ def test_webdav_browse(archivers, request): assert 'subdir/' in page # precise sizes in bytes, with dots as thousands separators assert ">5.242.880<" in page - # the heading is a breadcrumb (every path segment links to its directory), followed - # by an icon link to download the directory as a tar archive. - assert '

test/input/' 'test/input/{UNICODE_NAME}' in page # funny file name is html-escaped in text and percent-encoded in the link diff --git a/src/borg/webdav.py b/src/borg/webdav.py index fde7d801f9..c06406e337 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -442,12 +442,15 @@ def render_page(title, rows, heading=None): def make_breadcrumbs(segments): - """Build h1 HTML for a path: each segment linked to its directory, for quick navigation.""" + """Build h1 HTML for a path: each parent segment links to its directory (for quick + navigation); the last segment is the current directory and is shown as plain text.""" parts = [] href = "/" - for segment in segments: + last = len(segments) - 1 + for i, segment in enumerate(segments): href += encode_path(segment) + "/" - parts.append(f'{html.escape(remove_surrogates(segment))}') + text = html.escape(remove_surrogates(segment)) + parts.append(text if i == last else f'{text}') return "/".join(parts) + "/" From dffca4246451749f20c4b6818cd174cf470482d2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 22:52:55 +0200 Subject: [PATCH 15/20] webdav: test the command itself, special files and request limits Raise patch coverage by exercising code paths the existing tests missed: - test_webdav_command_serves_and_stops runs the actual `borg webdav` command in the foreground, checks it serves a request, stops it with SIGTERM and confirms the repository lock was released (a following borg command works). This covers do_webdav(), which nothing exercised before. - test_webdav_special_files (POSIX) backs up a fifo and checks it is listed but not downloadable, refused with 403 on GET, hidden from WebDAV, and included in a ?tar download as a fifo entry. - extend test_webdav_errors with a GET on a symlink (403) and an oversized PROPFIND body (413, rejected via Content-Length without reading the body). - extend the tar test with a ?tar download at the archive root (whole archive, no path prefix stripped). Co-Authored-By: Claude Opus 4.8 --- .../testsuite/archiver/webdav_cmd_test.py | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 4aed2fa775..9354c432c5 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -5,9 +5,12 @@ import http.client import io import os +import signal +import socket import stat import tarfile import threading +import time import urllib.error import urllib.request import xml.etree.ElementTree as ET @@ -223,6 +226,9 @@ def test_webdav_tar_subdir_and_head(archivers, request): assert status == 200 assert headers["Content-Type"] == "application/x-tar" assert body == b"" + # ?tar at the archive root exports the whole archive (no path prefix stripped) + _, tar = _get_tar(base_url + "/test/?tar=1") + assert "input/file1" in tar.getnames() @pytest.mark.skipif(not are_hardlinks_supported(), reason="hardlinks not supported") @@ -259,6 +265,23 @@ def test_webdav_errors(archivers, request): http_request(base_url + "/test/input/file1", method) assert exc_info.value.code == 405 assert "PROPFIND" in exc_info.value.headers["Allow"] + # a symlink is shown in listings but is not downloadable via GET + if are_symlinks_supported(): + with pytest.raises(urllib.error.HTTPError) as exc_info: + get(base_url + "/test/input/link1") + assert exc_info.value.code == 403 + # a PROPFIND body larger than 1 MiB is rejected: the server checks Content-Length + # and answers 413 without reading the oversized body. + conn = http.client.HTTPConnection(urlsplit(base_url).netloc) + try: + conn.putrequest("PROPFIND", "/test/input/file1", skip_accept_encoding=True) + conn.putheader("Depth", "0") + conn.putheader("Content-Length", str(1024 * 1024 + 1)) + conn.endheaders() + conn.send(b"x") # only a token byte; the header alone triggers the rejection + assert conn.getresponse().status == 413 + finally: + conn.close() def test_webdav_options(archivers, request): @@ -469,3 +492,68 @@ def test_webdav_conditional_get(archivers, request): # a non-matching etag serves the content normally status, _, body = get(url, headers={"If-None-Match": '"different"'}) assert status == 200 and body == b"data1" + + +@pytest.mark.skipif(is_win32 or not hasattr(os, "mkfifo"), reason="fifo (a special file) needs POSIX") +def test_webdav_special_files(archivers, request): + # A named pipe stands in for special files (devices, fifos, sockets): it is shown in + # browser listings but not downloadable and not exposed via WebDAV - yet a tar download + # (which preserves metadata) does include it. + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=b"data1") + os.mkfifo(os.path.join(archiver.input_path, "pipe")) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test", "input") + with webdav_server(archiver) as base_url: + # shown in the listing, but as plain text (not a download link) + status, _, body = get(base_url + "/test/input/") + assert status == 200 + page = body.decode("utf-8") + assert "pipe" in page + assert 'href="pipe"' not in page + # a GET on it is refused (it is not a regular file) + with pytest.raises(urllib.error.HTTPError) as exc_info: + get(base_url + "/test/input/pipe") + assert exc_info.value.code == 403 + # not exposed via WebDAV (the protocol has no concept of a fifo) + _, _, xml_body = propfind(base_url + "/test/input/", depth="1") + assert not any("pipe" in href for href in propfind_hrefs(xml_body)) + # but included in a tar download, as a fifo entry + _, tar = _get_tar(base_url + "/test/input/?tar=1") + assert tar.getmember("input/pipe").isfifo() + + +def _free_port(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.mark.skipif(is_win32, reason="uses a POSIX signal to stop the foreground server") +def test_webdav_command_serves_and_stops(archivers, request): + # exercise the actual `borg webdav` command (in the foreground): it must serve requests + # and, on SIGTERM, shut down cleanly and release the repository lock. + archiver = request.getfixturevalue(archivers) + _create_archive(archiver) + port = _free_port() + served = [] + + def client_then_stop(): + base = f"http://127.0.0.1:{port}/" + for _ in range(100): # wait until the server is up (do_webdav installs its handler by then) + try: + with urllib.request.urlopen(base) as response: + served.append(response.status) + break + except OSError: + time.sleep(0.1) + time.sleep(0.2) # small margin so the SIGTERM handler is surely installed + os.kill(os.getpid(), signal.SIGTERM) # ask the foreground server to stop + + stopper = threading.Thread(target=client_then_stop, daemon=True) + stopper.start() + cmd(archiver, "webdav", "--foreground", "--port", str(port)) # blocks until stopped + stopper.join(timeout=10) + assert served == [200] # the command actually served a request + # the repository lock was released on shutdown, so a following borg command still works + cmd(archiver, "repo-info") From ad4818df5fd7c9636156b768d4f9d010ee49aeed Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 23 Jul 2026 23:00:57 +0200 Subject: [PATCH 16/20] webdav: make header CR/LF sanitization explicit (response splitting) CodeQL keeps flagging the Content-Disposition and Location headers as py/http-response-splitting. The values are already newline-free (the Content-Disposition fallback replaces non-printables, the RFC 8187 name and the Location are percent-encoded), but that is not visible to the analyser, and borg's CodeQL runs as advanced setup which does not honour inline "# codeql[...]" suppression comments (the marker sat right on the flagged line). Route every header value that derives from a client-supplied path through a small strip_crlf() helper right at the sink. It is a no-op by construction, but makes the CR/LF safety explicit as an actual sanitizer instead of relying on a suppression comment. Drop the now-misleading "# codeql[...]" markers. Co-Authored-By: Claude Opus 4.8 --- src/borg/webdav.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/borg/webdav.py b/src/borg/webdav.py index c06406e337..49bc697d39 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -143,6 +143,16 @@ def resolve(self, segments): return node, pipeline +def strip_crlf(value): + """Remove CR and LF from an HTTP header value, so it can never split the response. + + The values we build are already newline-free (percent-encoded or non-printables + replaced), so this is defense in depth - and the explicit, recognizable form of the + CR/LF safety that we (and static analysers) rely on at the header sinks. + """ + return value.replace("\r", "").replace("\n", "") + + def encode_path(path): """Percent-encode a borg item path (str with surrogateescape) for use in a URL.""" return quote(path.encode("utf-8", "surrogateescape")) @@ -644,9 +654,10 @@ def _redirect_to_dir(self, segments): # Build the redirect target by percent-encoding the parsed path segments: # quote() only outputs URL-safe ASCII, so the Location value cannot contain # CR/LF or other header-splitting characters, no matter what the client sent. - location = "/" + "/".join(encode_path(s) for s in segments) + "/" + # The redundant strip_crlf() makes that guarantee explicit at the header sink. + location = strip_crlf("/" + "/".join(encode_path(s) for s in segments) + "/") self.send_response(301) - self.send_header("Location", location) # codeql[py/http-response-splitting] + self.send_header("Location", location) self.send_header("Content-Length", "0") self.end_headers() @@ -726,8 +737,8 @@ def _send_file(self, name, node, pipeline, head): self.send_header("ETag", etag) self.send_header("Accept-Ranges", "bytes") self.send_header("X-Content-Type-Options", "nosniff") - content_disposition = self._content_disposition(name) # sanitized, see there - self.send_header("Content-Disposition", content_disposition) # codeql[py/http-response-splitting] + content_disposition = self._content_disposition(name) # CR/LF-sanitized, see there + self.send_header("Content-Disposition", content_disposition) self.end_headers() if head or not node.chunks or node.size == 0: return @@ -813,7 +824,7 @@ def want(item): content_disposition = self._content_disposition(segments[-1] + ".tar") self.send_response(200) self.send_header("Content-Type", "application/x-tar") - self.send_header("Content-Disposition", content_disposition) # codeql[py/http-response-splitting] + self.send_header("Content-Disposition", content_disposition) self.send_header("Transfer-Encoding", "chunked") self.send_header("X-Content-Type-Options", "nosniff") self.end_headers() @@ -913,7 +924,8 @@ def _content_disposition(name): fallback = "".join(c if c.isprintable() else "_" for c in remove_surrogates(name)) fallback = fallback.encode("ascii", "replace").decode("ascii").replace('"', "'") encoded = quote(name.encode("utf-8", "surrogateescape")) - return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{encoded}" + result = f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{encoded}" + return strip_crlf(result) # no-op by construction, but makes the CR/LF safety explicit def data_cache_capacity(): From 6d0937dc5c1ad90095b48a7ab498e1fd3b169617 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 24 Jul 2026 00:32:06 +0200 Subject: [PATCH 17/20] webdav: document known WebDAV client issues Add a "Client notes and known issues" section to the webdav docs covering the common client-side quirks of a read-only WebDAV server: the Windows Explorer ~47 MiB download limit / WebClient service / Basic-auth-over-HTTP restriction, Finder writing (and harmlessly failing to write) .DS_Store/AppleDouble/.Trash, davfs2 needing use_locks 0, and the protocol-level notes (Depth: infinity refused, trailing-slash redirect, symlinks/special files and metadata only via the ?tar download). Co-Authored-By: Claude Opus 4.8 --- docs/usage/webdav.rst | 53 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docs/usage/webdav.rst b/docs/usage/webdav.rst index 07a8d845d8..d148232a74 100644 --- a/docs/usage/webdav.rst +++ b/docs/usage/webdav.rst @@ -11,3 +11,56 @@ Examples # Serve only one archive on a different port. $ borg webdav --port 8123 --match-archives my-archive + + +Client notes and known issues +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +WebDAV's rough edges are almost all on the client side, and a *read-only* server +(like this one) hits a specific subset of them. A plain web browser avoids most of +these - the notes below matter mainly when *mounting* the server as a file system. + +**Windows Explorer** (the "WebClient" / mini-redirector) + +- Downloads are limited to about 47 MiB by default (the ``FileSizeLimitInBytes`` + registry value). Bigger files fail when copied from a mapped drive - use a web + browser (including for the ``?tar`` directory download) or another WebDAV client. +- The ``WebClient`` service must be running for ``net use`` / "Map network drive" + to work; if it is stopped, mounting silently fails. +- Windows refuses HTTP Basic authentication over plain HTTP by default (only over + HTTPS). This does not matter now (the server has no authentication), but it is a + wall for any future networked, authenticated setup. +- Explorer is chatty (a PROPFIND per navigation, many short connections), so + browsing large directories over a mount can feel slow. + +**macOS Finder** (``mount_webdav`` / WebDAVFS) + +- Finder tries to write ``.DS_Store``, ``._*`` (AppleDouble) and ``.Trash`` files + into every folder it opens. On this read-only server those writes are rejected + (the file system is read-only); Finder tolerates it but may occasionally show a + spurious "operation could not be completed" dialog. This is harmless. +- WebDAVFS caches directory listings; a stale view usually clears on unmount and + remount. + +**Linux davfs2** + +- davfs2 uses WebDAV locking by default and will try to LOCK files it opens, which + a read-only server rejects. Set ``use_locks 0`` in ``davfs2.conf`` (or the + per-mount config) to avoid the failed lock attempts. davfs2 also caches whole + files in a local cache directory. +- The GNOME (gvfs) and KDE (KIO) DAV backends work without such tweaks, but are + also PROPFIND-heavy on large directories. + +**Protocol-level (any client)** + +- ``PROPFIND`` with ``Depth: infinity`` (a recursive enumeration that can be very + expensive) is refused with ``403``, as permitted by RFC 4918. Well-behaved + clients use ``Depth: 0`` or ``1``. +- Collections must be addressed with a trailing slash; a request for ``/dir`` is + redirected to ``/dir/``. +- Symbolic links and special files (devices, fifos, sockets) have no + representation in WebDAV, so they are not visible in a mounted file system. They + are shown in the web browser listings and are included in ``?tar`` downloads. +- WebDAV transfers file contents plus modification time and size, but no owner, + group, mode, xattrs or ACLs. Use the ``?tar`` directory download (or + ``borg extract`` / ``borg export-tar``) when you need a full-fidelity restore. From 5e762106dc98b9bbe85bb9c7fa5a1bca9da1dc63 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 24 Jul 2026 21:59:30 +0200 Subject: [PATCH 18/20] webdav: TCP_NODELAY and output buffering at the HTTP level Two standard HTTP-server tunings for a server that answers many small requests: - disable_nagle_algorithm (TCP_NODELAY): Nagle's algorithm interacting with the client's delayed ACKs can add ~40 ms to a small request/response over a real network. WebDAV clients issue lots of small PROPFIND/HEAD/GET requests, so this matters for mounted use (no effect on loopback, where the benchmark is flat). - wbufsize: buffer the response so the several small writes of one response (status line, headers, HTML rows, tar block framing) coalesce into few packets/syscalls instead of one send() each; writes larger than the buffer (file/tar chunk data) still pass straight through, and handle_one_request() flushes wfile after every request so nothing is delayed. HTTP/1.1 keep-alive (connection reuse) was already enabled. Behaviour is unchanged; all tests pass. Co-Authored-By: Claude Opus 4.8 --- src/borg/webdav.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/borg/webdav.py b/src/borg/webdav.py index 49bc697d39..53f78228db 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -474,7 +474,18 @@ def make_row(href, text, size="", mtime_ns=None): class WebDAVHandler(BaseHTTPRequestHandler): + # HTTP/1.1 keeps connections alive by default, so a client can reuse one connection + # for its many requests instead of reconnecting each time. protocol_version = "HTTP/1.1" + # Disable Nagle's algorithm (set TCP_NODELAY): its interaction with delayed ACKs can + # add ~40 ms to a small request/response, which really hurts WebDAV clients that issue + # lots of small PROPFIND/HEAD/GET requests. + disable_nagle_algorithm = True + # Buffer the response so the several small writes of one response (status line, headers, + # HTML rows, tar block framing) coalesce into few packets/syscalls; writes larger than + # the buffer (file/tar chunk data) still pass straight through. handle_one_request() + # flushes wfile after each request, so buffering does not delay the response. + wbufsize = 64 * 1024 server_version = f"borg-webdav/{__version__}" sys_version = "" # do not tell clients about the python version we use From 49835ebc1441aa1da88c1f3d6868c3ab2b1a0c65 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 24 Jul 2026 21:59:47 +0200 Subject: [PATCH 19/20] webdav: keep the query string when redirecting to add a trailing slash A GET on a directory URL without the trailing slash (e.g. /test/input?tar=1) redirects with a 301 to the slashed form. _redirect_to_dir rebuilt the Location from the path segments only, dropping the query string - so /test/input?tar=1 redirected to /test/input/ and the tar download turned into a plain listing. Preserve the query in the redirect target. The query is already percent-encoded by the client and still passes through strip_crlf(), so it cannot split the response. Co-Authored-By: Claude Opus 4.8 --- src/borg/testsuite/archiver/webdav_cmd_test.py | 4 ++++ src/borg/webdav.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 9354c432c5..25b6d196a8 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -229,6 +229,10 @@ def test_webdav_tar_subdir_and_head(archivers, request): # ?tar at the archive root exports the whole archive (no path prefix stripped) _, tar = _get_tar(base_url + "/test/?tar=1") assert "input/file1" in tar.getnames() + # a directory tar URL without the trailing slash redirects but keeps the query + # string, so the (urllib-followed) redirect still delivers the tar, not the listing + _, tar = _get_tar(base_url + "/test/input?tar=1") + assert "input/file1" in tar.getnames() @pytest.mark.skipif(not are_hardlinks_supported(), reason="hardlinks not supported") diff --git a/src/borg/webdav.py b/src/borg/webdav.py index 53f78228db..6cf8462393 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -663,10 +663,15 @@ def _propfind_resources(self, segments, depth): def _redirect_to_dir(self, segments): # Build the redirect target by percent-encoding the parsed path segments: - # quote() only outputs URL-safe ASCII, so the Location value cannot contain - # CR/LF or other header-splitting characters, no matter what the client sent. - # The redundant strip_crlf() makes that guarantee explicit at the header sink. - location = strip_crlf("/" + "/".join(encode_path(s) for s in segments) + "/") + # quote() only outputs URL-safe ASCII for the path. Preserve the query string + # (e.g. "?tar=1") so redirecting a directory URL that lacks the trailing slash + # does not drop it. The query is already percent-encoded by the client; the + # strip_crlf() below removes any CR/LF, so the Location cannot split the response. + location = "/" + "/".join(encode_path(s) for s in segments) + "/" + query = self.path.partition("?")[2] + if query: + location += "?" + query + location = strip_crlf(location) self.send_response(301) self.send_header("Location", location) self.send_header("Content-Length", "0") From 8f896b74c75767480088b0cd32db6b42c1aad4e9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 24 Jul 2026 22:05:09 +0200 Subject: [PATCH 20/20] webdav: abort when a non-empty item has no chunks list If an archive item has size > 0 but no chunks list (e.g. corrupted metadata), we already sent Content-Length > 0 (or, for a tar member, a header declaring that size), so returning without a body would leave the client waiting forever for bytes that never come (or produce a silently corrupt tar member). Detect this anomaly in both _send_file and _send_tar_content and abort the connection - the same "never serve silently corrupted data" handling used for a chunk that is missing in the repository. Empty files (size == 0, no chunks) and HEAD requests keep returning an empty body as before. Add a test that injects a non-empty, chunk-less node and checks the download is aborted (the client sees a short read, not a hang). Co-Authored-By: Claude Opus 4.8 --- .../testsuite/archiver/webdav_cmd_test.py | 34 +++++++++++++++++++ src/borg/webdav.py | 30 +++++++++++++--- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 25b6d196a8..aba6b54c2f 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -498,6 +498,40 @@ def test_webdav_conditional_get(archivers, request): assert status == 200 and body == b"data1" +def test_webdav_file_without_chunks(archivers, request): + # An anomalous item - size > 0 but no chunks list (e.g. corrupted metadata) - must not + # leave the client hanging after an advertised Content-Length: the server aborts the + # connection instead, so the client sees a short read rather than waiting forever. + archiver = request.getfixturevalue(archivers) + _create_archive(archiver) + args = SimpleNamespace( + sort_by="ts", match_archives=None, first=None, last=None, older=None, newer=None, oldest=None, newest=None + ) + repository = Repository(archiver.repository_path, exclusive=True) + with repository: + manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + server = make_server(manifest, args, port=0) + # corrupt the in-memory tree: pretend file1 is non-empty but has no chunks + node, _ = server.RequestHandlerClass.vfs.resolve(["test", "input", "file1"]) + node.size = 5 + node.chunks = None + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + conn = http.client.HTTPConnection("127.0.0.1", server.server_address[1]) + conn.request("GET", "/test/input/file1") + response = conn.getresponse() + assert response.status == 200 + assert response.getheader("Content-Length") == "5" # a body was promised... + with pytest.raises(http.client.IncompleteRead): + response.read() # ...but the connection is aborted with no body + conn.close() + finally: + server.shutdown() + server.server_close() + thread.join(timeout=10) + + @pytest.mark.skipif(is_win32 or not hasattr(os, "mkfifo"), reason="fifo (a special file) needs POSIX") def test_webdav_special_files(archivers, request): # A named pipe stands in for special files (devices, fifos, sockets): it is shown in diff --git a/src/borg/webdav.py b/src/borg/webdav.py index 6cf8462393..d97d940f14 100644 --- a/src/borg/webdav.py +++ b/src/borg/webdav.py @@ -756,7 +756,18 @@ def _send_file(self, name, node, pipeline, head): content_disposition = self._content_disposition(name) # CR/LF-sanitized, see there self.send_header("Content-Disposition", content_disposition) self.end_headers() - if head or not node.chunks or node.size == 0: + if head or node.size == 0: + return # no body to send (and Content-Length is 0 for an empty file) + if not node.chunks: + # anomaly: a non-empty file with no chunks list (e.g. corrupted metadata). We + # already sent Content-Length > 0, so abort the connection instead of leaving + # the client waiting forever for body bytes that will never arrive. + logger.error( + "webdav: file %s has size %d but no chunks, aborting the connection.", + remove_surrogates(name), + node.size, + ) + self.close_connection = True return # select only the chunks overlapping the requested range, so nothing else # gets fetched and decrypted (the chunk sizes are known in advance). @@ -883,10 +894,21 @@ def want(item): def _send_tar_content(self, item, size, pipeline): """Stream *item*'s file content into the tar, padded to a 512-byte block boundary. - Returns False (after logging) if a chunk is missing in the repository, so the - caller aborts the connection instead of emitting silently corrupted content. + Returns False (after logging) if the content cannot be produced in full (a chunk + missing in the repository, or a non-empty item with no chunks list), so the caller + aborts the connection instead of emitting a silently corrupted (short) tar member. """ - for entry in item.get("chunks") or []: + chunks = item.get("chunks") or [] + if size > 0 and not chunks: + # anomaly: a non-empty item with no chunks list (e.g. corrupted metadata). The + # tar header already declared size > 0, so we cannot emit a valid member. + logger.error( + "webdav: tar member %s has size %d but no chunks, aborting the connection.", + remove_surrogates(item.path), + size, + ) + return False + for entry in chunks: # fetch under the lock, write to the client outside it (see _send_tar). with self.repo_lock: data = next(pipeline.fetch_many([entry], ro_type=ROBJ_FILE_STREAM, replacement_chunk=False))